diff --git a/docs/_ldio/ldio-adapters/ldio-json-to-json-ld.md b/docs/_ldio/ldio-adapters/ldio-json-to-json-ld.md index a0eb0f35e..905dcce46 100644 --- a/docs/_ldio/ldio-adapters/ldio-json-to-json-ld.md +++ b/docs/_ldio/ldio-adapters/ldio-json-to-json-ld.md @@ -5,6 +5,7 @@ title: Json To JsonLd Transformer --- # LDIO Json To JsonLd Adapter + ***Ldio:JsonToLdAdapter*** The json-to-ld-adapter receives json messages and adds a linked data context to transform the messages to json-ld. @@ -14,4 +15,5 @@ The json-to-ld-adapter receives json messages and adds a linked data context to | Property | Description | Required | Default | Example | Supported values | |:----------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------|:---------|:--------|:--------------------------|:-------------------------------------------------| | _context_ | URI of json-ld context Or an JSON-LD context object. | Yes | N/A | http://example.com/my-api | URI or Json Object (containing "@context" entry) | +| _force-content-type_ | Flag that indicates if `application/json` should be forced as mime type. | No | false | true | true or false | | _max-jsonld-cache-capacity_ | After retrieving an external JSON-LD context, it is cached for reuse. This property allows to specify the size of this cache (number of stored contexts). | No | 100 | 100 | Integer | diff --git a/ldi-api/pom.xml b/ldi-api/pom.xml index 2750612ef..cf00179bf 100644 --- a/ldi-api/pom.xml +++ b/ldi-api/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-api/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/types/LdiOneToManyTransformer.java b/ldi-api/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/types/LdiOneToManyTransformer.java index abbc04c32..2e853649f 100644 --- a/ldi-api/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/types/LdiOneToManyTransformer.java +++ b/ldi-api/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/types/LdiOneToManyTransformer.java @@ -4,6 +4,12 @@ import java.util.List; +/** + * The LDI Transformer provides a user to take a linked data model (RDF) from + * the pipeline and perform transformations onto it, which can result in many linked data models. + * Afterwards, each of these models will be put back onto the pipeline towards a next transformer or an LDI Output. + */ +@FunctionalInterface public interface LdiOneToManyTransformer extends LdiComponent { List transform(Model model); } diff --git a/ldi-api/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/types/LdiOneToOneTransformer.java b/ldi-api/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/types/LdiOneToOneTransformer.java index 5d8f5cb8e..bf60773d4 100644 --- a/ldi-api/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/types/LdiOneToOneTransformer.java +++ b/ldi-api/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/types/LdiOneToOneTransformer.java @@ -8,6 +8,7 @@ * onto it. Afterwards, this model will be put back onto the pipeline towards a * next transformer or an LDI Output. */ +@FunctionalInterface public interface LdiOneToOneTransformer extends LdiComponent { Model transform(Model model); } diff --git a/ldi-core/change-detection-filter/pom.xml b/ldi-core/change-detection-filter/pom.xml index 788ec4a7e..ec37f0188 100644 --- a/ldi-core/change-detection-filter/pom.xml +++ b/ldi-core/change-detection-filter/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT change-detection-filter diff --git a/ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/ChangeDetectionFilter.java b/ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/ChangeDetectionFilter.java index 0e5db3c96..aaa0c62a4 100644 --- a/ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/ChangeDetectionFilter.java +++ b/ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/ChangeDetectionFilter.java @@ -36,6 +36,7 @@ public ChangeDetectionFilter(HashedStateMemberRepository hashedStateMemberReposi /** * Filters out the model by returning an empty model when the model's hash has already been processed + * * @param model The model to be filtered * @return Either the same model if not processed yet, otherwise an empty model */ @@ -46,15 +47,18 @@ public Model transform(Model model) { canonicalizeInputModel(model, outputStream); String hashedModel = hashModelBytes(outputStream.toByteArray()); final HashedStateMember hashedStateMember = new HashedStateMember(subject.getURI(), hashedModel); - if(hashedStateMemberRepository.containsHashedStateMember(hashedStateMember)) { + if (hashedStateMemberRepository.containsHashedStateMember(hashedStateMember)) { return ModelFactory.createDefaultModel(); } hashedStateMemberRepository.saveHashedStateMember(hashedStateMember); return model; } + /** + * Clean up the resources, should be called when the component is not used anymore + */ public void destroyState() { - if(!keepState) { + if (!keepState) { hashedStateMemberRepository.destroyState(); } } diff --git a/ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/repositories/HashedStateMemberRepository.java b/ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/repositories/HashedStateMemberRepository.java index 870e3deb9..722e2eb34 100644 --- a/ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/repositories/HashedStateMemberRepository.java +++ b/ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/repositories/HashedStateMemberRepository.java @@ -2,10 +2,22 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.entities.HashedStateMember; +/** + * Repository to maintain the HashedStateMember objects into the database + */ public interface HashedStateMemberRepository { + /** + * Checks if the database already contain a record that contains the exact same HashedStateMember + * + * @param hashedStateMember representation object that contains a memberId and the hash of the state member + * @return true if an exact same record has found, otherwise false + */ boolean containsHashedStateMember(HashedStateMember hashedStateMember); void saveHashedStateMember(HashedStateMember hashedStateMember); + /** + * Clean up the resources, should be called when the repository is not used anymore + */ void destroyState(); } diff --git a/ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/repositories/mapper/HashedStateMemberEntityMapper.java b/ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/repositories/mapper/HashedStateMemberEntityMapper.java index 8144921c7..1476191fa 100644 --- a/ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/repositories/mapper/HashedStateMemberEntityMapper.java +++ b/ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/repositories/mapper/HashedStateMemberEntityMapper.java @@ -7,6 +7,12 @@ public class HashedStateMemberEntityMapper { private HashedStateMemberEntityMapper() { } + /** + * Maps a HashedStateMember from a domain object to an entity object + * + * @param hashedStateMember the domain object + * @return the entity object + */ public static HashedStateMemberEntity fromHashedStateMember(HashedStateMember hashedStateMember) { return new HashedStateMemberEntity(hashedStateMember.memberId(), hashedStateMember.memberHash()); } diff --git a/ldi-core/file-archiving/pom.xml b/ldi-core/file-archiving/pom.xml index a2acdc4e0..5a1ad2cf9 100644 --- a/ldi-core/file-archiving/pom.xml +++ b/ldi-core/file-archiving/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT file-archiving diff --git a/ldi-core/geojson-to-wkt/pom.xml b/ldi-core/geojson-to-wkt/pom.xml index 76b825666..41168098c 100644 --- a/ldi-core/geojson-to-wkt/pom.xml +++ b/ldi-core/geojson-to-wkt/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT geojson-to-wkt diff --git a/ldi-core/geojson-to-wkt/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/GeoJsonToWktTransformer.java b/ldi-core/geojson-to-wkt/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/GeoJsonToWktTransformer.java index 090361088..16a889d4b 100644 --- a/ldi-core/geojson-to-wkt/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/GeoJsonToWktTransformer.java +++ b/ldi-core/geojson-to-wkt/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/GeoJsonToWktTransformer.java @@ -12,10 +12,18 @@ import static be.vlaanderen.informatievlaanderen.ldes.ldi.WktConverter.GEOJSON_GEOMETRY; +/** + * Ldi Transformer components that is in fact a wrapper around a GeoJsonConverter + */ public class GeoJsonToWktTransformer implements LdiOneToOneTransformer { private final GeoJsonConverter geoJsonConverter; + /** + * Constructs either around a converter that converts the GeoJson to simple WKT or to WKT in RDF format + * + * @param transformToRdfWkt boolean that determines whether to convert the input to RDF+WKT if true or to simple RDF if false + */ public GeoJsonToWktTransformer(boolean transformToRdfWkt) { this.geoJsonConverter = transformToRdfWkt ? new GeoJsonToRdfPlusWktConverter() : diff --git a/ldi-core/geojson-to-wkt/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/converter/GeoJsonConverter.java b/ldi-core/geojson-to-wkt/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/converter/GeoJsonConverter.java index d5101aeb9..3ce92616b 100644 --- a/ldi-core/geojson-to-wkt/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/converter/GeoJsonConverter.java +++ b/ldi-core/geojson-to-wkt/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/converter/GeoJsonConverter.java @@ -7,6 +7,12 @@ public interface GeoJsonConverter { + /** + * Creates new Geometry statements based on + * @param statement + * @param geometryModel + * @return + */ List createNewGeometryStatements(Statement statement, Model geometryModel); } diff --git a/ldi-core/json-to-ld-adapter/pom.xml b/ldi-core/json-to-ld-adapter/pom.xml index 2748cd921..c7ca51a5d 100644 --- a/ldi-core/json-to-ld-adapter/pom.xml +++ b/ldi-core/json-to-ld-adapter/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT json-to-ld-adapter diff --git a/ldi-core/json-to-ld-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/JsonToLdAdapter.java b/ldi-core/json-to-ld-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/JsonToLdAdapter.java index 2c19bfb07..462ab1658 100644 --- a/ldi-core/json-to-ld-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/JsonToLdAdapter.java +++ b/ldi-core/json-to-ld-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/JsonToLdAdapter.java @@ -15,6 +15,9 @@ import java.util.stream.Stream; +/** + * LdiAdapter implementation that adapts json to json-ld by adding a context to the json data + */ public class JsonToLdAdapter implements LdiAdapter { private final Logger log = LoggerFactory.getLogger(JsonToLdAdapter.class); @@ -34,6 +37,13 @@ public JsonToLdAdapter(String context, boolean forceContentType, Context jenaCon this.jenaContext = jenaContext; } + /** + * Implementation of the adaptation function that converts json to json-ld + * + * @param content the json content, should have application/json as mime-type + * @return the data adapted to json-ld + * @throws UnsupportedMimeTypeException when an invalid mime-type is provided and application/json is not forced + */ @Override public Stream apply(Content content) { if (!validateMimeType(content.mimeType())) { @@ -63,7 +73,7 @@ private Stream translateJsonToLD(String data) { } throw new IllegalArgumentException("Only objects and arrays can be transformed to RDF. " + - "The following json does not match this criteria: " + json); + "The following json does not match this criteria: " + json); } catch (JsonParseException e) { throw new ParseToJsonException(e, data); } @@ -81,7 +91,7 @@ private Model mapJsonObjectToModel(JsonValue json) { return model; } else { throw new IllegalArgumentException("Only objects can be transformed to RDF. " + - "The following json does not match this criteria: " + json); + "The following json does not match this criteria: " + json); } } diff --git a/ldi-core/ldes-client/pom.xml b/ldi-core/ldes-client/pom.xml index 309264459..5541d4fea 100644 --- a/ldi-core/ldes-client/pom.xml +++ b/ldi-core/ldes-client/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/ldes-client/starting-node/pom.xml b/ldi-core/ldes-client/starting-node/pom.xml index 776ac534c..3cad5c7aa 100644 --- a/ldi-core/ldes-client/starting-node/pom.xml +++ b/ldi-core/ldes-client/starting-node/pom.xml @@ -5,7 +5,7 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 starting-node diff --git a/ldi-core/ldes-client/starting-node/src/main/java/ldes/client/startingtreenode/RedirectRequestExecutor.java b/ldi-core/ldes-client/starting-node/src/main/java/ldes/client/startingtreenode/RedirectRequestExecutor.java index 06afd9db5..a709f107b 100644 --- a/ldi-core/ldes-client/starting-node/src/main/java/ldes/client/startingtreenode/RedirectRequestExecutor.java +++ b/ldi-core/ldes-client/starting-node/src/main/java/ldes/client/startingtreenode/RedirectRequestExecutor.java @@ -11,6 +11,9 @@ import java.util.List; +/** + * Wrapper around a RequestExecutor instance to more easily handle redirect responses + */ public class RedirectRequestExecutor { private final RequestExecutor requestExecutor; diff --git a/ldi-core/ldes-client/starting-node/src/main/java/ldes/client/startingtreenode/StartingTreeNodeRelationsFinder.java b/ldi-core/ldes-client/starting-node/src/main/java/ldes/client/startingtreenode/StartingTreeNodeRelationsFinder.java index a857fe796..dccd07c57 100644 --- a/ldi-core/ldes-client/starting-node/src/main/java/ldes/client/startingtreenode/StartingTreeNodeRelationsFinder.java +++ b/ldi-core/ldes-client/starting-node/src/main/java/ldes/client/startingtreenode/StartingTreeNodeRelationsFinder.java @@ -30,6 +30,12 @@ public StartingTreeNodeRelationsFinder(RequestExecutor requestExecutor) { this.requestExecutor = new RedirectRequestExecutor(requestExecutor); } + /** + * Determines all starting relations that needs to be queued. + * + * @param startingNodeRequest can contain a collection, view or treeNode. + * @return all nodes that must be further queued by the discoverer + */ public List findAllStartingTreeNodes(final StartingNodeRequest startingNodeRequest) { final Response response = requestExecutor.execute(startingNodeRequest); final Model model = getModelFromResponse(startingNodeRequest.lang(), response.getBody().orElseThrow(), startingNodeRequest.url()); diff --git a/ldi-core/ldes-client/starting-node/src/main/java/ldes/client/startingtreenode/domain/valueobjects/StartingNodeSpecification.java b/ldi-core/ldes-client/starting-node/src/main/java/ldes/client/startingtreenode/domain/valueobjects/StartingNodeSpecification.java index 020a4ebcb..c74862069 100644 --- a/ldi-core/ldes-client/starting-node/src/main/java/ldes/client/startingtreenode/domain/valueobjects/StartingNodeSpecification.java +++ b/ldi-core/ldes-client/starting-node/src/main/java/ldes/client/startingtreenode/domain/valueobjects/StartingNodeSpecification.java @@ -4,6 +4,9 @@ import java.util.function.Predicate; +/** + * Interface to declare the specification of the provided starting node of an LDES + */ public interface StartingNodeSpecification extends Predicate { StartingTreeNode extractStartingNode(Model model); diff --git a/ldi-core/ldes-client/starting-node/src/main/java/ldes/client/startingtreenode/domain/valueobjects/TreeNodeSpecification.java b/ldi-core/ldes-client/starting-node/src/main/java/ldes/client/startingtreenode/domain/valueobjects/TreeNodeSpecification.java index 1d7565f19..f6b1999ca 100644 --- a/ldi-core/ldes-client/starting-node/src/main/java/ldes/client/startingtreenode/domain/valueobjects/TreeNodeSpecification.java +++ b/ldi-core/ldes-client/starting-node/src/main/java/ldes/client/startingtreenode/domain/valueobjects/TreeNodeSpecification.java @@ -9,6 +9,9 @@ import static org.apache.jena.rdf.model.ResourceFactory.createProperty; import static org.apache.jena.rdf.model.ResourceFactory.createResource; +/** + * Used when the starting node is already part of a view, or in other words, a fragment + */ public class TreeNodeSpecification implements StartingNodeSpecification { public static final String TREE = "https://w3id.org/tree#"; public static final Resource TREE_NODE_RESOURCE = createResource(TREE + "Node"); diff --git a/ldi-core/ldes-client/starting-node/src/main/java/ldes/client/startingtreenode/domain/valueobjects/ViewSpecification.java b/ldi-core/ldes-client/starting-node/src/main/java/ldes/client/startingtreenode/domain/valueobjects/ViewSpecification.java index d5d0c5590..ceaf890ab 100644 --- a/ldi-core/ldes-client/starting-node/src/main/java/ldes/client/startingtreenode/domain/valueobjects/ViewSpecification.java +++ b/ldi-core/ldes-client/starting-node/src/main/java/ldes/client/startingtreenode/domain/valueobjects/ViewSpecification.java @@ -9,6 +9,9 @@ import static org.apache.jena.rdf.model.ResourceFactory.createProperty; +/** + * Used when the provided starting node of the LDES is the LDES definition itself or the tree:view definition + */ public class ViewSpecification implements StartingNodeSpecification { public static final String TREE = "https://w3id.org/tree#"; public static final Property TREE_VIEW = createProperty(TREE, "view"); diff --git a/ldi-core/ldes-client/tree-node-fetcher/pom.xml b/ldi-core/ldes-client/tree-node-fetcher/pom.xml index a39706121..f9ece0352 100644 --- a/ldi-core/ldes-client/tree-node-fetcher/pom.xml +++ b/ldi-core/ldes-client/tree-node-fetcher/pom.xml @@ -5,7 +5,7 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT diff --git a/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/TreeNodeFetcher.java b/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/TreeNodeFetcher.java index 46e18e145..7d70c0819 100644 --- a/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/TreeNodeFetcher.java +++ b/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/TreeNodeFetcher.java @@ -16,6 +16,9 @@ import java.time.LocalDateTime; import java.util.List; +/** + * Responsible for fetching the next TreeNodes + */ public class TreeNodeFetcher { private static final LocalDateTime maxSupportedDateTime = LocalDateTime.of(294276, 12, 31, 23, 59, 59); @@ -27,6 +30,10 @@ public TreeNodeFetcher(RequestExecutor requestExecutor, TimestampExtractor times this.timestampExtractor = timestampExtractor; } + /** + * @param treeNodeRequest based on the relations found in the previous TreeNode + * @return the new TreeNode with all its information + */ public TreeNodeResponse fetchTreeNode(TreeNodeRequest treeNodeRequest) { final Response response = requestExecutor.execute(treeNodeRequest.createRequest()); diff --git a/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/domain/valueobjects/ModelResponse.java b/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/domain/valueobjects/ModelResponse.java index ec85d45df..08203c2f9 100644 --- a/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/domain/valueobjects/ModelResponse.java +++ b/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/domain/valueobjects/ModelResponse.java @@ -13,8 +13,11 @@ import static ldes.client.treenodefetcher.domain.valueobjects.Constants.*; +/** + * Wrapper around the RDF response to more easily extract the required information from that response + */ public class ModelResponse { - private TimestampExtractor timestampExtractor; + private final TimestampExtractor timestampExtractor; private final ModelExtract modelExtract = new ModelExtract(new StatementTripleBoundary(TripleBoundary.stopNowhere)); private final Model model; diff --git a/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/domain/valueobjects/MutabilityStatus.java b/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/domain/valueobjects/MutabilityStatus.java index 79a7e31eb..e0a367567 100644 --- a/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/domain/valueobjects/MutabilityStatus.java +++ b/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/domain/valueobjects/MutabilityStatus.java @@ -3,6 +3,10 @@ import java.time.LocalDateTime; import java.util.Arrays; +/** + * Keeps track when the received http response is about to change, is in fact a reflection of the + * Cache-Control header + */ public class MutabilityStatus { public static final int DEFAULT_MAX_AGE = 60; diff --git a/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/domain/valueobjects/TreeNodeRequest.java b/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/domain/valueobjects/TreeNodeRequest.java index 3ba5914b9..b123706dc 100644 --- a/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/domain/valueobjects/TreeNodeRequest.java +++ b/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/domain/valueobjects/TreeNodeRequest.java @@ -9,6 +9,9 @@ import java.util.List; +/** + * Contains the endpoint to connect to the server. This can only be a fragment + */ public class TreeNodeRequest { private final String treeNodeUrl; private final Lang lang; diff --git a/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/domain/valueobjects/TreeNodeResponse.java b/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/domain/valueobjects/TreeNodeResponse.java index 57fc267ea..88d61885b 100644 --- a/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/domain/valueobjects/TreeNodeResponse.java +++ b/ldi-core/ldes-client/tree-node-fetcher/src/main/java/ldes/client/treenodefetcher/domain/valueobjects/TreeNodeResponse.java @@ -4,13 +4,17 @@ import java.util.List; +/** + * Wrapper around the RDF response that represents a fragment to more easily extract the required information + * from that response + */ public class TreeNodeResponse { private final List relation; private final List members; private final MutabilityStatus mutabilityStatus; public TreeNodeResponse(List relations, List members, - MutabilityStatus mutabilityStatus) { + MutabilityStatus mutabilityStatus) { this.relation = relations; this.members = members; this.mutabilityStatus = mutabilityStatus; diff --git a/ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml b/ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml index fea4d06b2..cf333374a 100644 --- a/ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml +++ b/ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml @@ -5,7 +5,7 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscoverer.java b/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscoverer.java index 1578ed813..f5a97213a 100644 --- a/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscoverer.java +++ b/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/LdesStructureDiscoverer.java @@ -10,6 +10,9 @@ import java.util.List; +/** + * Responsible for discovering the entire structure below a provided starting uri + */ public class LdesStructureDiscoverer { private static final Logger log = LoggerFactory.getLogger(LdesStructureDiscoverer.class); private final String startingUrl; @@ -22,6 +25,9 @@ public LdesStructureDiscoverer(String startingUrl, Lang sourceFormat, RequestExe this.requestExecutor = requestExecutor; } + /** + * @return the entire structure of the (sub)set below the starting uri + */ public LdesStructure discoverLdesStructure() { final LdesStructure ldesStructure = new LdesStructure(startingUrl); getStartingTreeRelations() diff --git a/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/StartingTreeNodeUrlSupplier.java b/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/StartingTreeNodeUrlSupplier.java index 71327d6ce..19c06b79a 100644 --- a/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/StartingTreeNodeUrlSupplier.java +++ b/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/StartingTreeNodeUrlSupplier.java @@ -16,6 +16,13 @@ public StartingTreeNodeUrlSupplier(RequestExecutor requestExecutor) { startingTreeNodeRelationsFinder = new StartingTreeNodeRelationsFinder(requestExecutor); } + /** + * Fetching the first relations of a (sub)set below an uri + * + * @param url the starting uri + * @param lang the RDF format in which the response is received + * @return a list of the first relations below the starting uri + */ public List getStartingTreeNodeRelations(String url, Lang lang) { final StartingNodeRequest startingNodeRequest = new StartingNodeRequest(url, lang, new RedirectHistory()); return startingTreeNodeRelationsFinder diff --git a/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/TreeRelationsFetcher.java b/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/TreeRelationsFetcher.java index 52a6d12af..64cadc6ad 100644 --- a/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/TreeRelationsFetcher.java +++ b/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/TreeRelationsFetcher.java @@ -18,6 +18,13 @@ public class TreeRelationsFetcher { public TreeRelationsFetcher(RequestExecutor requestExecutor) { this.requestExecutor = requestExecutor; } + + /** + * Fetches a new TreeNode and extract all the relations out of it + * + * @param treeNodeRequest based on the relations found in the previous TreeNode + * @return a new list of all the next TreeNode relations that were present in the received TreeNode + */ public List fetchTreeRelations(TreeNodeRequest treeNodeRequest) throws UnsupportedOperationException { final Response response = requestExecutor.execute(treeNodeRequest.createRequest()); diff --git a/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/domain/valueobjects/LdesRelation.java b/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/domain/valueobjects/LdesRelation.java index 6da87a91e..59f18e01c 100644 --- a/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/domain/valueobjects/LdesRelation.java +++ b/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/domain/valueobjects/LdesRelation.java @@ -2,12 +2,43 @@ import java.util.List; +/** + * Representation of a relation in an LDES + *
+ * Is in fact the component interface in the composite pattern + */ public interface LdesRelation { + /** + * Add a relation to this relation + * + * @param ldesRelation the relation to add to an existing relation + */ void addRelation(LdesRelation ldesRelation); - boolean containsChild(LdesRelation child); + + /** + * Count all the relations below this relation, including all the relations of their child relations and so on + * + * @return the total number of relations of and their child relations + */ int countTotalRelations(); + + /** + * @return the number of relations this relation contains + */ int countChildRelations(); + + /** + * @return list of all the relations this relation has + */ List getRelations(); + + /** + * @return the uri of this relation + */ String getUri(); + + /** + * @return string representation of this relation and their child relations + */ String asString(); } diff --git a/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/domain/valueobjects/LdesStructure.java b/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/domain/valueobjects/LdesStructure.java index 129acdb10..d2bf9c143 100644 --- a/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/domain/valueobjects/LdesStructure.java +++ b/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/domain/valueobjects/LdesStructure.java @@ -5,6 +5,11 @@ import java.util.ArrayList; import java.util.List; +/** + * Representation of all the relations of an LDES + *
+ * Is in fact the composite in the composite pattern + */ public class LdesStructure implements LdesRelation { private final String rootUrl; private final List relations; @@ -19,11 +24,6 @@ public void addRelation(LdesRelation ldesRelation) { relations.add(ldesRelation); } - @Override - public boolean containsChild(LdesRelation child) { - return relations.contains(child); - } - @Override public int countTotalRelations() { return countChildRelations() + relations.stream().mapToInt(LdesRelation::countTotalRelations).sum(); @@ -49,8 +49,7 @@ public String getUri() { */ @Override public String asString() { - final LdesRelationWriter ldesRelationWriter = new LdesRelationWriter(); - return ldesRelationWriter.writeToString(this); + return new LdesRelationWriter().writeToString(this); } /** diff --git a/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/domain/valueobjects/ModelResponse.java b/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/domain/valueobjects/ModelResponse.java index 65df554c8..873cc29aa 100644 --- a/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/domain/valueobjects/ModelResponse.java +++ b/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/domain/valueobjects/ModelResponse.java @@ -9,6 +9,9 @@ import static org.apache.jena.rdf.model.ResourceFactory.createProperty; +/** + * Wrapper around the RDF TreeNode response to more easily extract the relations from that response + */ public class ModelResponse { private static final Resource ANY_RESOURCE = null; private static final String W3C_TREE = "https://w3id.org/tree#"; diff --git a/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/domain/valueobjects/TreeNodeRequest.java b/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/domain/valueobjects/TreeNodeRequest.java index 752f82c7c..925618ae8 100644 --- a/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/domain/valueobjects/TreeNodeRequest.java +++ b/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/domain/valueobjects/TreeNodeRequest.java @@ -9,6 +9,9 @@ import java.util.List; +/** + * Contains the endpoint to connect to the server. This can be only be a fragment + */ public class TreeNodeRequest { private final String treeNodeUrl; private final Lang lang; diff --git a/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/domain/valueobjects/TreeRelation.java b/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/domain/valueobjects/TreeRelation.java index 576b6f264..8174dc6f3 100644 --- a/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/domain/valueobjects/TreeRelation.java +++ b/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/domain/valueobjects/TreeRelation.java @@ -10,6 +10,11 @@ import static org.apache.jena.rdf.model.ResourceFactory.createProperty; +/** + * Represents all the relations of a single TreeNode of a LDES + *
+ * Is in fact a leaf in the composite pattern + */ public class TreeRelation implements LdesRelation { private static final String W3C_TREE = "https://w3id.org/tree#"; private static final Property W3ID_TREE_NODE = createProperty(W3C_TREE, "node"); @@ -42,11 +47,6 @@ public void addRelation(LdesRelation ldesRelation) { relations.add(ldesRelation); } - @Override - public boolean containsChild(LdesRelation child) { - return relations.contains(child); - } - @Override public int countTotalRelations() { return countChildRelations() + relations.stream().mapToInt(LdesRelation::countTotalRelations).sum(); diff --git a/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/services/LdesRelationWriter.java b/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/services/LdesRelationWriter.java index f8bd1874a..6b6c06670 100644 --- a/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/services/LdesRelationWriter.java +++ b/ldi-core/ldes-client/tree-node-relations-fetcher/src/main/java/ldes/client/treenoderelationsfetcher/services/LdesRelationWriter.java @@ -5,6 +5,9 @@ import java.util.Comparator; import java.util.List; +/** + * Writer utility class that is responsible for writing an LDES relation into a tree kind string representation + */ public class LdesRelationWriter { private static final String RELATION_PREFIX = "+- "; private static final String RELATIONS_CONNECTION_STRING = "| "; diff --git a/ldi-core/ldes-client/tree-node-supplier/pom.xml b/ldi-core/ldes-client/tree-node-supplier/pom.xml index e3491404a..43f32806d 100644 --- a/ldi-core/ldes-client/tree-node-supplier/pom.xml +++ b/ldi-core/ldes-client/tree-node-supplier/pom.xml @@ -5,7 +5,7 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 tree-node-supplier diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/entities/MemberRecord.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/entities/MemberRecord.java index 6a4053f85..0e2d4e2d3 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/entities/MemberRecord.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/entities/MemberRecord.java @@ -10,8 +10,8 @@ public class MemberRecord implements Comparable{ private final String memberId; - private LocalDateTime createdAt; - private Model model; + private final LocalDateTime createdAt; + private final Model model; public MemberRecord(String memberId, Model model, LocalDateTime createdAt) { this.memberId = memberId; diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/entities/MemberVersionRecord.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/entities/MemberVersionRecord.java index eecabd085..a1da63acd 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/entities/MemberVersionRecord.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/entities/MemberVersionRecord.java @@ -3,6 +3,9 @@ import java.time.LocalDateTime; import java.util.Objects; +/** + * Record that represents the versionOf and timestamp of a specific member + */ public class MemberVersionRecord { private final String versionOf; private final LocalDateTime timestamp; diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/entities/TreeNodeRecord.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/entities/TreeNodeRecord.java index fb874ad86..ae0584b43 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/entities/TreeNodeRecord.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/entities/TreeNodeRecord.java @@ -12,13 +12,10 @@ public class TreeNodeRecord { private final String treeNodeUrl; private TreeNodeStatus treeNodeStatus; private LocalDateTime earliestNextVisit; - private List memberIds; + private final List memberIds; public TreeNodeRecord(String treeNodeUrl) { - this.treeNodeUrl = treeNodeUrl; - this.treeNodeStatus = TreeNodeStatus.NOT_VISITED; - this.earliestNextVisit = LocalDateTime.now(); - this.memberIds = new ArrayList<>(); + this(treeNodeUrl, TreeNodeStatus.NOT_VISITED, LocalDateTime.now(), new ArrayList<>()); } public TreeNodeRecord(String treeNodeUrl, TreeNodeStatus treeNodeStatus, LocalDateTime earliestNextVisit, List memberIds) { @@ -32,17 +29,33 @@ public String getTreeNodeUrl() { return treeNodeUrl; } + /** + * @return a representation of how much of this TreeNode has been processed + */ public TreeNodeStatus getTreeNodeStatus() { return treeNodeStatus; } + /** + * Keeps track when this TreeNode could be changed at the earliest, fetching this TreeNode again before this + * timestamp is pointless + * + * @return a timestamp suggesting when to fetch this TreeNode again + */ public LocalDateTime getEarliestNextVisit() { return earliestNextVisit; } + + /** + * @return a list of all the id of all the members that are part of this TreeNode + */ public List getMemberIds() { return memberIds; } + /** + * Updates the TreeNodeStatus based on the mutabilityStatus received from the HTTP response + */ public void updateStatus(MutabilityStatus mutabilityStatus) { if (mutabilityStatus.isMutable()) { treeNodeStatus = TreeNodeStatus.MUTABLE_AND_ACTIVE; @@ -52,14 +65,31 @@ public void updateStatus(MutabilityStatus mutabilityStatus) { earliestNextVisit = mutabilityStatus.getEarliestNextVisit(); } - public boolean hasReceived(String id) { - return memberIds.contains(id); + /** + * Check whether the member has already been received + * + * @param memberId the id of the member that needs to be checked + * @return true if the member is new, false if the member has already been received before + */ + public boolean hasReceived(String memberId) { + return memberIds.contains(memberId); } + /** + * Add newly received members to this TreeNode + * + * @param receivedMemberIds the ids of the new members + */ public void addToReceived(List receivedMemberIds) { memberIds.addAll(receivedMemberIds); } + /** + * Marks this TreeNode as completely processed. + *
+ * To save some resources, the list of members is cleared, as it does not matter anymore which members of + * this TreeNode have been processed + */ public void markImmutableWithoutUnprocessedMembers() { memberIds.clear(); treeNodeStatus = TreeNodeStatus.IMMUTABLE_WITHOUT_UNPROCESSED_MEMBERS; diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberIdRepositoryFactory.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberIdRepositoryFactory.java index 86e498123..436355f40 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberIdRepositoryFactory.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberIdRepositoryFactory.java @@ -13,6 +13,12 @@ public class MemberIdRepositoryFactory { private MemberIdRepositoryFactory() { } + /** + * @param statePersistenceStrategy via what persistence strategy the repository should work + * @param properties a representation of the required config properties to set up the persistence unit + * @param instanceName will be used to be able to more easily keep track of the return repo + * @return the memberIdRepository for a specific instance (could be a NiFi flow or a LDIO pipeline) + */ public static MemberIdRepository getMemberIdRepository(StatePersistenceStrategy statePersistenceStrategy, HibernateProperties properties, String instanceName) { return switch (statePersistenceStrategy) { diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberRepositoryFactory.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberRepositoryFactory.java index 1a4fe2c9f..9f5154b24 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberRepositoryFactory.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberRepositoryFactory.java @@ -13,6 +13,12 @@ public class MemberRepositoryFactory { private MemberRepositoryFactory() { } + /** + * @param statePersistenceStrategy via what persistence strategy the repository should work + * @param properties a representation of the required config properties to set up the persistence unit + * @param instanceName will be used to be able to more easily keep track of the return repo + * @return the memberNRepository for a specific instance (could be a NiFi flow or a LDIO pipeline) + */ public static MemberRepository getMemberRepository(StatePersistenceStrategy statePersistenceStrategy, HibernateProperties properties, String instanceName) { return switch (statePersistenceStrategy) { diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberVersionRepositoryFactory.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberVersionRepositoryFactory.java index 81147876a..10fd541e3 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberVersionRepositoryFactory.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberVersionRepositoryFactory.java @@ -10,15 +10,23 @@ public class MemberVersionRepositoryFactory { - private MemberVersionRepositoryFactory() { - } + private MemberVersionRepositoryFactory() { + } - public static MemberVersionRepository getMemberVersionRepositoryFactory(StatePersistenceStrategy statePersistenceStrategy, - HibernateProperties properties, String instanceName) { - return switch (statePersistenceStrategy) { - case SQLITE -> new SqlMemberVersionRepository(SqliteEntityManagerFactory.getInstance(properties), instanceName); - case MEMORY -> new InMemoryMemberVersionRepository(); - case POSTGRES -> new SqlMemberVersionRepository(PostgresEntityManagerFactory.getInstance(instanceName, properties.getProperties()), instanceName); - }; - } + /** + * @param statePersistenceStrategy via what persistence strategy the repository should work + * @param properties a representation of the required config properties to set up the persistence unit + * @param instanceName will be used to be able to more easily keep track of the return repo + * @return the memberVersionRepository for a specific instance (could be a NiFi flow or a LDIO pipeline) + */ + public static MemberVersionRepository getMemberVersionRepositoryFactory(StatePersistenceStrategy statePersistenceStrategy, + HibernateProperties properties, String instanceName) { + return switch (statePersistenceStrategy) { + case SQLITE -> + new SqlMemberVersionRepository(SqliteEntityManagerFactory.getInstance(properties), instanceName); + case MEMORY -> new InMemoryMemberVersionRepository(); + case POSTGRES -> + new SqlMemberVersionRepository(PostgresEntityManagerFactory.getInstance(instanceName, properties.getProperties()), instanceName); + }; + } } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/TreeNodeRecordComparator.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/TreeNodeRecordComparator.java index 4492f700d..a923e355c 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/TreeNodeRecordComparator.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/TreeNodeRecordComparator.java @@ -2,9 +2,17 @@ import ldes.client.treenodesupplier.domain.entities.TreeNodeRecord; +import java.time.chrono.ChronoLocalDateTime; import java.util.Comparator; public class TreeNodeRecordComparator implements Comparator { + /** + * Compares two TreeNodeRecords by {@link TreeNodeRecord#getEarliestNextVisit()} + * @param o1 the first object to be compared. + * @param o2 the second object to be compared. + * @return integer comparing result + * @see java.time.LocalDateTime#compareTo(ChronoLocalDateTime) + */ @Override public int compare(TreeNodeRecord o1, TreeNodeRecord o2) { return o1.getEarliestNextVisit().compareTo(o2.getEarliestNextVisit()); diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/TreeNodeRecordRepositoryFactory.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/TreeNodeRecordRepositoryFactory.java index 042c24b71..0aedc632a 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/TreeNodeRecordRepositoryFactory.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/TreeNodeRecordRepositoryFactory.java @@ -12,6 +12,12 @@ public class TreeNodeRecordRepositoryFactory { private TreeNodeRecordRepositoryFactory() { } + /** + * @param statePersistenceStrategy via what persistence strategy the repository should work + * @param properties a representation of the required config properties to set up the persistence unit + * @param instanceName will be used to be able to more easily keep track of the return repo + * @return the treeNodeRecordRepository for a specific instance (could be a NiFi flow or a LDIO pipeline) + */ public static TreeNodeRecordRepository getTreeNodeRecordRepository( StatePersistenceStrategy statePersistenceStrategy, HibernateProperties properties, String instanceName) { return switch (statePersistenceStrategy) { diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/StartingTreeNode.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/StartingTreeNode.java index 4136b908a..9834bb1f3 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/StartingTreeNode.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/StartingTreeNode.java @@ -2,6 +2,9 @@ import org.apache.jena.riot.Lang; +/** + * Contains the rootNode endpoint information to start the client and the expected RDF format for the response + */ public class StartingTreeNode { private final String startingNodeUrl; diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/StatePersistence.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/StatePersistence.java index fa596aeca..1f84fc065 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/StatePersistence.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/StatePersistence.java @@ -2,10 +2,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.HibernateProperties; import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.StatePersistenceStrategy; -import ldes.client.treenodesupplier.domain.services.MemberIdRepositoryFactory; -import ldes.client.treenodesupplier.domain.services.MemberRepositoryFactory; -import ldes.client.treenodesupplier.domain.services.MemberVersionRepositoryFactory; -import ldes.client.treenodesupplier.domain.services.TreeNodeRecordRepositoryFactory; +import ldes.client.treenodesupplier.domain.services.*; import ldes.client.treenodesupplier.repository.MemberIdRepository; import ldes.client.treenodesupplier.repository.MemberRepository; import ldes.client.treenodesupplier.repository.MemberVersionRepository; @@ -13,17 +10,17 @@ public class StatePersistence { - private final MemberRepository memberRepository; - private final MemberIdRepository memberIdRepository; - private final TreeNodeRecordRepository treeNodeRecordRepository; - private final MemberVersionRepository memberVersionRepository; + private final MemberRepository memberRepository; + private final MemberIdRepository memberIdRepository; + private final TreeNodeRecordRepository treeNodeRecordRepository; + private final MemberVersionRepository memberVersionRepository; - public StatePersistence(MemberRepository memberRepository, MemberIdRepository memberIdRepository, TreeNodeRecordRepository treeNodeRecordRepository, MemberVersionRepository memberVersionRepository) { - this.memberRepository = memberRepository; - this.memberIdRepository = memberIdRepository; - this.treeNodeRecordRepository = treeNodeRecordRepository; - this.memberVersionRepository = memberVersionRepository; - } + public StatePersistence(MemberRepository memberRepository, MemberIdRepository memberIdRepository, TreeNodeRecordRepository treeNodeRecordRepository, MemberVersionRepository memberVersionRepository) { + this.memberRepository = memberRepository; + this.memberIdRepository = memberIdRepository; + this.treeNodeRecordRepository = treeNodeRecordRepository; + this.memberVersionRepository = memberVersionRepository; + } public static StatePersistence from(StatePersistenceStrategy statePersistenceStrategy, HibernateProperties properties, String instanceName) { @@ -35,17 +32,17 @@ public static StatePersistence from(StatePersistenceStrategy statePersistenceStr MemberVersionRepositoryFactory.getMemberVersionRepositoryFactory(statePersistenceStrategy, properties, instanceName)); } - public MemberRepository getMemberRepository() { - return memberRepository; - } + public MemberRepository getMemberRepository() { + return memberRepository; + } - public MemberIdRepository getMemberIdRepository() { - return memberIdRepository; - } + public MemberIdRepository getMemberIdRepository() { + return memberIdRepository; + } - public TreeNodeRecordRepository getTreeNodeRecordRepository() { - return treeNodeRecordRepository; - } + public TreeNodeRecordRepository getTreeNodeRecordRepository() { + return treeNodeRecordRepository; + } public MemberVersionRepository getMemberVersionRepository() { return memberVersionRepository; diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/SuppliedMember.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/SuppliedMember.java index 285519e8a..d333d5e18 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/SuppliedMember.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/SuppliedMember.java @@ -2,6 +2,9 @@ import org.apache.jena.rdf.model.Model; +/** + * Wrapper around the received RDF model + */ public class SuppliedMember { private final String id; diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/TreeNodeStatus.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/TreeNodeStatus.java index 0d5fcc00c..8bfceea0a 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/TreeNodeStatus.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/TreeNodeStatus.java @@ -1,5 +1,8 @@ package ldes.client.treenodesupplier.domain.valueobject; +/** + * Representation how much of a TreeNode has been processed + */ public enum TreeNodeStatus { NOT_VISITED, MUTABLE_AND_ACTIVE, IMMUTABLE_WITH_UNPROCESSED_MEMBERS, IMMUTABLE_WITHOUT_UNPROCESSED_MEMBERS } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/filters/ExactlyOnceFilter.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/filters/ExactlyOnceFilter.java index d073969b2..977ac3be4 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/filters/ExactlyOnceFilter.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/filters/ExactlyOnceFilter.java @@ -3,10 +3,17 @@ import ldes.client.treenodesupplier.domain.valueobject.SuppliedMember; import ldes.client.treenodesupplier.repository.MemberIdRepository; +/** + * Filter that makes sure that a member is processed exactly once + */ public class ExactlyOnceFilter implements MemberFilter { private final MemberIdRepository memberIdRepository; private final boolean keepState; + /** + * @param memberIdRepository repository to keep track if a member already has been processed + * @param keepState if the state must be kept, or can be reset on restart of the filter + */ public ExactlyOnceFilter(MemberIdRepository memberIdRepository, boolean keepState) { this.memberIdRepository = memberIdRepository; this.keepState = keepState; @@ -22,9 +29,12 @@ public void saveAllowedMember(SuppliedMember member) { memberIdRepository.addMemberId(member.getId()); } + /** + * Clean up the database when the filter is not required anymore and the state must not be kept + */ @Override public void destroyState() { - if(!keepState) { + if (!keepState) { memberIdRepository.destroyState(); } } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/filters/LatestStateFilter.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/filters/LatestStateFilter.java index 91a899551..b9b4e3a50 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/filters/LatestStateFilter.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/filters/LatestStateFilter.java @@ -12,6 +12,9 @@ import static org.apache.jena.rdf.model.ResourceFactory.createProperty; +/** + * Filter that makes sure that a member represents only the latest state + */ public class LatestStateFilter implements MemberFilter { private final MemberVersionRepository memberVersionRepository; @@ -20,6 +23,12 @@ public class LatestStateFilter implements MemberFilter { private final String versionOfPath; + /** + * @param memberVersionRepository repository that keeps track of the latest processed versions of the members + * @param keepState if the state must be kept, or can be reset on restart of the filter + * @param timestampPath URI of the predicate that contains the timestamp + * @param versionOfPath URI of the predicate that contains the version-of + */ public LatestStateFilter(MemberVersionRepository memberVersionRepository, boolean keepState, String timestampPath, String versionOfPath) { this.memberVersionRepository = memberVersionRepository; this.keepState = keepState; @@ -41,9 +50,12 @@ public void saveAllowedMember(SuppliedMember member) { memberVersionRepository.addMemberVersion(new MemberVersionRecord(versionOf, timestamp)); } + /** + * Clean up the database when the filter is not required anymore and the state must not be kept + */ @Override public void destroyState() { - if(!keepState) { + if (!keepState) { memberVersionRepository.destroyState(); } } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/filters/MemberFilter.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/filters/MemberFilter.java index 2749d7077..b8f58358a 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/filters/MemberFilter.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/filters/MemberFilter.java @@ -2,10 +2,27 @@ import ldes.client.treenodesupplier.domain.valueobject.SuppliedMember; +/** + * Filter interface to filter out some supplied members + */ public interface MemberFilter { + /** + * Checks whether the supplied member can go through the filter + * + * @param member member under filtration test + * @return true if the member can go through the filter, otherwise false + */ boolean isAllowed(SuppliedMember member); + /** + * Saves the newly allowed member, as this can affect the filtration process + * + * @param member member to be saved + */ void saveAllowedMember(SuppliedMember member); + /** + * Release resources when the filter is not required anymore + */ void destroyState(); } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/membersuppliers/FilteredMemberSupplier.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/membersuppliers/FilteredMemberSupplier.java index f2ae09fc3..e05ed01d1 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/membersuppliers/FilteredMemberSupplier.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/membersuppliers/FilteredMemberSupplier.java @@ -5,6 +5,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * This is a decorator for the {@link MemberSupplier} which makes it possible filter out some members before + * supplying them + */ public class FilteredMemberSupplier extends MemberSupplierDecorator { private static final Logger log = LoggerFactory.getLogger(FilteredMemberSupplier.class); private final MemberFilter filter; @@ -15,6 +19,12 @@ public FilteredMemberSupplier(MemberSupplier memberSupplier, MemberFilter filter Runtime.getRuntime().addShutdownHook(new Thread(this::destroyState)); } + /** + * Extended method that will return the first member that gets through the provided filter. If it gets through it, + * then the member will first be saved before it is returned + * + * @return the first member that gets through the filter + */ @Override public SuppliedMember get() { SuppliedMember member = super.get(); diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/membersuppliers/MemberSupplier.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/membersuppliers/MemberSupplier.java index 505340c7d..39b1d16c4 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/membersuppliers/MemberSupplier.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/membersuppliers/MemberSupplier.java @@ -4,8 +4,17 @@ import java.util.function.Supplier; +/** + * Interface that supplies the fetched members of an LDES + */ public interface MemberSupplier extends Supplier { + /** + * Initialization and/or set up of some processes or resources to supply the fetched members + */ void init(); + /** + * Release resources when the supplier is not required anymore + */ void destroyState(); } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/membersuppliers/MemberSupplierDecorator.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/membersuppliers/MemberSupplierDecorator.java index 7559f93d4..504ed9a45 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/membersuppliers/MemberSupplierDecorator.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/membersuppliers/MemberSupplierDecorator.java @@ -2,6 +2,9 @@ import ldes.client.treenodesupplier.domain.valueobject.SuppliedMember; +/** + * Decorator class implementation of the {@link MemberSupplier} + */ public abstract class MemberSupplierDecorator implements MemberSupplier { private final MemberSupplier memberSupplier; @@ -9,7 +12,12 @@ protected MemberSupplierDecorator(MemberSupplier memberSupplier) { this.memberSupplier = memberSupplier; } - + /** + * Implementation of the decorator method that just is responsible for calling the provided base member supplier + * and return its result + * + * @return Result of the base member supplier + */ @Override public SuppliedMember get() { return memberSupplier.get(); diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/membersuppliers/MemberSupplierImpl.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/membersuppliers/MemberSupplierImpl.java index 7fb0e05e3..84f3f4575 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/membersuppliers/MemberSupplierImpl.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/membersuppliers/MemberSupplierImpl.java @@ -3,31 +3,39 @@ import ldes.client.treenodesupplier.TreeNodeProcessor; import ldes.client.treenodesupplier.domain.valueobject.SuppliedMember; +/** + * Base implementation of the {@link MemberSupplier} + */ public class MemberSupplierImpl implements MemberSupplier { - private final TreeNodeProcessor treeNodeProcessor; - private final boolean keepState; + private final TreeNodeProcessor treeNodeProcessor; + private final boolean keepState; - public MemberSupplierImpl(TreeNodeProcessor treeNodeProcessor, boolean keepState) { - this.treeNodeProcessor = treeNodeProcessor; - this.keepState = keepState; - Runtime.getRuntime().addShutdownHook(new Thread(this::destroyState)); - } + public MemberSupplierImpl(TreeNodeProcessor treeNodeProcessor, boolean keepState) { + this.treeNodeProcessor = treeNodeProcessor; + this.keepState = keepState; + Runtime.getRuntime().addShutdownHook(new Thread(this::destroyState)); + } - @Override - public SuppliedMember get() { - return treeNodeProcessor.getMember(); - } + /** + * Base implementation that returns the fetched TreeNode member + * + * @return The plain fetched TreeNode member + */ + @Override + public SuppliedMember get() { + return treeNodeProcessor.getMember(); + } - @Override - public void destroyState() { - if (!keepState) { - treeNodeProcessor.destroyState(); - } - } + @Override + public void destroyState() { + if (!keepState) { + treeNodeProcessor.destroyState(); + } + } - @Override - public void init() { - treeNodeProcessor.init(); - } + @Override + public void init() { + treeNodeProcessor.init(); + } } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/membersuppliers/VersionMaterialisedMemberSupplier.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/membersuppliers/VersionMaterialisedMemberSupplier.java index b664d5bdd..500c4f218 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/membersuppliers/VersionMaterialisedMemberSupplier.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/membersuppliers/VersionMaterialisedMemberSupplier.java @@ -6,21 +6,26 @@ /** * This is a decorator for the {@link MemberSupplier} which makes it possible to materialize the version objects to - * state objects before providing them. + * state objects before supplying them. */ public class VersionMaterialisedMemberSupplier extends MemberSupplierDecorator { - private final VersionMaterialiser versionMaterialiser; + private final VersionMaterialiser versionMaterialiser; - public VersionMaterialisedMemberSupplier(MemberSupplier memberSupplier, VersionMaterialiser versionMaterialiser) { - super(memberSupplier); - this.versionMaterialiser = versionMaterialiser; - } + public VersionMaterialisedMemberSupplier(MemberSupplier memberSupplier, VersionMaterialiser versionMaterialiser) { + super(memberSupplier); + this.versionMaterialiser = versionMaterialiser; + } - @Override - public SuppliedMember get() { - final SuppliedMember suppliedMember = super.get(); - final Model stateObject = versionMaterialiser.transform(suppliedMember.getModel()); - return new SuppliedMember(suppliedMember.getId(), stateObject); - } + /** + * Materializes the supplied member before returning them + * + * @return the materialized version of the provided base supplied member + */ + @Override + public SuppliedMember get() { + final SuppliedMember suppliedMember = super.get(); + final Model stateObject = versionMaterialiser.transform(suppliedMember.getModel()); + return new SuppliedMember(suppliedMember.getId(), stateObject); + } } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/MemberIdRepository.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/MemberIdRepository.java index 831384459..515d9e53a 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/MemberIdRepository.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/MemberIdRepository.java @@ -1,7 +1,25 @@ package ldes.client.treenodesupplier.repository; +/** + * Repository that keeps track of the id of the processed members + */ public interface MemberIdRepository { - void addMemberId(String memberId); - boolean contains(String memberId); - void destroyState(); + /** + * Saves the memberId to the database + * + * @param memberId id of the member to save + */ + void addMemberId(String memberId); + + /** + * Checks whether the id of the member already has been processed + * @param memberId id of the member to be checked + * @return true if the member already has been saved, otherwise false + */ + boolean contains(String memberId); + + /** + * Cleans the database + */ + void destroyState(); } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/MemberRepository.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/MemberRepository.java index f425e2b4b..b6197dce8 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/MemberRepository.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/MemberRepository.java @@ -5,12 +5,28 @@ import java.util.Optional; import java.util.stream.Stream; +/** + * * Repository that keeps track of the processed members + */ public interface MemberRepository { + /** + * @return the first MemberRecord in the repository + */ Optional getTreeMember(); + + /** + * @param member MemberRecord to delete from the repository + */ void deleteMember(MemberRecord member); + /** + * @param treeMemberStream the stream of MemberRecords to save + */ void saveTreeMembers(Stream treeMemberStream); + /** + * Clean up the repository when it is not used anymore + */ void destroyState(); } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/MemberVersionRepository.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/MemberVersionRepository.java index 7f49c1d7d..395b929bd 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/MemberVersionRepository.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/MemberVersionRepository.java @@ -2,12 +2,29 @@ import ldes.client.treenodesupplier.domain.entities.MemberVersionRecord; +/** + * Repository that keeps track of the version-of and timestamp objects of the processed members + */ public interface MemberVersionRepository { - void addMemberVersion(MemberVersionRecord memberVersion); + /** + * Saves the version-of and timestamp objects of a member to the repository + */ + void addMemberVersion(MemberVersionRecord memberVersion); - boolean isVersionAfterTimestamp(MemberVersionRecord memberVersion); + /** + * Checks if the record with the version-of with the specified timestamp is after the last saved timestamp + * belonging to this version-of object, or if no other record with the specified version-of is already saved + * + * @param memberVersion the object containing the timestamp and version-of to check + * @return true if the timestamp of the record is after the last saved member or if this version-of is + * not yet saved, otherwise false + */ + boolean isVersionAfterTimestamp(MemberVersionRecord memberVersion); - void destroyState(); + /** + * Clean up the repository when it is not used anymore + */ + void destroyState(); } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/TreeNodeRecordRepository.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/TreeNodeRecordRepository.java index 3ed85a2d6..c9dbdb1bd 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/TreeNodeRecordRepository.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/TreeNodeRecordRepository.java @@ -6,17 +6,48 @@ import java.util.Optional; public interface TreeNodeRecordRepository { - void saveTreeNodeRecord(TreeNodeRecord processedTreenode); - + /** + * Saves a processed TreeNodeRecord to the repository + */ + void saveTreeNodeRecord(TreeNodeRecord processedTreeNode); + + /** + * Checks whether a tree node with the specified id exists + * + * @return true if the tree node exists + */ boolean existsById(String treeNodeId); + /** + * Searches the first TreeNodeRecord with the specified TreeNodeStatus and has the earliest + * {@link TreeNodeRecord#getEarliestNextVisit()} value + * + * @param treeNodeStatus the status that the desired TreeNodeRecord must have + * @return A TreeNodeRecord with the specified status, or an empty optional + */ Optional getTreeNodeRecordWithStatusAndEarliestNextVisit(TreeNodeStatus treeNodeStatus); + /** + * Checks whether a tree node with the specified id and TreeNodeStatus exists + * + * @return true if the tree node exists + */ boolean existsByIdAndStatus(String treeNodeId, TreeNodeStatus treeNodeStatus); + /** + * Clean up the repository when it is not used anymore + */ void destroyState(); + /** + * Checks whether any processable TreeNodeRecords are present in the repository + * + * @return a boolean whether + */ boolean containsTreeNodeRecords(); + /** + * Makes sure the context is clear again + */ void resetContext(); } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/inmemory/InMemoryMemberIdRepository.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/inmemory/InMemoryMemberIdRepository.java index 69bae4c45..ce63d8f65 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/inmemory/InMemoryMemberIdRepository.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/inmemory/InMemoryMemberIdRepository.java @@ -6,18 +6,20 @@ import java.util.List; public class InMemoryMemberIdRepository implements MemberIdRepository { - private List memberIds = new ArrayList<>(); - @Override - public void addMemberId(String memberId) { - memberIds.add(memberId); - } + private final List memberIds = new ArrayList<>(); - @Override - public boolean contains(String memberId) { - return memberIds.contains(memberId); - } - @Override - public void destroyState() { - memberIds = new ArrayList<>(); - } + @Override + public void addMemberId(String memberId) { + memberIds.add(memberId); + } + + @Override + public boolean contains(String memberId) { + return memberIds.contains(memberId); + } + + @Override + public void destroyState() { + memberIds.clear(); + } } diff --git a/ldi-core/ldi-common/pom.xml b/ldi-core/ldi-common/pom.xml index bc65d9a7f..919814076 100644 --- a/ldi-core/ldi-common/pom.xml +++ b/ldi-core/ldi-common/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT ldi-common diff --git a/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/extractor/EmptyPropertyExtractor.java b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/extractor/EmptyPropertyExtractor.java index bd5f7b881..981ad39bf 100644 --- a/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/extractor/EmptyPropertyExtractor.java +++ b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/extractor/EmptyPropertyExtractor.java @@ -6,8 +6,15 @@ import java.util.ArrayList; import java.util.List; +/** + * Default implementation of the PropertyExtractor + */ public class EmptyPropertyExtractor implements PropertyExtractor { + /** + * @param model ignored input + * @return an empty list + */ @Override public List getProperties(Model model) { return new ArrayList<>(); diff --git a/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/extractor/PropertyExtractor.java b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/extractor/PropertyExtractor.java index afe059b48..49b8c34fd 100644 --- a/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/extractor/PropertyExtractor.java +++ b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/extractor/PropertyExtractor.java @@ -5,6 +5,7 @@ import java.util.List; +@FunctionalInterface public interface PropertyExtractor { List getProperties(Model model); diff --git a/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/extractor/PropertyPathExtractor.java b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/extractor/PropertyPathExtractor.java index e90506198..a609e0979 100644 --- a/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/extractor/PropertyPathExtractor.java +++ b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/extractor/PropertyPathExtractor.java @@ -7,6 +7,9 @@ import java.util.ArrayList; import java.util.List; +/** + * Implementation that extracts all the properties that belongs to the specified propertyPath + */ public class PropertyPathExtractor implements PropertyExtractor { private static final String OBJECT_VAR_NAME = "object"; diff --git a/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/formatter/GenericRdfWriter.java b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/formatter/GenericRdfWriter.java index eb3ac629e..bc555ea07 100644 --- a/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/formatter/GenericRdfWriter.java +++ b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/formatter/GenericRdfWriter.java @@ -1,12 +1,16 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi.rdf.formatter; import org.apache.jena.rdf.model.Model; -import org.apache.jena.riot.*; +import org.apache.jena.riot.RDFWriter; +import org.apache.jena.riot.RDFWriterBuilder; import java.io.OutputStream; -import static be.vlaanderen.informatievlaanderen.ldes.ldi.rdf.formatter.PrefixAdder.*; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.rdf.formatter.PrefixAdder.addPrefixesToModel; +/** + * Default implementation of the LdiRdfWriter + */ public class GenericRdfWriter implements LdiRdfWriter { private final RDFWriterBuilder rdfWriter; diff --git a/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/formatter/LdiRdfWriter.java b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/formatter/LdiRdfWriter.java index 0c8ecabe9..44ee76487 100644 --- a/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/formatter/LdiRdfWriter.java +++ b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/formatter/LdiRdfWriter.java @@ -1,21 +1,34 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi.rdf.formatter; import be.vlaanderen.informatievlaanderen.ldes.ldi.exceptions.JsonLDFrameException; +import com.apicatalog.jsonld.JsonLdError; import org.apache.jena.rdf.model.Model; import org.apache.jena.riot.Lang; import java.io.OutputStream; -import com.apicatalog.jsonld.JsonLdError; - import static org.apache.commons.lang3.StringUtils.isBlank; +/** + * Interface that is responsible for writing linked data away + */ public interface LdiRdfWriter { + /** + * @return the RDF format that will be used to write the linked data + */ String getContentType(); + /** + * @param model RDF model that needs to be written + * @return string representation of the provided model + */ String write(Model model); + /** + * @param model RDF model that needs to be written + * @param outputStream whereto the RDF model needs to be written + */ void writeToOutputStream(Model model, OutputStream outputStream); static LdiRdfWriter getRdfWriter(LdiRdfWriterProperties properties) { diff --git a/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/formatter/PrefixAdder.java b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/formatter/PrefixAdder.java index b915e4bf4..4e1e8e2e8 100644 --- a/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/formatter/PrefixAdder.java +++ b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/rdf/formatter/PrefixAdder.java @@ -18,6 +18,9 @@ private PrefixAdder() { private static final String VALID_PREFIX_REGEX = "^[a-zA-Z_][\\w.-]*$"; // NCName regex private static final Logger LOGGER = LoggerFactory.getLogger(PrefixAdder.class); + /** + * Adds prefix to the RDF model, which can come in handy for RDF formats like text/turtle + */ public static Model addPrefixesToModel(Model model) { Map nameSpaceMap = new HashMap<>(); Map localNamesMap = new HashMap<>(); @@ -28,7 +31,7 @@ public static Model addPrefixesToModel(Model model) { } private static void removePrefixesWithNonCompliantLocalName(Map nameSpaceMap, - Map localNamesMap) { + Map localNamesMap) { localNamesMap.forEach((localName, prefix) -> { if (!localName.matches(VALID_LOCALNAME_REGEX)) { nameSpaceMap.remove(prefix); @@ -37,7 +40,7 @@ private static void removePrefixesWithNonCompliantLocalName(Map } private static void extractNamespaces(Map nameSpaceMap, Map localNamesMap, - Statement statement) { + Statement statement) { addPotentialPrefixToNamespaceMap(nameSpaceMap, localNamesMap, statement.getPredicate().getNameSpace(), statement.getPredicate().getLocalName()); @@ -49,8 +52,8 @@ private static void extractNamespaces(Map nameSpaceMap, Map nameSpaceMap, - Map localNamesMap, - String predicateNameSpace, String localName) { + Map localNamesMap, + String predicateNameSpace, String localName) { String candidateForPrefix = getPrefixCandidate(predicateNameSpace); if (isValidPrefixCandidate(candidateForPrefix)) { nameSpaceMap.put(candidateForPrefix, predicateNameSpace); diff --git a/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/timestampextractor/TimestampExtractor.java b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/timestampextractor/TimestampExtractor.java index 9ca8c4c1f..4b009d3f0 100644 --- a/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/timestampextractor/TimestampExtractor.java +++ b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/timestampextractor/TimestampExtractor.java @@ -7,7 +7,17 @@ public interface TimestampExtractor { + /** + * @param model where from the timestamp should be extracted + * @return the first found timestamp in the model + */ LocalDateTime extractTimestamp(Model model); + + /** + * @param subject where to the timestamp should belong in the statement + * @param model where from the timestamp should be extracted + * @return first found timestamp in the model that belongs to the provided subject + */ LocalDateTime extractTimestampWithSubject(Resource subject, Model model); diff --git a/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/timestampextractor/TimestampFromCurrentTimeExtractor.java b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/timestampextractor/TimestampFromCurrentTimeExtractor.java index 67e47a339..92ce438d3 100644 --- a/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/timestampextractor/TimestampFromCurrentTimeExtractor.java +++ b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/timestampextractor/TimestampFromCurrentTimeExtractor.java @@ -5,6 +5,9 @@ import java.time.LocalDateTime; +/** + * Default or empty implementation of the TimestampExtractor + */ public class TimestampFromCurrentTimeExtractor implements TimestampExtractor { @Override public LocalDateTime extractTimestamp(Model model) { diff --git a/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/timestampextractor/TimestampFromPathExtractor.java b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/timestampextractor/TimestampFromPathExtractor.java index b4175aef7..53c86777d 100644 --- a/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/timestampextractor/TimestampFromPathExtractor.java +++ b/ldi-core/ldi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/timestampextractor/TimestampFromPathExtractor.java @@ -13,6 +13,10 @@ public class TimestampFromPathExtractor implements TimestampExtractor { private final Property timestampPath; + /** + * + * @param timestampPath Predicate where the timestamp should be extracted from + */ public TimestampFromPathExtractor(Property timestampPath) { this.timestampPath = timestampPath; } diff --git a/ldi-core/ldi-infra-sql/pom.xml b/ldi-core/ldi-infra-sql/pom.xml index 6ec397e1f..793064339 100644 --- a/ldi-core/ldi-infra-sql/pom.xml +++ b/ldi-core/ldi-infra-sql/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT ldi-infra-sql diff --git a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/EntityManagerFactory.java b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/EntityManagerFactory.java index e4a3a3d86..38dc1c0ac 100644 --- a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/EntityManagerFactory.java +++ b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/EntityManagerFactory.java @@ -2,8 +2,19 @@ import javax.persistence.EntityManager; +/** + * Custom interface to manage the EntityManager, EntityManagerFactory from the javax library and their lifecycle more easily + */ public interface EntityManagerFactory { + /** + * @return the enityManager that is managed by the javax library + */ EntityManager getEntityManager(); + /** + * Destroy the EntityManager and EntityManagerFactory from the javax library for the specified instanceName + * + * @param instanceName name of the instances from which the EntityManager and EntityManagerFactory must be destroyed + */ void destroyState(String instanceName); } diff --git a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/HibernateProperties.java b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/HibernateProperties.java index 299b4c69f..5fe148755 100644 --- a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/HibernateProperties.java +++ b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/HibernateProperties.java @@ -11,5 +11,8 @@ public interface HibernateProperties { String UPDATE = "update"; String CREATE_DROP = "create-drop"; + /** + * @return a map that has the hibernate property names as keys with the right value to it + */ Map getProperties(); } diff --git a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/postgres/PostgresEntityManagerFactory.java b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/postgres/PostgresEntityManagerFactory.java index 6049df5e3..29141ceb6 100644 --- a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/postgres/PostgresEntityManagerFactory.java +++ b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/postgres/PostgresEntityManagerFactory.java @@ -23,6 +23,7 @@ public static synchronized PostgresEntityManagerFactory getInstance(String insta return instances.computeIfAbsent(instanceName, s -> new PostgresEntityManagerFactory(properties)); } + @Override public EntityManager getEntityManager() { return em; } diff --git a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/sqlite/SqliteEntityManagerFactory.java b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/sqlite/SqliteEntityManagerFactory.java index f866850ad..8df74952f 100644 --- a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/sqlite/SqliteEntityManagerFactory.java +++ b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/sqlite/SqliteEntityManagerFactory.java @@ -42,10 +42,12 @@ public static synchronized SqliteEntityManagerFactory getInstance(HibernatePrope }); } + @Override public EntityManager getEntityManager() { return em; } + @Override public void destroyState(String instanceName) { em.close(); emf.close(); diff --git a/ldi-core/ngsiv2-to-ld-adapter/pom.xml b/ldi-core/ngsiv2-to-ld-adapter/pom.xml index 35a68f911..8f411ab0c 100644 --- a/ldi-core/ngsiv2-to-ld-adapter/pom.xml +++ b/ldi-core/ngsiv2-to-ld-adapter/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT ngsiv2-to-ld-adapter diff --git a/ldi-core/ngsiv2-to-ld-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/NgsiV2ToLdAdapter.java b/ldi-core/ngsiv2-to-ld-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/NgsiV2ToLdAdapter.java index 60b349e56..250e3e3d1 100644 --- a/ldi-core/ngsiv2-to-ld-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/NgsiV2ToLdAdapter.java +++ b/ldi-core/ngsiv2-to-ld-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/NgsiV2ToLdAdapter.java @@ -15,16 +15,15 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +/** + * Adapter that will transform a NGSI V2 input into NGSI LD. + */ public class NgsiV2ToLdAdapter implements LdiAdapter { private final String coreContext; private final String ldContext; private final String dataIdentifier; - public NgsiV2ToLdAdapter(String dataIdentifier, String coreContext) { - this(dataIdentifier, coreContext, null); - } - public NgsiV2ToLdAdapter(String dataIdentifier, String coreContext, String ldContext) { if (dataIdentifier == null) { throw new IllegalArgumentException("Can't identify data with the data array key"); @@ -38,7 +37,7 @@ public NgsiV2ToLdAdapter(String dataIdentifier, String coreContext, String ldCon this.ldContext = ldContext; } - public Stream translateJsonToLD(String data) { + Stream translateJsonToLD(String data) { try { LinkedDataModel model = new ObjectMapper() .readerFor(LinkedDataModel.class) diff --git a/ldi-core/ngsiv2-to-ld-adapter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/services/NgsiV2ToLdAdapterTest.java b/ldi-core/ngsiv2-to-ld-adapter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/NgsiV2ToLdAdapterTest.java similarity index 98% rename from ldi-core/ngsiv2-to-ld-adapter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/services/NgsiV2ToLdAdapterTest.java rename to ldi-core/ngsiv2-to-ld-adapter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/NgsiV2ToLdAdapterTest.java index 9f3c81058..1e684d1a4 100644 --- a/ldi-core/ngsiv2-to-ld-adapter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/services/NgsiV2ToLdAdapterTest.java +++ b/ldi-core/ngsiv2-to-ld-adapter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/NgsiV2ToLdAdapterTest.java @@ -1,6 +1,5 @@ -package be.vlaanderen.informatievlaanderen.ldes.ldi.services; +package be.vlaanderen.informatievlaanderen.ldes.ldi; -import be.vlaanderen.informatievlaanderen.ldes.ldi.NgsiV2ToLdAdapter; import be.vlaanderen.informatievlaanderen.ldes.ldi.config.NgsiV2ToLdMapping; import be.vlaanderen.informatievlaanderen.ldes.ldi.valuobjects.LinkedDataModel; import org.apache.jena.atlas.json.JsonObject; @@ -137,8 +136,7 @@ void whenDateModifiedFound_thenDateModifiedTranslated() { @Test void whenCoreContextIsNull_thenInvalidNgsiLdContextExceptionIsThrown() { - String coreContext = null; - assertThrows(IllegalArgumentException.class, () -> new NgsiV2ToLdAdapter(dataIdentifier, coreContext)); + assertThrows(IllegalArgumentException.class, () -> new NgsiV2ToLdAdapter(dataIdentifier, null, localLdContext)); } @Test diff --git a/ldi-core/pom.xml b/ldi-core/pom.xml index bea62f54c..6f2976a9a 100644 --- a/ldi-core/pom.xml +++ b/ldi-core/pom.xml @@ -3,7 +3,7 @@ linked-data-interactions be.vlaanderen.informatievlaanderen.ldes - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/rdf-adapter/pom.xml b/ldi-core/rdf-adapter/pom.xml index 5244d431e..7a66a1e1e 100644 --- a/ldi-core/rdf-adapter/pom.xml +++ b/ldi-core/rdf-adapter/pom.xml @@ -5,7 +5,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/rdf-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/RdfAdapter.java b/ldi-core/rdf-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/RdfAdapter.java index e61a02f96..aac5e3b47 100644 --- a/ldi-core/rdf-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/RdfAdapter.java +++ b/ldi-core/rdf-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/RdfAdapter.java @@ -11,6 +11,9 @@ import static org.apache.jena.riot.RDFLanguages.nameToLang; +/** + * Basic adapter that will convert an RDF string to a linked data model + */ public class RdfAdapter implements LdiAdapter { private final Context context; diff --git a/ldi-core/repository-materialiser/pom.xml b/ldi-core/repository-materialiser/pom.xml index c7cf14d02..b65dcee63 100644 --- a/ldi-core/repository-materialiser/pom.xml +++ b/ldi-core/repository-materialiser/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT diff --git a/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/Materialiser.java b/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/Materialiser.java index 9ea379a4c..0c5cfe7b8 100644 --- a/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/Materialiser.java +++ b/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/Materialiser.java @@ -16,6 +16,9 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; +/** + * Component that will write linked data models to a specified triples store or RDF repository + */ public class Materialiser { private final MaterialiserConnection materialiserConnection; diff --git a/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/exceptions/ModelParseIOException.java b/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/exceptions/ModelParseIOException.java deleted file mode 100644 index 1fa2cd005..000000000 --- a/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/exceptions/ModelParseIOException.java +++ /dev/null @@ -1,17 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.ldi.exceptions; - -public class ModelParseIOException extends RuntimeException { - - private final String model; - private final String cause; - - public ModelParseIOException(String model, String cause) { - this.model = model; - this.cause = cause; - } - - @Override - public String getMessage() { - return "Could not parse content to model. cause:\n" + cause + "\nContent:\n" + model; - } -} diff --git a/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/services/JenaToRDF4JConverter.java b/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/services/JenaToRDF4JConverter.java index ae20817b3..57a35e862 100644 --- a/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/services/JenaToRDF4JConverter.java +++ b/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/services/JenaToRDF4JConverter.java @@ -38,14 +38,13 @@ private static IRI convertProperty(org.apache.jena.rdf.model.Property property) private static Value convertValue(RDFNode node) { if (node.isResource()) { - org.apache.jena.rdf.model.Resource resource = (org.apache.jena.rdf.model.Resource) node; - return convertResource(resource); + return convertResource(node.asResource()); } else { Literal literal = node.asLiteral(); if(literal.getLanguage() != null && !literal.getLanguage().isEmpty()) { - return SimpleValueFactory.getInstance().createLiteral(literal.getValue().toString(), literal.getLanguage()); + return SimpleValueFactory.getInstance().createLiteral(literal.getString(), literal.getLanguage()); } - return SimpleValueFactory.getInstance().createLiteral(literal.getValue().toString(), SimpleValueFactory.getInstance().createIRI(literal.getDatatypeURI())); + return SimpleValueFactory.getInstance().createLiteral(literal.getString(), SimpleValueFactory.getInstance().createIRI(literal.getDatatypeURI())); } } } diff --git a/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/MaterialiserConnection.java b/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/MaterialiserConnection.java index 48854ff58..177d5e336 100644 --- a/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/MaterialiserConnection.java +++ b/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/MaterialiserConnection.java @@ -8,6 +8,10 @@ import java.util.Optional; +/** + * Wrapper around the RepositoryConnectionHolder, to make more an abstraction of the actions that can be done + * through the RepositoryConnection and already handling with some overhead of the named graph + */ public class MaterialiserConnection { private final String namedGraph; private final RepositoryConnectionHolder holder; diff --git a/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/RepositoryConnectionHolder.java b/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/RepositoryConnectionHolder.java index 545c3a01b..b7112e140 100644 --- a/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/RepositoryConnectionHolder.java +++ b/ldi-core/repository-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/RepositoryConnectionHolder.java @@ -7,6 +7,9 @@ import org.eclipse.rdf4j.repository.http.HTTPRepository; import org.eclipse.rdf4j.repository.manager.RepositoryManager; +/** + * Wrapper around the RDF4J RepositoryConnection to isolate it more and give the maintenance of the connection out of hands + */ public class RepositoryConnectionHolder { private final RepositoryManager repositoryManager; private final String repositoryId; @@ -30,9 +33,9 @@ public synchronized RepositoryConnection getConnection() { } public void shutdown() { - if(connection != null) { + if (connection != null) { connection.close(); } repositoryManager.shutDown(); - } + } } diff --git a/ldi-core/repository-materialiser/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/MaterialiserIT.java b/ldi-core/repository-materialiser/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/MaterialiserIT.java index 71be4e28f..42a1281cc 100644 --- a/ldi-core/repository-materialiser/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/MaterialiserIT.java +++ b/ldi-core/repository-materialiser/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/MaterialiserIT.java @@ -151,6 +151,24 @@ void given_ComplexModelToUpdate_test_Process(String namedGraph) throws IOExcepti assertThat(Models.isomorphic(updatedDbModel, updatedRdfModel)).isTrue(); } + @ParameterizedTest + @ArgumentsSource(NamedGraphProvider.class) + void given_geoData_when_processModel_then_addValidGeoDataToRepo(String namedGraph) throws IOException { + materialiser = new Materialiser(subject, LOCAL_REPOSITORY_ID, namedGraph); + final Model expectedModel = Rio.parse(new FileInputStream("src/test/resources/geo/measurement.ttl"), "", RDFFormat.TURTLE); + final org.apache.jena.rdf.model.Model inputModel = RDFParser.source("geo/measurement.ttl").toModel(); + materialiser.process(List.of(inputModel)); + + Model dbModel = new LinkedHashModel(); + materialiser.getMaterialiserConnection() + .getStatements(null, null, null) + .stream() + .map(MaterialiserIT::deleteContextFromStatement) + .forEach(dbModel::add); + + assertThat(Models.isomorphic(expectedModel, dbModel)).isTrue(); + } + void populateAndCheckRepository(List files, String namedGraph) throws IOException { List models = new ArrayList<>(); for (String testFile : files) { diff --git a/ldi-core/repository-materialiser/src/test/resources/geo/measurement.ttl b/ldi-core/repository-materialiser/src/test/resources/geo/measurement.ttl new file mode 100644 index 000000000..b5a95ef27 --- /dev/null +++ b/ldi-core/repository-materialiser/src/test/resources/geo/measurement.ttl @@ -0,0 +1,25 @@ +@prefix geosparql: . +@prefix ldes: . +@prefix observaties-en-metingen: . +@prefix prov: . +@prefix rdf: . +@prefix sensoren-en-bemonstering: . +@prefix terms: . +@prefix tree: . +@prefix waterkwaliteit: . + + + rdf:type sensoren-en-bemonstering:Station; + terms:isVersionOf ; + terms:modified "2024-05-24T13:31:35.183Z"^^; + prov:generatedAtTime "2024-05-30T14:31:13.278Z"^^; + observaties-en-metingen:geometrie + [ rdf:type observaties-en-metingen:Punt; + geosparql:asWKT " POINT (5.735558 51.10408)"^^geosparql:wktLiteral + ]; + observaties-en-metingen:identificator + "421877"; + waterkwaliteit:bemonsterdObject + [ rdf:type waterkwaliteit:WaterObject; + terms:description "Maaseik/Jagersborg/Witbeek" + ] . \ No newline at end of file diff --git a/ldi-core/request-executor/pom.xml b/ldi-core/request-executor/pom.xml index 4b8f06653..7bbf7317a 100644 --- a/ldi-core/request-executor/pom.xml +++ b/ldi-core/request-executor/pom.xml @@ -5,7 +5,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 request-executor diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/RequestExecutor.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/RequestExecutor.java index f6be83f5a..0415c0424 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/RequestExecutor.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/RequestExecutor.java @@ -3,6 +3,9 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.Request; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.Response; +/** + * Custom interface to make it more manageable to deal with HttpRequests and HttpResponses + */ public interface RequestExecutor { Response execute(Request request); diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/RequestExecutorSupplier.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/RequestExecutorSupplier.java index 1ba65233c..7d7dd7317 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/RequestExecutorSupplier.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/RequestExecutorSupplier.java @@ -1,5 +1,13 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor; +/** + * Interface to supply a RequestExecutor + *
+ * Will typically be implemented by some config class that will create a RequestExecutor based on that class level config + */ public interface RequestExecutorSupplier { + /** + * @return instance of RequestExecutor, based on class based configuration + */ RequestExecutor createRequestExecutor(); } diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsConfig.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsConfig.java index 60d2b3f76..5475a8345 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsConfig.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsConfig.java @@ -41,6 +41,7 @@ public ClientCredentialsConfig(Collection
headers, this.enableRedirectHandling = enableRedirectHandling; } + @Override public RequestExecutor createRequestExecutor() { return new ClientCredentialsRequestExecutor(new OAuth20ServiceTokenCacheWrapper(createService())); } diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsRequest.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsRequest.java index 9218df1e9..4088b990e 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsRequest.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsRequest.java @@ -8,6 +8,10 @@ import com.github.scribejava.core.model.OAuthRequest; import com.github.scribejava.core.model.Verb; +/** + * Wrapper around the effective own custom Request object to convert it to an OAuthRequest + * in the effective HttpClient + */ public class ClientCredentialsRequest { private final Request request; diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsRequestExecutor.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsRequestExecutor.java index ca662002c..3d78bec70 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsRequestExecutor.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/ClientCredentialsRequestExecutor.java @@ -16,7 +16,8 @@ import com.github.scribejava.core.model.OAuthRequest; /** - * Handles sending the actual HTTP request to the server. + * Implementation of the RequestExecutor and acts like a wrapper around an OAuthService, therefor dealing with + * sending http request to secured server is easier */ public class ClientCredentialsRequestExecutor implements RequestExecutor { diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/OAuth20ServiceTokenCacheWrapper.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/OAuth20ServiceTokenCacheWrapper.java index 9d89ef997..5ae205b3e 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/OAuth20ServiceTokenCacheWrapper.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/OAuth20ServiceTokenCacheWrapper.java @@ -19,6 +19,10 @@ public OAuth20ServiceTokenCacheWrapper(OAuth20Service oAuth20Service) { this.oAuth20Service = oAuth20Service; } + /** + * Deals with tokens and the possibility that the token has been expired + * @return an unexpired and valid OAuth2 Access Token + */ public OAuth2AccessToken getAccessTokenClientCredentialsGrant() { return tokenExpiryWrapper.getAccessToken().orElseGet(() -> { try { diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/OAuth2AccessTokenExpiryWrapper.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/OAuth2AccessTokenExpiryWrapper.java index 549d2589d..f63181f2e 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/OAuth2AccessTokenExpiryWrapper.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/clientcredentials/OAuth2AccessTokenExpiryWrapper.java @@ -5,12 +5,15 @@ import com.github.scribejava.core.model.OAuth2AccessToken; +/** + * Class to deal with the validity of an OAuth2 token + */ public class OAuth2AccessTokenExpiryWrapper { private final OAuth2AccessToken token; private final LocalDateTime expiryTs; - public OAuth2AccessTokenExpiryWrapper(OAuth2AccessToken token, LocalDateTime expiryTs) { + private OAuth2AccessTokenExpiryWrapper(OAuth2AccessToken token, LocalDateTime expiryTs) { this.token = token; this.expiryTs = expiryTs; } diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/TokenService.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/TokenService.java index db7a073db..979e834d4 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/TokenService.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/TokenService.java @@ -4,10 +4,23 @@ public interface TokenService { + /** + * Awaits on the token header until the EDC token is available + * + * @return http header that contains the EDC token + */ RequestHeader waitForTokenHeader(); + /** + * Clears the EDC token, so it cannot be used anymore + */ void invalidateToken(); + /** + * Sets the EDC token based on the string + * + * @param token string representation of the EDC token + */ void updateToken(String token); void pause(); diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/TransferService.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/TransferService.java index af6e4984b..cb291051c 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/TransferService.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/edc/services/TransferService.java @@ -4,6 +4,12 @@ public interface TransferService { + /** + * Sets the transfer string for the request body and sends the request + * + * @param transfer string representation of the transfer + * @return the response from the transfer request + */ Response startTransfer(String transfer); void refreshTransfer(); diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/DefaultConfig.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/DefaultConfig.java index eebba062b..f0c7d5c63 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/DefaultConfig.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/DefaultConfig.java @@ -17,6 +17,11 @@ public DefaultConfig(Collection
headers, boolean enableRedirectHandling) this.enableRedirectHandling = enableRedirectHandling; } + /** + * + * @return instance of DefaultRequestExecutor + */ + @Override public RequestExecutor createRequestExecutor() { final HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); if (!enableRedirectHandling) { diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/DefaultRequest.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/DefaultRequest.java index 4df8c8a2d..2ece7ebeb 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/DefaultRequest.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/DefaultRequest.java @@ -12,6 +12,10 @@ import java.nio.charset.StandardCharsets; +/** + * Wrapper around the effective own custom Request object to convert it to an Apache HttpUriRequest that will be used + * in the effective HttpClient + */ public class DefaultRequest { private final Request request; @@ -20,6 +24,9 @@ public DefaultRequest(Request request) { this.request = request; } + /** + * @return an Apache HttpUriRequest, based on the provided Request present in this wrapper class + */ public HttpUriRequest getHttpRequest() { final HttpRequestBase httpRequest = createRequest(); request.getRequestHeaders().forEach(header -> httpRequest.addHeader(header.getKey(), header.getValue())); diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/DefaultRequestExecutor.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/DefaultRequestExecutor.java index 148e72150..fc9192467 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/DefaultRequestExecutor.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/DefaultRequestExecutor.java @@ -9,6 +9,9 @@ import java.io.IOException; +/** + * Default implementation of the RequestExecutor and acts like a wrapper around the Apache HttpClient + */ public class DefaultRequestExecutor implements RequestExecutor { private final HttpClient httpClient; diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/DefaultResponse.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/DefaultResponse.java index 4fffe28ea..3b86d0ef8 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/DefaultResponse.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/noauth/DefaultResponse.java @@ -10,6 +10,9 @@ import java.util.Arrays; import java.util.List; +/** + * Wrapper around the Apache HttpResponse, so that the response can be managed easier + */ public class DefaultResponse { private final HttpResponse httpResponse; diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/ratelimiter/RateLimiterConfig.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/ratelimiter/RateLimiterConfig.java index 24a9e1c77..fa240b738 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/ratelimiter/RateLimiterConfig.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/ratelimiter/RateLimiterConfig.java @@ -23,11 +23,22 @@ private RateLimiterConfig(int limitForPeriod, Duration limitRefreshPeriod, Durat this.timeoutDuration = timeoutDuration; } + /** + * @param maxRequestsPerMinute the maximum amount of request that may be performed during the time span of 1 minute + * @deprecated replaced by {@link #limitForPeriod} which provides a more flexible way of configuring the RateLimit + */ + @Deprecated(forRemoval = true) public static RateLimiterConfig limitPerMinute(int maxRequestsPerMinute) { log.warn(RATE_LIMIT_PER_MINUTE_MIGRATION_WARNING); return new RateLimiterConfig(maxRequestsPerMinute, Duration.ofMinutes(1), Duration.ofMinutes(1)); } + /** + * @param limit the maximum number of request that may be performed in the specified + * @param period string presentation of the ISO-8601 duration wherein the maximum number of requests may be performed + * @see #limitForPeriod(int, Duration) + * @see #limitPerMinute + */ public static RateLimiterConfig limitForPeriod(int limit, String period) { try { return limitForPeriod(limit, Duration.parse(period)); @@ -36,6 +47,12 @@ public static RateLimiterConfig limitForPeriod(int limit, String period) { } } + /** + * @param limit the maximum number of request that may be performed in the specified + * @param period {@link java.time.Duration} instance of the duration wherein the maximum number of request may be performed + * @see #limitForPeriod(int, String) + * @see #limitPerMinute + */ public static RateLimiterConfig limitForPeriod(int limit, Duration period) { return new RateLimiterConfig(limit, period, period); } diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/retry/BasicIntervalFunctionDecorator.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/retry/BasicIntervalFunctionDecorator.java index bb16116ce..c533fc600 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/retry/BasicIntervalFunctionDecorator.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/retry/BasicIntervalFunctionDecorator.java @@ -13,6 +13,9 @@ import static org.apache.commons.lang3.Validate.notNull; +/** + * Wrapper to deal with the retry intervals when an http response execution has failed + */ public class BasicIntervalFunctionDecorator implements IntervalBiFunction { private final Logger log = LoggerFactory.getLogger(BasicIntervalFunctionDecorator.class); diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/retry/RetryAfter.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/retry/RetryAfter.java index 7e47e54ae..899e0041d 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/retry/RetryAfter.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/retry/RetryAfter.java @@ -21,8 +21,7 @@ public RetryAfter(LocalDateTime retryAfterTimeStamp) { * "https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After">specification * this header can be a date or an integer. * - * @param retryHeader - * represents either a valid http date or an integer for milliseconds + * @param retryHeader represents either a valid http date or an integer for milliseconds */ public static RetryAfter from(String retryHeader) { if (NumberUtils.isParsable(retryHeader)) { @@ -35,8 +34,7 @@ public static RetryAfter from(String retryHeader) { } /** - * Returns the number of millis until a retry can be done. - * Return value is always positive. + * @return a long that represents the millis until a retry can be done, is always positive. */ public long getMillisUntilRetry() { LocalDateTime now = LocalDateTime.now(); diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/valueobjects/PostRequest.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/valueobjects/PostRequest.java index a834ecf0d..aaf36e0fb 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/valueobjects/PostRequest.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/valueobjects/PostRequest.java @@ -9,6 +9,9 @@ public class PostRequest extends Request { public static final String METHOD_NAME = "POST"; + /** + * Representation of the request body as a byte array, this is for binary support purposes + */ private final byte[] body; public PostRequest(String url, RequestHeaders requestHeaders, String body) { @@ -20,6 +23,7 @@ public PostRequest(String url, RequestHeaders requestHeaders, byte[] body) { this.body = body; } + @Override public String getMethod() { return METHOD_NAME; } @@ -34,6 +38,11 @@ public Request with(RequestHeaders requestHeaders) { return new PostRequest(url, requestHeaders, body); } + /** + * Converts the request body, which is a byte array, to a string. Mind that the byte array cannot always to proper converted to a string + * + * @return the request body as a string + */ public String getBodyAsString() { return new String(body); } diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/valueobjects/Request.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/valueobjects/Request.java index 6398cd987..29cfcf017 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/valueobjects/Request.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/valueobjects/Request.java @@ -25,10 +25,19 @@ public RequestHeaders getRequestHeaders() { return requestHeaders; } + /** + * @return A string representation via which HTTP method the request will be executed + */ public abstract String getMethod(); + /** + * @return a copy of the instance with the url replaced + */ public abstract Request with(String url); + /** + * @return a copy of the instance with the request headers replaced + */ public abstract Request with(RequestHeaders requestHeaders); @Override diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/valueobjects/RequestHeader.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/valueobjects/RequestHeader.java index 92fe9a6e7..4b19341c8 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/valueobjects/RequestHeader.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/valueobjects/RequestHeader.java @@ -14,6 +14,9 @@ public RequestHeader(String key, String value) { this.value = notNull(value); } + /** + * @param headerString representation of the header as following:
Header-Key: Header-Value + */ public static RequestHeader from(String headerString) { int indexFirstColon = headerString.indexOf(":"); String key = headerString.substring(0, indexFirstColon).trim(); diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/valueobjects/RequestHeaders.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/valueobjects/RequestHeaders.java index 53a643b23..b2497d1d4 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/valueobjects/RequestHeaders.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/valueobjects/RequestHeaders.java @@ -12,6 +12,11 @@ public RequestHeaders(List requestHeaders) { this.headers = requestHeaders; } + /** + * + * @param requestHeader new header that needs to be added to the existing headers + * @return A copy of the instance with the new header added + */ public RequestHeaders withRequestHeader(RequestHeader requestHeader) { ArrayList updatedRequestHeaders = new ArrayList<>(headers); updatedRequestHeaders.add(requestHeader); diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/valueobjects/Response.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/valueobjects/Response.java index 08113660a..71897bcb9 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/valueobjects/Response.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/valueobjects/Response.java @@ -11,6 +11,9 @@ import static org.apache.commons.lang3.StringUtils.lowerCase; +/** + * Contains the response details received from the server. + */ public class Response { private final Request request; @@ -21,6 +24,7 @@ public class Response { public Response(Request request, List
headers, int httpStatus, String body) { this(request, headers, httpStatus, body == null ? null : body.getBytes()); } + public Response(Request request, List
headers, int httpStatus, byte[] body) { this.request = request; this.httpStatus = httpStatus; diff --git a/ldi-core/rml-adapter/pom.xml b/ldi-core/rml-adapter/pom.xml index 80dee905f..538ab0ac9 100644 --- a/ldi-core/rml-adapter/pom.xml +++ b/ldi-core/rml-adapter/pom.xml @@ -5,7 +5,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/rml-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/RmlAdapter.java b/ldi-core/rml-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/RmlAdapter.java index 19f9e0dbc..39023187c 100644 --- a/ldi-core/rml-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/RmlAdapter.java +++ b/ldi-core/rml-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/RmlAdapter.java @@ -33,6 +33,9 @@ import static java.util.stream.StreamSupport.stream; +/** + * Adapter that converts non-linked data object to a linked data RDF object + */ public class RmlAdapter implements LdiAdapter { private final RdfRmlMapper rmlMapper; diff --git a/ldi-core/rml-adapter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/RmlAdapterTest.java b/ldi-core/rml-adapter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/RmlAdapterTest.java index 56a4d351a..e9f86e503 100644 --- a/ldi-core/rml-adapter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/RmlAdapterTest.java +++ b/ldi-core/rml-adapter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/RmlAdapterTest.java @@ -2,6 +2,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; import org.apache.commons.io.FileUtils; +import org.apache.jena.rdf.model.Model; import org.apache.jena.riot.Lang; import org.apache.jena.riot.RDFParser; import org.apache.jena.riot.RDFWriter; @@ -44,17 +45,13 @@ void test_awv_location() { void test_awv_observation() { var models = runRmlTest("awv/observation/data.xml", "awv/observation/mapping.ttl", "application/xml"); - // var expected = - // RDFParser.source("awv/observation/expected.nt").lang(Lang.NQUADS).toModel(); - assertEquals(1, models.size()); - // assertTrue(models.get(0).isIsomorphicWith(expected)); } - private List runRmlTest(String dataPath, String mappingPath, String mimeType) { + private List runRmlTest(String dataPath, String mappingPath, String mimeType) { RmlAdapter rmlAdapter = new RmlAdapter(getFileContent(mappingPath)); - List models = rmlAdapter.apply(LdiAdapter.Content.of(getFileContent( + List models = rmlAdapter.apply(LdiAdapter.Content.of(getFileContent( dataPath), mimeType)).toList(); models.forEach(model -> { diff --git a/ldi-core/sparql-construct/pom.xml b/ldi-core/sparql-construct/pom.xml index 00ea12b6e..281c30524 100644 --- a/ldi-core/sparql-construct/pom.xml +++ b/ldi-core/sparql-construct/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/sparql-construct/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/SparqlConstructTransformer.java b/ldi-core/sparql-construct/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/SparqlConstructTransformer.java index ae25b944b..83ea97472 100644 --- a/ldi-core/sparql-construct/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/SparqlConstructTransformer.java +++ b/ldi-core/sparql-construct/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/SparqlConstructTransformer.java @@ -11,6 +11,9 @@ import java.util.Iterator; import java.util.List; +/** + * Will modify the model based on the given SPARQL Construct Query + */ public class SparqlConstructTransformer implements LdiOneToManyTransformer { private final Query query; diff --git a/ldi-core/version-materialiser/pom.xml b/ldi-core/version-materialiser/pom.xml index e031994e2..7f452e6c8 100644 --- a/ldi-core/version-materialiser/pom.xml +++ b/ldi-core/version-materialiser/pom.xml @@ -3,7 +3,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/version-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/Tree.java b/ldi-core/version-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/Tree.java index 47ac2f620..370c3339d 100644 --- a/ldi-core/version-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/Tree.java +++ b/ldi-core/version-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/Tree.java @@ -3,6 +3,9 @@ import org.apache.jena.iri.IRI; import org.apache.jena.iri.IRIFactory; +/** + * Tree spec constants + */ public class Tree { private Tree() { } diff --git a/ldi-core/version-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/VersionMaterialiser.java b/ldi-core/version-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/VersionMaterialiser.java index 6b53785ca..a918beb6b 100644 --- a/ldi-core/version-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/VersionMaterialiser.java +++ b/ldi-core/version-materialiser/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/VersionMaterialiser.java @@ -9,8 +9,15 @@ import static org.apache.jena.rdf.model.ResourceFactory.createProperty; +/** + * Will transform a Version Object into a State Object. + */ public class VersionMaterialiser implements LdiOneToOneTransformer { private final Property versionPredicate; + /** + * Represents whether only the statements of the node containing the versionOf property needs to be returned, + * including potential nested blank nodes + */ private final boolean restrictToMembers; public VersionMaterialiser(Property versionPredicate, boolean restrictToMembers) { @@ -50,7 +57,7 @@ private Statement deversionStatement(Map versionIdEntityIdMa // Object references a versioned entity, replace it with the 'de-versioned' // identifier. if (statement.getObject().isResource() - && versionIdEntityIdMap.containsKey(object)) { + && versionIdEntityIdMap.containsKey(object)) { object = versionIdEntityIdMap.get(object); } @@ -73,8 +80,7 @@ private static Map getVersionIDEntityIDMap(Model model, Prop * potential nested blank nodes. * Excludes statements about referenced entities, provided as context. * - * @param inputModel - * The model to reduce. + * @param inputModel The model to reduce. * @return The reduced model. */ private Model reduceToLDESMemberOnlyModel(Model inputModel) { diff --git a/ldi-core/version-object-creator/pom.xml b/ldi-core/version-object-creator/pom.xml index d254fc57a..0ed0efd86 100644 --- a/ldi-core/version-object-creator/pom.xml +++ b/ldi-core/version-object-creator/pom.xml @@ -3,7 +3,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/VersionObjectCreator.java b/ldi-core/version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/VersionObjectCreator.java index 120aa3b1b..9acfe85f5 100644 --- a/ldi-core/version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/VersionObjectCreator.java +++ b/ldi-core/version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/VersionObjectCreator.java @@ -16,6 +16,9 @@ import static org.apache.jena.rdf.model.ResourceFactory.*; +/** + * Will transform a State Object into a Version Object. + */ public class VersionObjectCreator implements LdiOneToOneTransformer { private static final Logger LOGGER = LoggerFactory.getLogger(VersionObjectCreator.class); @@ -31,15 +34,30 @@ public class VersionObjectCreator implements LdiOneToOneTransformer { public static final String LINKED_DATA_MODEL_IS_EMPTY = "Received an empty data model"; public static final String CREATED_VERSION = "Created version: {}"; + /** + * Extractor that will be used to generate a timestamp for the version object + */ private final PropertyExtractor dateObservedPropertyExtractor; + /** + * Representation of the member type resource that the state object represents + */ private final Resource memberTypeResource; + /** + * Represents that needs to be used to expand the named subject of the member to a version object member id + */ private final String delimiter; + /** + * Property path that will be used as a predicate to foresee the state object of a timestamp + */ private final Property generatedAtTimeProperty; + /** + * Property path that will be used a predicate to foresee the state object of a version-of + */ private final Property versionOfProperty; public VersionObjectCreator(PropertyExtractor dateObservedPropertyExtractor, Resource memberTypeResource, - String delimiter, - Property generatedAtTimeProperty, Property versionOfProperty) { + String delimiter, + Property generatedAtTimeProperty, Property versionOfProperty) { this.dateObservedPropertyExtractor = dateObservedPropertyExtractor; this.memberTypeResource = memberTypeResource; this.delimiter = delimiter; diff --git a/ldi-core/version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/MemberInfo.java b/ldi-core/version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/MemberInfo.java index 851a006d5..bcdc2d440 100644 --- a/ldi-core/version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/MemberInfo.java +++ b/ldi-core/version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/MemberInfo.java @@ -1,5 +1,8 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects; +/** + * Contains the information of a version object member + */ public class MemberInfo { private final String versionOf; private final String observedAt; diff --git a/ldi-extensions/ldes-discoverer/pom.xml b/ldi-extensions/ldes-discoverer/pom.xml index 31c6000b1..ccab8a0d6 100644 --- a/ldi-extensions/ldes-discoverer/pom.xml +++ b/ldi-extensions/ldes-discoverer/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-extensions - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT ldes-discoverer diff --git a/ldi-extensions/pom.xml b/ldi-extensions/pom.xml index 05bd8c547..f01ef2099 100644 --- a/ldi-extensions/pom.xml +++ b/ldi-extensions/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT be.vlaanderen.informatievlaanderen.ldes.ldi diff --git a/ldi-nifi/ldi-nifi-common/pom.xml b/ldi-nifi/ldi-nifi-common/pom.xml index 7e84bcd10..df59af287 100644 --- a/ldi-nifi/ldi-nifi-common/pom.xml +++ b/ldi-nifi/ldi-nifi-common/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-nifi - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/CommonProperties.java b/ldi-nifi/ldi-nifi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/CommonProperties.java index 4374221dc..a3400dd86 100644 --- a/ldi-nifi/ldi-nifi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/CommonProperties.java +++ b/ldi-nifi/ldi-nifi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/CommonProperties.java @@ -1,5 +1,6 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config; +import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.validators.RDFLanguageValidator; import org.apache.jena.riot.Lang; import org.apache.jena.riot.RDFLanguages; import org.apache.nifi.components.PropertyDescriptor; @@ -10,6 +11,11 @@ public class CommonProperties { private CommonProperties() { } + /** + * The desired RDF format for output + */ + public static final Lang DEFAULT_DATA_DESTINATION_FORMAT = Lang.NQUADS; + public static final PropertyDescriptor DATA_SOURCE_FORMAT = new PropertyDescriptor.Builder() .name("DATA_SOURCE_FORMAT") .displayName("Data source format") @@ -18,7 +24,21 @@ private CommonProperties() { .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) .build(); + public static final PropertyDescriptor DATA_DESTINATION_FORMAT = new PropertyDescriptor.Builder() + .name("DATA_DESTINATION_FORMAT") + .displayName("Data destination format") + .description("RDF format identifier of the data destination") + .required(false) + .addValidator(new RDFLanguageValidator()) + .defaultValue(DEFAULT_DATA_DESTINATION_FORMAT.getHeaderString()) + .build(); + + public static Lang getDataSourceFormat(final ProcessContext context) { return RDFLanguages.nameToLang(context.getProperty(DATA_SOURCE_FORMAT).getValue()); } + + public static Lang getDataDestinationFormat(ProcessContext context) { + return RDFLanguages.nameToLang(context.getProperty(DATA_DESTINATION_FORMAT).getValue()); + } } diff --git a/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml b/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml index ec995c6c2..863310ba0 100644 --- a/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT archive-file-in diff --git a/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml b/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml index a513d3a2b..7a822b702 100644 --- a/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT archive-file-out diff --git a/ldi-nifi/ldi-nifi-processors/change-detection-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/change-detection-processor/pom.xml index d627a62b0..2b03e7626 100644 --- a/ldi-nifi/ldi-nifi-processors/change-detection-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/change-detection-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT change-detection-processor diff --git a/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml index c10d4d903..1aa5686eb 100644 --- a/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/create-version-object-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/CreateVersionObjectProcessor.java b/ldi-nifi/ldi-nifi-processors/create-version-object-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/CreateVersionObjectProcessor.java index bb8cc6be9..d85672284 100644 --- a/ldi-nifi/ldi-nifi-processors/create-version-object-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/CreateVersionObjectProcessor.java +++ b/ldi-nifi/ldi-nifi-processors/create-version-object-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/CreateVersionObjectProcessor.java @@ -23,6 +23,8 @@ import java.util.*; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.CommonProperties.DATA_DESTINATION_FORMAT; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.CommonProperties.getDataDestinationFormat; import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.CreateVersionObjectProcessorPropertyDescriptors.*; import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.CreateVersionObjectProcessorRelationships.*; import static org.apache.jena.rdf.model.ResourceFactory.createProperty; diff --git a/ldi-nifi/ldi-nifi-processors/create-version-object-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/CreateVersionObjectProcessorPropertyDescriptors.java b/ldi-nifi/ldi-nifi-processors/create-version-object-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/CreateVersionObjectProcessorPropertyDescriptors.java index d681b3880..2fa5b31de 100644 --- a/ldi-nifi/ldi-nifi-processors/create-version-object-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/CreateVersionObjectProcessorPropertyDescriptors.java +++ b/ldi-nifi/ldi-nifi-processors/create-version-object-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/CreateVersionObjectProcessorPropertyDescriptors.java @@ -1,9 +1,9 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config; import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.validators.RDFLanguageValidator; -import org.apache.jena.rdf.model.*; -import org.apache.jena.riot.Lang; -import org.apache.jena.riot.RDFLanguages; +import org.apache.jena.rdf.model.Property; +import org.apache.jena.rdf.model.Resource; +import org.apache.jena.rdf.model.ResourceFactory; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.components.Validator; import org.apache.nifi.processor.ProcessContext; @@ -13,7 +13,6 @@ public final class CreateVersionObjectProcessorPropertyDescriptors { private static final String DEFAULT_DATE_OBSERVED_VALUE_RDF_PROPERTY = "https://uri.etsi.org/ngsi-ld/observedAt"; private static final String DEFAULT_DELIMITER = "/"; private static final String DEFAULT_VERSION_OF_KEY = "http://purl.org/dc/terms/isVersionOf"; - private static final String DEFAULT_DATA_DESTINATION_FORMAT = "application/n-quads"; private static final String DEFAULT_DATA_INPUT_FORMAT = "application/ld+json"; private static final String DEFAULT_PROV_GENERATED_AT_TIME = "http://www.w3.org/ns/prov#generatedAtTime"; @@ -66,15 +65,6 @@ private CreateVersionObjectProcessorPropertyDescriptors() { .defaultValue(DEFAULT_DATA_INPUT_FORMAT) .build(); - public static final PropertyDescriptor DATA_DESTINATION_FORMAT = new PropertyDescriptor.Builder() - .name("DATA_DESTINATION_FORMAT") - .displayName("Data destination format") - .description("RDF format identifier of the data destination") - .required(false) - .addValidator(new RDFLanguageValidator()) - .defaultValue(DEFAULT_DATA_DESTINATION_FORMAT) - .build(); - public static final PropertyDescriptor GENERATED_AT_TIME_PROPERTY = new PropertyDescriptor.Builder() .name("GENERATED_AT_TIME_PROPERTY") .displayName("GeneratedAtTime property") @@ -109,8 +99,4 @@ public static Property getGeneratedAtTimeProperty(ProcessContext context) { } } - - public static Lang getDataDestinationFormat(ProcessContext context) { - return RDFLanguages.nameToLang(context.getProperty(DATA_DESTINATION_FORMAT).getValue()); - } } diff --git a/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml index ae966beb9..75be8f3e3 100644 --- a/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT geojson-to-wkt-processor diff --git a/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml index 42d07f6f5..96e0f2bcb 100644 --- a/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT json-to-ld-processor diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml index ae369c28b..26c10ba6c 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java index c282ff7e2..22bfb1163 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java @@ -48,6 +48,7 @@ import java.util.List; import java.util.Set; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.CommonProperties.DATA_DESTINATION_FORMAT; import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.LdesProcessorProperties.*; import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.LdesProcessorRelationships.DATA_RELATIONSHIP; import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.PersistenceProperties.*; diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/LdesProcessorProperties.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/LdesProcessorProperties.java index 022b9fcd1..ee6a2bd8f 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/LdesProcessorProperties.java +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/LdesProcessorProperties.java @@ -17,6 +17,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.CommonProperties.DATA_DESTINATION_FORMAT; import static java.lang.Boolean.FALSE; import static java.lang.Boolean.TRUE; @@ -25,10 +26,6 @@ public final class LdesProcessorProperties { * The expected RDF format of the LDES data source */ public static final Lang DEFAULT_DATA_SOURCE_FORMAT = Lang.JSONLD; - /** - * The desired RDF format for output - */ - public static final Lang DEFAULT_DATA_DESTINATION_FORMAT = Lang.NQUADS; private LdesProcessorProperties() { } @@ -50,15 +47,6 @@ private LdesProcessorProperties() { .defaultValue(DEFAULT_DATA_SOURCE_FORMAT.getHeaderString()) .build(); - public static final PropertyDescriptor DATA_DESTINATION_FORMAT = new PropertyDescriptor.Builder() - .name("DATA_DESTINATION_FORMAT") - .displayName("Data destination format") - .description("RDF format identifier of the data destination") - .required(false) - .addValidator(new RDFLanguageValidator()) - .defaultValue(DEFAULT_DATA_DESTINATION_FORMAT.getHeaderString()) - .build(); - public static final PropertyDescriptor TIMESTAMP_PATH = new PropertyDescriptor.Builder() .name("TIMESTAMP_PATH") .displayName("Timestamp path") diff --git a/ldi-nifi/ldi-nifi-processors/ldi-processors-bundle/pom.xml b/ldi-nifi/ldi-nifi-processors/ldi-processors-bundle/pom.xml index a9cbcf822..7ce312654 100644 --- a/ldi-nifi/ldi-nifi-processors/ldi-processors-bundle/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/ldi-processors-bundle/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT ldi-processors-bundle @@ -67,6 +67,12 @@ ${project.version} nar + + be.vlaanderen.informatievlaanderen.ldes.ldi.nifi + rml-adapter-processor + ${project.version} + nar + diff --git a/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml index 1444abb49..60562b260 100644 --- a/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml @@ -7,7 +7,7 @@ ldi-nifi-processors be.vlaanderen.informatievlaanderen.ldes.ldi.nifi - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT ngsiv2-to-ld-processor diff --git a/ldi-nifi/ldi-nifi-processors/pom.xml b/ldi-nifi/ldi-nifi-processors/pom.xml index 7a75973db..eac8ff20b 100644 --- a/ldi-nifi/ldi-nifi-processors/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-nifi - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 @@ -24,6 +24,7 @@ archive-file-in ldi-processors-bundle change-detection-processor + rml-adapter-processor diff --git a/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/pom.xml index 83a23f8aa..079f4d3f6 100644 --- a/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT rdf4j-repository-materialisation-processor diff --git a/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/pom.xml new file mode 100644 index 000000000..40126dc14 --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/pom.xml @@ -0,0 +1,28 @@ + + + 4.0.0 + + be.vlaanderen.informatievlaanderen.ldes.ldi.nifi + ldi-nifi-processors + 2.7.0-SNAPSHOT + + + rml-adapter-processor + nar + + + + be.vlaanderen.informatievlaanderen.ldes.ldi + rml-adapter + ${project.version} + + + be.vlaanderen.informatievlaanderen.ldes.ldi + ldi-nifi-common + ${project.version} + + + + \ No newline at end of file diff --git a/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/RmlAdapterProcessor.java b/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/RmlAdapterProcessor.java new file mode 100644 index 000000000..c48a4579c --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/RmlAdapterProcessor.java @@ -0,0 +1,101 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.processors; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.RmlAdapter; +import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.services.FlowManager; +import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; +import org.apache.jena.riot.RDFWriter; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; + +import java.util.Collection; +import java.util.List; +import java.util.Set; + +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.CommonProperties.*; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.RmlAdapterProperties.*; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.services.FlowManager.FAILURE; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.services.FlowManager.SUCCESS; + +@SuppressWarnings("java:S2160") // nifi handles equals/hashcode of processors +@Tags({"ldes", "vsds", "rml"}) +@CapabilityDescription("Converts a non-LD object to an RDF object") +public class RmlAdapterProcessor extends AbstractProcessor { + private RmlAdapter adapter; + + @Override + protected List getSupportedPropertyDescriptors() { + return List.of(RML_MAPPING_CONTENT, RML_MAPPING_FILE, DATA_DESTINATION_FORMAT); + } + + @Override + public Set getRelationships() { + return Set.of(SUCCESS, FAILURE); + } + + @Override + protected Collection customValidate(ValidationContext validationContext) { + final boolean isRmlMappingContentSet = validationContext.getProperty(RML_MAPPING_CONTENT).isSet(); + final boolean isRmlMappingFileSet = validationContext.getProperty(RML_MAPPING_FILE).isSet(); + + if (isRmlMappingContentSet && isRmlMappingFileSet) { + return List.of( + new ValidationResult.Builder() + .subject("RML mapping") + .valid(false) + .explanation("both RML mapping content and RML mapping file cannot be set at the same time") + .build() + ); + } + + if (!isRmlMappingContentSet && !isRmlMappingFileSet) { + return List.of( + new ValidationResult.Builder() + .subject("RML mapping") + .valid(false) + .explanation("either RML mapping content or RML mapping file must be set") + .build() + ); + } + + return List.of(new ValidationResult.Builder().valid(true).build()); + } + + @OnScheduled + public void onScheduled(final ProcessContext context) { + final String mappingString = getRmlMapping(context); + adapter = new RmlAdapter(mappingString); + } + + @Override + public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { + final FlowFile flowFile = session.get(); + + if (flowFile == null) { + return; + } + + final String content = FlowManager.receiveData(session, flowFile); + final String mimeType = flowFile.getAttribute("mime.type"); + + try { + adapter.apply(LdiAdapter.Content.of(content, mimeType)) + .map(model -> RDFWriter.source(model).lang(getDataDestinationFormat(context)).asString()) + .forEach(data -> FlowManager.sendRDFToRelation(session, data, SUCCESS, getDataDestinationFormat(context))); + session.remove(flowFile); + } catch (Exception e) { + getLogger().error("Error transforming input to an RDF object: {}", e.getMessage()); + session.transfer(flowFile, FAILURE); + } + + } +} diff --git a/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/RmlAdapterProperties.java b/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/RmlAdapterProperties.java new file mode 100644 index 000000000..18403f62d --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/RmlAdapterProperties.java @@ -0,0 +1,52 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config; + +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.util.StandardValidators; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; + +public class RmlAdapterProperties { + private RmlAdapterProperties() { + } + + public static final PropertyDescriptor RML_MAPPING_CONTENT = new PropertyDescriptor.Builder() + .name("RML_MAPPING_CONTENT") + .displayName("RML mapping content") + .description("Content of the RML mapping") + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .required(false) + .build(); + + public static final PropertyDescriptor RML_MAPPING_FILE = new PropertyDescriptor.Builder() + .name("RML_MAPPING_FILE") + .displayName("RML mapping file uri") + .description("File uri of the RML mapping") + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .addValidator(StandardValidators.FILE_EXISTS_VALIDATOR) + .required(false) + .build(); + + public static String getRmlMapping(ProcessContext context) { + if (context.getProperty(RML_MAPPING_CONTENT).isSet()) { + return context.getProperty(RML_MAPPING_CONTENT).getValue(); + } + + final String fileUri = context.getProperty(RML_MAPPING_FILE).getValue(); + + final Path path = Path.of(fileUri); + + if (!Files.isReadable(path)) { + throw new IllegalArgumentException("File does not exist or is not readable: %s".formatted(fileUri)); + } + + try { + return Files.readString(path); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor new file mode 100644 index 000000000..daa305a98 --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor @@ -0,0 +1 @@ +be.vlaanderen.informatievlaanderen.ldes.ldi.processors.RmlAdapterProcessor diff --git a/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/RmlAdapterProcessorTest.java b/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/RmlAdapterProcessorTest.java new file mode 100644 index 000000000..0a4ffa62b --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/RmlAdapterProcessorTest.java @@ -0,0 +1,124 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.processors; + +import org.apache.commons.io.FileUtils; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.riot.Lang; +import org.apache.jena.riot.RDFParser; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.opentest4j.AssertionFailedError; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Stream; + +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.CommonProperties.DATA_DESTINATION_FORMAT; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.RmlAdapterProperties.RML_MAPPING_CONTENT; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.RmlAdapterProperties.RML_MAPPING_FILE; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.services.FlowManager.FAILURE; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.services.FlowManager.SUCCESS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class RmlAdapterProcessorTest { + public static final Lang DESTINATION_FORMAT = Lang.NQUADS; + private TestRunner testRunner; + + @BeforeEach + void setUp() { + testRunner = TestRunners.newTestRunner(RmlAdapterProcessor.class); + testRunner.setProperty(DATA_DESTINATION_FORMAT, DESTINATION_FORMAT.getHeaderString()); + } + + private static Stream provideMappings() { + return Stream.of( + Arguments.of("smartphones/mapping.ttl", "smartphones/data.json", 5), + Arguments.of("usercarts/mapping.ttl", "usercarts/data.json", 20) + ); + } + + @ParameterizedTest + @MethodSource("provideMappings") + void test_mappings(String mappingFileName, String dataFileName, int expectedSuccessCount) { + testRunner.setProperty(RML_MAPPING_CONTENT, readFileContent(mappingFileName)); + final String data = readFileContent(dataFileName); + + testRunner.enqueue(data, Map.of("mime.type", "application/json")); + testRunner.run(); + + testRunner.assertQueueEmpty(); + testRunner.assertTransferCount(SUCCESS, expectedSuccessCount); + testRunner.assertTransferCount(FAILURE, 0); + } + + @Test + void test_awv_location() { + final Model expected = RDFParser.source("awv/location/expected.nt").lang(DESTINATION_FORMAT).toModel(); + final String data = readFileContent("awv/location/data.xml"); + testRunner.setProperty(RML_MAPPING_FILE, "src/test/resources/awv/location/mapping.ttl"); + + testRunner.enqueue(data, Map.of("mime.type", "application/xml")); + testRunner.run(); + + final List flowFiles = testRunner.getFlowFilesForRelationship(SUCCESS); + final Model result = RDFParser.fromString(flowFiles.getFirst().getContent()).lang(DESTINATION_FORMAT).toModel(); + + testRunner.assertQueueEmpty(); + assertThat(flowFiles).hasSize(1); + assertThat(result).matches(expected::isIsomorphicWith); + } + + @Test + void given_NonExistingFile_when_RunProcessor_then_ThrowException() { + testRunner.setProperty(RML_MAPPING_FILE, "non-existing-file.ttl"); + + assertThatThrownBy(() -> testRunner.run()) + .isInstanceOf(AssertionFailedError.class); + } + + @Test + void test_emptyFlowFile() { + testRunner.setProperty(RML_MAPPING_FILE, "src/test/resources/awv/location/mapping.ttl"); + + testRunner.run(); + + testRunner.assertTransferCount(SUCCESS, 0); + testRunner.assertTransferCount(FAILURE, 0); + } + + @Test + void given_BothRmlMappingContentAndFileAreSet_when_RunProcessor_then_ThrowException() { + testRunner.setProperty(RML_MAPPING_FILE, "src/test/resources/awv/location/mapping.ttl"); + testRunner.setProperty(RML_MAPPING_CONTENT, readFileContent("awv/location/mapping.ttl")); + + assertThatThrownBy(() -> testRunner.run()) + .isInstanceOf(AssertionFailedError.class) + .hasMessageContaining("both RML mapping content and RML mapping file cannot be set at the same time"); + } + + @Test + void given_NeitherRmlMappingContentAndFileAreSet_when_RunProcessor_then_ThrowException() { + assertThatThrownBy(() -> testRunner.run()) + .isInstanceOf(AssertionFailedError.class) + .hasMessageContaining("either RML mapping content or RML mapping file must be set"); + } + + private String readFileContent(String fileName) { + try { + final ClassLoader classLoader = getClass().getClassLoader(); + final File file = new File(Objects.requireNonNull(classLoader.getResource(fileName)).getFile()); + return FileUtils.readFileToString(file, "UTF-8"); + } catch (NullPointerException | IOException e) { + throw new RuntimeException(e); + } + } +} \ No newline at end of file diff --git a/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/test/resources/awv/location/data.xml b/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/test/resources/awv/location/data.xml new file mode 100644 index 000000000..cb7199538 --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/test/resources/awv/location/data.xml @@ -0,0 +1,17 @@ + + + 2019-12-05T10:26:49+01:00 + + H291L10 + Parking Kruibeke + A0140002 + 437 + 94,695 + R10 + 144474,5297 + 208293,5324 + 4,289731136 + 51,18460764 + + \ No newline at end of file diff --git a/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/test/resources/awv/location/expected.nt b/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/test/resources/awv/location/expected.nt new file mode 100644 index 000000000..488da500e --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/test/resources/awv/location/expected.nt @@ -0,0 +1,9 @@ + "http://www.w3.org/ns/sosa/FeatureOfInterest" . + "Parking Kruibeke" . + "H291L10" . + . + "A0140002" . + "94,695" . + "R10" . + "437"^^ . + "POINT(4.289731136 51.18460764)"^^ . \ No newline at end of file diff --git a/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/test/resources/awv/location/mapping.ttl b/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/test/resources/awv/location/mapping.ttl new file mode 100644 index 000000000..982744c31 --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/test/resources/awv/location/mapping.ttl @@ -0,0 +1,197 @@ +@prefix rr: . +@prefix rml: . +@prefix rdf: . +@prefix rdfs: . +@prefix ql: . +@prefix map: . +@prefix xsd: . +@prefix sd: . +@prefix dcterms: . +@prefix v: . +@prefix sosa: . +@prefix dct: . +@prefix miv: . +@prefix locn: . +@prefix geo: . +@prefix ldi: . + +map:fn_000 rml:logicalSource map:source_000 ; + rr:predicateObjectMap map:pom_010, map:pom_011, map:pomexec_000 . + +map:gm_000 rdf:type rr:GraphMap ; + rr:template "https://www.wegenenverkeer.be/id/verkeersmetingen/meetpunt/{./@unieke_id}" . + +map:jc_000 rr:child "./@unieke_id" ; + rr:parent "./@unieke_id" . + +map:map_geometry_000 rml:logicalSource map:source_000 ; + rdf:type rr:TriplesMap ; + rdfs:label "geometry" ; + rr:predicateObjectMap map:pom_009 ; + rr:subjectMap map:s_001 . + +map:map_location_000 rml:logicalSource map:source_000 ; + rdf:type rr:TriplesMap ; + rdfs:label "location" ; + rr:predicateObjectMap map:pom_000, map:pom_001, map:pom_002, map:pom_003, map:pom_004, map:pom_005, map:pom_006, map:pom_007, map:pom_008 ; + rr:subjectMap map:s_000 . + +map:om_000 rdf:type rr:ObjectMap ; + rr:constant "http://www.w3.org/ns/sosa/FeatureOfInterest" ; + rr:termType rr:IRI . + +map:om_001 rml:reference "../tijd_laatste_config_wijziging" ; + rdf:type rr:ObjectMap ; + rr:datatype xsd:DateTime ; + rr:termType rr:Literal . + +map:om_002 rml:reference "beschrijvende_id" ; + rdf:type rr:ObjectMap ; + rr:termType rr:Literal . + +map:om_003 rml:reference "volledige_naam" ; + rdf:type rr:ObjectMap ; + rr:termType rr:Literal . + +map:om_004 rml:reference "Ident_8" ; + rdf:type rr:ObjectMap ; + rr:termType rr:Literal . + +map:om_005 rml:reference "lve_nr" ; + rdf:type rr:ObjectMap ; + rr:datatype xsd:Integer ; + rr:termType rr:Literal . + +map:om_006 rml:reference "Kmp_Rsys" ; + rdf:type rr:ObjectMap ; + rr:termType rr:Literal . + +map:om_007 rml:reference "Rijstrook" ; + rdf:type rr:ObjectMap ; + rr:termType rr:Literal . + +map:om_008 rdf:type rr:ObjectMap ; + rr:joinCondition map:jc_000 ; + rr:parentTriplesMap map:map_geometry_000 . + +map:om_009 map:fn_000 ; + rdf:type ; + rr:datatype geo:wktLiteral ; + rr:termType rr:Literal . + +map:om_010 rdf:type rr:ObjectMap ; + rr:template "{lengtegraad_EPSG_4326} {breedtegraad_EPSG_4326}" ; + rr:termType rr:Literal . + +map:om_011 rdf:type rr:ObjectMap ; + rr:constant "POINT" ; + rr:termType rr:Literal . + +map:omexec_000 rr:constant ; + rr:termType rr:IRI . + +map:pm_000 rdf:type rr:PredicateMap ; + rr:constant rdf:type . + +map:pm_001 rdf:type rr:PredicateMap ; + rr:constant dct:modified . + +map:pm_002 rdf:type rr:PredicateMap ; + rr:constant dct:identifier . + +map:pm_003 rdf:type rr:PredicateMap ; + rr:constant rdfs:label . + +map:pm_004 rdf:type rr:PredicateMap ; + rr:constant miv:Ident_8 . + +map:pm_005 rdf:type rr:PredicateMap ; + rr:constant miv:lve_nr . + +map:pm_006 rdf:type rr:PredicateMap ; + rr:constant miv:Kmp_Rsys . + +map:pm_007 rdf:type rr:PredicateMap ; + rr:constant miv:Rijstrook . + +map:pm_008 rdf:type rr:PredicateMap ; + rr:constant locn:geometry . + +map:pm_009 rdf:type rr:PredicateMap ; + rr:constant geo:asWKT . + +map:pm_010 rdf:type rr:PredicateMap ; + rr:constant ldi:coordinates . + +map:pm_011 rdf:type rr:PredicateMap ; + rr:constant ldi:wktType . + +map:pmexec_000 rr:constant . + +map:pom_000 rdf:type rr:PredicateObjectMap ; + rr:objectMap map:om_000 ; + rr:predicateMap map:pm_000 . + +map:pom_001 rdf:type rr:PredicateObjectMap ; + rr:objectMap map:om_001 ; + rr:predicateMap map:pm_001 . + +map:pom_002 rdf:type rr:PredicateObjectMap ; + rr:objectMap map:om_002 ; + rr:predicateMap map:pm_002 . + +map:pom_003 rdf:type rr:PredicateObjectMap ; + rr:objectMap map:om_003 ; + rr:predicateMap map:pm_003 . + +map:pom_004 rdf:type rr:PredicateObjectMap ; + rr:objectMap map:om_004 ; + rr:predicateMap map:pm_004 . + +map:pom_005 rdf:type rr:PredicateObjectMap ; + rr:objectMap map:om_005 ; + rr:predicateMap map:pm_005 . + +map:pom_006 rdf:type rr:PredicateObjectMap ; + rr:objectMap map:om_006 ; + rr:predicateMap map:pm_006 . + +map:pom_007 rdf:type rr:PredicateObjectMap ; + rr:objectMap map:om_007 ; + rr:predicateMap map:pm_007 . + +map:pom_008 rdf:type rr:PredicateObjectMap ; + rr:objectMap map:om_008 ; + rr:predicateMap map:pm_008 . + +map:pom_009 rdf:type rr:PredicateObjectMap ; + rr:objectMap map:om_009 ; + rr:predicateMap map:pm_009 . + +map:pom_010 rdf:type rr:PredicateObjectMap ; + rr:objectMap map:om_010 ; + rr:predicateMap map:pm_010 . + +map:pom_011 rdf:type rr:PredicateObjectMap ; + rr:objectMap map:om_011 ; + rr:predicateMap map:pm_011 . + +map:pomexec_000 rr:objectMap map:omexec_000 ; + rr:predicateMap map:pmexec_000 . + +map:rules_000 map:map_geometry_000, map:map_location_000 ; + rdf:type . + +map:s_000 rdf:type rr:SubjectMap ; + rr:graphMap map:gm_000 ; + rr:template "https://www.wegenenverkeer.be/id/verkeersmetingen/meetpunt/{./@unieke_id}" . + +map:s_001 rdf:type rr:SubjectMap ; + rr:template "https://www.wegenenverkeer.be/id/verkeersmetingen/meetpunt/{./@unieke_id}/geo" . + +map:source_000 rml:iterator "/mivconfig/meetpunt" ; + rml:referenceFormulation ql:XPath ; + rml:source "locations.xml" ; + rdf:type rml:LogicalSource ; + rdfs:label "locations-source" . + diff --git a/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/test/resources/smartphones/data.json b/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/test/resources/smartphones/data.json new file mode 100644 index 000000000..9ce9b695a --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/test/resources/smartphones/data.json @@ -0,0 +1,95 @@ +{ + "products": [ + { + "id": 1, + "title": "iPhone 9", + "description": "An apple mobile which is nothing like apple", + "price": 549, + "discountPercentage": 12.96, + "rating": 4.69, + "stock": 94, + "brand": "Apple", + "category": "smartphones", + "thumbnail": "https://i.dummyjson.com/data/products/1/thumbnail.jpg", + "images": [ + "https://i.dummyjson.com/data/products/1/1.jpg", + "https://i.dummyjson.com/data/products/1/2.jpg", + "https://i.dummyjson.com/data/products/1/3.jpg", + "https://i.dummyjson.com/data/products/1/4.jpg", + "https://i.dummyjson.com/data/products/1/thumbnail.jpg" + ] + }, + { + "id": 2, + "title": "iPhone X", + "description": "SIM-Free, Model A19211 6.5-inch Super Retina HD display with OLED technology A12 Bionic chip with ...", + "price": 899, + "discountPercentage": 17.94, + "rating": 4.44, + "stock": 34, + "brand": "Apple", + "category": "smartphones", + "thumbnail": "https://i.dummyjson.com/data/products/2/thumbnail.jpg", + "images": [ + "https://i.dummyjson.com/data/products/2/1.jpg", + "https://i.dummyjson.com/data/products/2/2.jpg", + "https://i.dummyjson.com/data/products/2/3.jpg", + "https://i.dummyjson.com/data/products/2/thumbnail.jpg" + ] + }, + { + "id": 3, + "title": "Samsung Universe 9", + "description": "Samsung's new variant which goes beyond Galaxy to the Universe", + "price": 1249, + "discountPercentage": 15.46, + "rating": 4.09, + "stock": 36, + "brand": "Samsung", + "category": "smartphones", + "thumbnail": "https://i.dummyjson.com/data/products/3/thumbnail.jpg", + "images": [ + "https://i.dummyjson.com/data/products/3/1.jpg" + ] + }, + { + "id": 4, + "title": "OPPOF19", + "description": "OPPO F19 is officially announced on April 2021.", + "price": 280, + "discountPercentage": 17.91, + "rating": 4.3, + "stock": 123, + "brand": "OPPO", + "category": "smartphones", + "thumbnail": "https://i.dummyjson.com/data/products/4/thumbnail.jpg", + "images": [ + "https://i.dummyjson.com/data/products/4/1.jpg", + "https://i.dummyjson.com/data/products/4/2.jpg", + "https://i.dummyjson.com/data/products/4/3.jpg", + "https://i.dummyjson.com/data/products/4/4.jpg", + "https://i.dummyjson.com/data/products/4/thumbnail.jpg" + ] + }, + { + "id": 5, + "title": "Huawei P30", + "description": "Huawei’s re-badged P30 Pro New Edition was officially unveiled yesterday in Germany and now the device has made its way to the UK.", + "price": 499, + "discountPercentage": 10.58, + "rating": 4.09, + "stock": 32, + "brand": "Huawei", + "category": "smartphones", + "thumbnail": "https://i.dummyjson.com/data/products/5/thumbnail.jpg", + "images": [ + "https://i.dummyjson.com/data/products/5/1.jpg", + "https://i.dummyjson.com/data/products/5/2.jpg", + "https://i.dummyjson.com/data/products/5/3.jpg" + ] + } + ], + "total": 5, + "skip": 0, + "limit": 5 +} \ No newline at end of file diff --git a/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/test/resources/smartphones/mapping.ttl b/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/test/resources/smartphones/mapping.ttl new file mode 100644 index 000000000..fdce20fd4 --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/test/resources/smartphones/mapping.ttl @@ -0,0 +1,90 @@ +@prefix rr: . +@prefix rml: . +@prefix rdf: . +@prefix rdfs: . +@prefix ql: . +@prefix map: . +@prefix xsd: . +@prefix sd: . +@prefix foaf: . +@prefix ex: . + +map:gm_000 rdf:type rr:GraphMap ; + rr:template "http://example.com/{id}" . + +map:map_person_000 rml:logicalSource map:source_000 ; + rdf:type rr:TriplesMap ; + rdfs:label "person" ; + rr:predicateObjectMap map:pom_000, map:pom_001, map:pom_002, map:pom_003, map:pom_004 ; + rr:subjectMap map:s_000 . + +map:om_000 rdf:type rr:ObjectMap ; + rr:constant "http://xmlns.com/foaf/0.1/SmartPhone" ; + rr:termType rr:IRI . + +map:om_001 rml:reference "title" ; + rdf:type rr:ObjectMap ; + rr:termType rr:Literal . + +map:om_002 rml:reference "price" ; + rdf:type rr:ObjectMap ; + rr:datatype xsd:int ; + rr:termType rr:Literal . + +map:om_003 rml:reference "thumbnail" ; + rdf:type rr:ObjectMap ; + rr:termType rr:IRI . + +map:om_004 rml:reference "images" ; + rdf:type rr:ObjectMap ; + rr:termType rr:IRI . + +map:pm_000 rdf:type rr:PredicateMap ; + rr:constant rdf:type . + +map:pm_001 rdf:type rr:PredicateMap ; + rr:constant ex:title . + +map:pm_002 rdf:type rr:PredicateMap ; + rr:constant ex:price . + +map:pm_003 rdf:type rr:PredicateMap ; + rr:constant ex:thumbnail . + +map:pm_004 rdf:type rr:PredicateMap ; + rr:constant ex:images . + +map:pom_000 rdf:type rr:PredicateObjectMap ; + rr:objectMap map:om_000 ; + rr:predicateMap map:pm_000 . + +map:pom_001 rdf:type rr:PredicateObjectMap ; + rr:objectMap map:om_001 ; + rr:predicateMap map:pm_001 . + +map:pom_002 rdf:type rr:PredicateObjectMap ; + rr:objectMap map:om_002 ; + rr:predicateMap map:pm_002 . + +map:pom_003 rdf:type rr:PredicateObjectMap ; + rr:objectMap map:om_003 ; + rr:predicateMap map:pm_003 . + +map:pom_004 rdf:type rr:PredicateObjectMap ; + rr:objectMap map:om_004 ; + rr:predicateMap map:pm_004 . + +map:rules_000 map:map_person_000 ; + rdf:type . + +map:s_000 rdf:type rr:SubjectMap ; + rr:template "http://example.com/{id}" ; + rr:graphMap [ + rr:template "http://example.com/cart/{id}" ; + ] . + +map:source_000 rml:iterator "$.products[*]" ; + rml:referenceFormulation ql:JSONPath ; + rml:source "data.json" ; + rdf:type rml:LogicalSource . + diff --git a/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/test/resources/usercarts/data.json b/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/test/resources/usercarts/data.json new file mode 100644 index 000000000..bf201056a --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/test/resources/usercarts/data.json @@ -0,0 +1,2938 @@ +{ + "users": [ + { + "id": 1, + "firstName": "Terry", + "lastName": "Medhurst", + "maidenName": "Smitham", + "age": 50, + "gender": "male", + "email": "atuny0@sohu.com", + "phone": "+63 791 675 8914", + "username": "atuny0", + "password": "9uQFF1Lh", + "birthDate": "2000-12-25", + "image": "https://robohash.org/hicveldicta.png", + "bloodGroup": "A−", + "height": 189, + "weight": 75.4, + "eyeColor": "Green", + "hair": { + "color": "Black", + "type": "Strands" + }, + "domain": "slashdot.org", + "ip": "117.29.86.254", + "address": { + "address": "1745 T Street Southeast", + "city": "Washington", + "coordinates": { + "lat": 38.867033, + "lng": -76.979235 + }, + "postalCode": "20020", + "state": "DC" + }, + "macAddress": "13:69:BA:56:A3:74", + "university": "Capitol University", + "bank": { + "cardExpire": "06/22", + "cardNumber": "50380955204220685", + "cardType": "maestro", + "currency": "Peso", + "iban": "NO17 0695 2754 967" + }, + "company": { + "address": { + "address": "629 Debbie Drive", + "city": "Nashville", + "coordinates": { + "lat": 36.208114, + "lng": -86.58621199999999 + }, + "postalCode": "37076", + "state": "TN" + }, + "department": "Marketing", + "name": "Blanda-O'Keefe", + "title": "Help Desk Operator" + }, + "ein": "20-9487066", + "ssn": "661-64-2976", + "userAgent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/12.0.702.0 Safari/534.24" + }, + { + "id": 2, + "firstName": "Sheldon", + "lastName": "Quigley", + "maidenName": "Cole", + "age": 28, + "gender": "male", + "email": "hbingley1@plala.or.jp", + "phone": "+7 813 117 7139", + "username": "hbingley1", + "password": "CQutx25i8r", + "birthDate": "2003-08-02", + "image": "https://robohash.org/doloremquesintcorrupti.png", + "bloodGroup": "O+", + "height": 187, + "weight": 74, + "eyeColor": "Brown", + "hair": { + "color": "Blond", + "type": "Curly" + }, + "domain": "51.la", + "ip": "253.240.20.181", + "address": { + "address": "6007 Applegate Lane", + "city": "Louisville", + "coordinates": { + "lat": 38.1343013, + "lng": -85.6498512 + }, + "postalCode": "40219", + "state": "KY" + }, + "macAddress": "13:F1:00:DA:A4:12", + "university": "Stavropol State Technical University", + "bank": { + "cardExpire": "10/23", + "cardNumber": "5355920631952404", + "cardType": "mastercard", + "currency": "Ruble", + "iban": "MD63 L6YC 8YH4 QVQB XHIK MTML" + }, + "company": { + "address": { + "address": "8821 West Myrtle Avenue", + "city": "Glendale", + "coordinates": { + "lat": 33.5404296, + "lng": -112.2488391 + }, + "postalCode": "85305", + "state": "AZ" + }, + "department": "Services", + "name": "Aufderhar-Cronin", + "title": "Senior Cost Accountant" + }, + "ein": "52-5262907", + "ssn": "447-08-9217", + "userAgent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/534.30 (KHTML, like Gecko) Ubuntu/11.04 Chromium/12.0.742.112 Chrome/12.0.742.112 Safari/534.30" + }, + { + "id": 3, + "firstName": "Terrill", + "lastName": "Hills", + "maidenName": "Hoeger", + "age": 38, + "gender": "male", + "email": "rshawe2@51.la", + "phone": "+63 739 292 7942", + "username": "rshawe2", + "password": "OWsTbMUgFc", + "birthDate": "1992-12-30", + "image": "https://robohash.org/consequunturautconsequatur.png", + "bloodGroup": "A−", + "height": 200, + "weight": 105.3, + "eyeColor": "Gray", + "hair": { + "color": "Blond", + "type": "Very curly" + }, + "domain": "earthlink.net", + "ip": "205.226.160.3", + "address": { + "address": "560 Penstock Drive", + "city": "Grass Valley", + "coordinates": { + "lat": 39.213076, + "lng": -121.077583 + }, + "postalCode": "95945", + "state": "CA" + }, + "macAddress": "F2:88:58:64:F7:76", + "university": "University of Cagayan Valley", + "bank": { + "cardExpire": "10/23", + "cardNumber": "3586082982526703", + "cardType": "jcb", + "currency": "Peso", + "iban": "AT24 1095 9625 1434 9703" + }, + "company": { + "address": { + "address": "18 Densmore Drive", + "city": "Essex", + "coordinates": { + "lat": 44.492953, + "lng": -73.101883 + }, + "postalCode": "05452", + "state": "VT" + }, + "department": "Marketing", + "name": "Lindgren LLC", + "title": "Mechanical Systems Engineer" + }, + "ein": "48-3951994", + "ssn": "633-89-1926", + "userAgent": "Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:21.0.0) Gecko/20121011 Firefox/21.0.0" + }, + { + "id": 4, + "firstName": "Miles", + "lastName": "Cummerata", + "maidenName": "Maggio", + "age": 49, + "gender": "male", + "email": "yraigatt3@nature.com", + "phone": "+86 461 145 4186", + "username": "yraigatt3", + "password": "sRQxjPfdS", + "birthDate": "1969-01-16", + "image": "https://robohash.org/facilisdignissimosdolore.png", + "bloodGroup": "B+", + "height": 157, + "weight": 95.9, + "eyeColor": "Gray", + "hair": { + "color": "Blond", + "type": "Very curly" + }, + "domain": "homestead.com", + "ip": "243.20.78.113", + "address": { + "address": "150 Carter Street", + "city": "Manchester", + "coordinates": { + "lat": 41.76556000000001, + "lng": -72.473091 + }, + "postalCode": "06040", + "state": "CT" + }, + "macAddress": "03:45:58:59:5A:7B", + "university": "Shenyang Pharmaceutical University", + "bank": { + "cardExpire": "07/24", + "cardNumber": "3580047879369323", + "cardType": "jcb", + "currency": "Yuan Renminbi", + "iban": "KZ43 658B M6VS TZOU OXSO" + }, + "company": { + "address": { + "address": "210 Green Road", + "city": "Manchester", + "coordinates": { + "lat": 41.7909099, + "lng": -72.51195129999999 + }, + "postalCode": "06042", + "state": "CT" + }, + "department": "Business Development", + "name": "Wolff and Sons", + "title": "Paralegal" + }, + "ein": "71-3644334", + "ssn": "487-28-6642", + "userAgent": "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.17 Safari/537.11" + }, + { + "id": 5, + "firstName": "Mavis", + "lastName": "Schultz", + "maidenName": "Yundt", + "age": 38, + "gender": "male", + "email": "kmeus4@upenn.edu", + "phone": "+372 285 771 1911", + "username": "kmeus4", + "password": "aUTdmmmbH", + "birthDate": "1968-11-03", + "image": "https://robohash.org/adverovelit.png", + "bloodGroup": "O+", + "height": 188, + "weight": 106.3, + "eyeColor": "Brown", + "hair": { + "color": "Brown", + "type": "Curly" + }, + "domain": "columbia.edu", + "ip": "103.72.86.183", + "address": { + "address": "2721 Lindsay Avenue", + "city": "Louisville", + "coordinates": { + "lat": 38.263793, + "lng": -85.700243 + }, + "postalCode": "40206", + "state": "KY" + }, + "macAddress": "F8:04:9E:ED:C0:68", + "university": "Estonian University of Life Sciences", + "bank": { + "cardExpire": "01/24", + "cardNumber": "4917245076693618", + "cardType": "visa-electron", + "currency": "Euro", + "iban": "IT41 T114 5127 716J RGYB ZRUX DSJ" + }, + "company": { + "address": { + "address": "8398 West Denton Lane", + "city": "Glendale", + "coordinates": { + "lat": 33.515353, + "lng": -112.240812 + }, + "postalCode": "85305", + "state": "AZ" + }, + "department": "Support", + "name": "Adams Inc", + "title": "Web Developer I" + }, + "ein": "18-7178563", + "ssn": "667-98-5357", + "userAgent": "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.1 Safari/535.1" + }, + { + "id": 6, + "firstName": "Alison", + "lastName": "Reichert", + "maidenName": "Franecki", + "age": 21, + "gender": "female", + "email": "jtreleven5@nhs.uk", + "phone": "+351 527 735 3642", + "username": "jtreleven5", + "password": "zY1nE46Zm", + "birthDate": "1969-07-21", + "image": "https://robohash.org/laboriosamfacilisrem.png", + "bloodGroup": "A+", + "height": 149, + "weight": 105.7, + "eyeColor": "Amber", + "hair": { + "color": "Blond", + "type": "Straight" + }, + "domain": "bandcamp.com", + "ip": "49.201.206.36", + "address": { + "address": "18 Densmore Drive", + "city": "Essex", + "coordinates": { + "lat": 44.492953, + "lng": -73.101883 + }, + "postalCode": "05452", + "state": "VT" + }, + "macAddress": "6C:34:D0:4B:4E:81", + "university": "Universidade da Beira Interior", + "bank": { + "cardExpire": "03/22", + "cardNumber": "345675888286047", + "cardType": "americanexpress", + "currency": "Euro", + "iban": "LB69 1062 QCY5 XS5T VOKU KJFG XP4S" + }, + "company": { + "address": { + "address": "6231 North 67th Avenue", + "city": "Glendale", + "coordinates": { + "lat": 33.5279666, + "lng": -112.2022551 + }, + "postalCode": "85301", + "state": "AZ" + }, + "department": "Accounting", + "name": "D'Amore and Sons", + "title": "Civil Engineer" + }, + "ein": "78-3192791", + "ssn": "158-68-0184", + "userAgent": "Mozilla/5.0 (Windows; U; Windows NT 6.0; nb-NO) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5" + }, + { + "id": 7, + "firstName": "Oleta", + "lastName": "Abbott", + "maidenName": "Wyman", + "age": 31, + "gender": "female", + "email": "dpettegre6@columbia.edu", + "phone": "+62 640 802 7111", + "username": "dpettegre6", + "password": "YVmhktgYVS", + "birthDate": "1982-02-21", + "image": "https://robohash.org/cupiditatererumquos.png", + "bloodGroup": "B−", + "height": 172, + "weight": 78.1, + "eyeColor": "Blue", + "hair": { + "color": "Chestnut", + "type": "Wavy" + }, + "domain": "ovh.net", + "ip": "25.207.107.146", + "address": { + "address": "637 Britannia Drive", + "city": "Vallejo", + "coordinates": { + "lat": 38.10476999999999, + "lng": -122.193849 + }, + "postalCode": "94591", + "state": "CA" + }, + "macAddress": "48:2D:A0:67:19:E0", + "university": "Institut Sains dan Teknologi Al Kamal", + "bank": { + "cardExpire": "10/23", + "cardNumber": "3589640949470047", + "cardType": "jcb", + "currency": "Rupiah", + "iban": "GI97 IKPF 9DUO X25M FG8D UXY" + }, + "company": { + "address": { + "address": "1407 Walden Court", + "city": "Crofton", + "coordinates": { + "lat": 39.019306, + "lng": -76.660653 + }, + "postalCode": "21114", + "state": "MD" + }, + "department": "Product Management", + "name": "Schimmel, Wilderman and Orn", + "title": "Sales Associate" + }, + "ein": "29-1568401", + "ssn": "478-11-2206", + "userAgent": "Mozilla/5.0 (Windows; U; Windows NT 5.1; ru-RU) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5" + }, + { + "id": 8, + "firstName": "Ewell", + "lastName": "Mueller", + "maidenName": "Durgan", + "age": 29, + "gender": "male", + "email": "ggude7@chron.com", + "phone": "+86 946 297 2275", + "username": "ggude7", + "password": "MWwlaeWcOoF6", + "birthDate": "1964-08-24", + "image": "https://robohash.org/quiaharumsapiente.png", + "bloodGroup": "A−", + "height": 146, + "weight": 52.1, + "eyeColor": "Blue", + "hair": { + "color": "Chestnut", + "type": "Wavy" + }, + "domain": "homestead.com", + "ip": "91.200.56.127", + "address": { + "address": "5601 West Crocus Drive", + "city": "Glendale", + "coordinates": { + "lat": 33.6152469, + "lng": -112.179737 + }, + "postalCode": "85306", + "state": "AZ" + }, + "macAddress": "72:DA:1B:D7:30:E9", + "university": "Wenzhou Medical College", + "bank": { + "cardExpire": "09/23", + "cardNumber": "30549925358905", + "cardType": "diners-club-carte-blanche", + "currency": "Yuan Renminbi", + "iban": "CY02 9914 5346 0PMT G6XW TP0R AWRZ" + }, + "company": { + "address": { + "address": "81 Seaton Place Northwest", + "city": "Washington", + "coordinates": { + "lat": 38.9149499, + "lng": -77.01170259999999 + }, + "postalCode": "20001", + "state": "DC" + }, + "department": "Services", + "name": "Corkery, Reichert and Hodkiewicz", + "title": "Clinical Specialist" + }, + "ein": "88-4396827", + "ssn": "238-41-5528", + "userAgent": "Mozilla/5.0 (X11; Linux amd64) AppleWebKit/534.36 (KHTML, like Gecko) Chrome/13.0.766.0 Safari/534.36" + }, + { + "id": 9, + "firstName": "Demetrius", + "lastName": "Corkery", + "maidenName": "Gleason", + "age": 22, + "gender": "male", + "email": "nloiterton8@aol.com", + "phone": "+86 356 590 9727", + "username": "nloiterton8", + "password": "HTQxxXV9Bq4", + "birthDate": "1971-03-11", + "image": "https://robohash.org/excepturiiuremolestiae.png", + "bloodGroup": "A+", + "height": 170, + "weight": 97.1, + "eyeColor": "Green", + "hair": { + "color": "Brown", + "type": "Strands" + }, + "domain": "goodreads.com", + "ip": "78.170.185.120", + "address": { + "address": "5403 Illinois Avenue", + "city": "Nashville", + "coordinates": { + "lat": 36.157077, + "lng": -86.853827 + }, + "postalCode": "37209", + "state": "TN" + }, + "macAddress": "98:EE:94:A2:91:C4", + "university": "Nanjing University of Economics", + "bank": { + "cardExpire": "02/24", + "cardNumber": "5372664789004621", + "cardType": "mastercard", + "currency": "Yuan Renminbi", + "iban": "BR68 9829 0581 3669 5088 5533 025N V" + }, + "company": { + "address": { + "address": "12245 West 71st Place", + "city": "Arvada", + "coordinates": { + "lat": 39.8267078, + "lng": -105.1366798 + }, + "postalCode": "80004", + "state": "CO" + }, + "department": "Human Resources", + "name": "Gorczany Group", + "title": "Community Outreach Specialist" + }, + "ein": "14-1066382", + "ssn": "717-26-3759", + "userAgent": "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_4_11; de) AppleWebKit/528.4+ (KHTML, like Gecko) Version/4.0dp1 Safari/526.11.2" + }, + { + "id": 10, + "firstName": "Eleanora", + "lastName": "Price", + "maidenName": "Cummings", + "age": 37, + "gender": "female", + "email": "umcgourty9@jalbum.net", + "phone": "+60 184 408 0824", + "username": "umcgourty9", + "password": "i0xzpX", + "birthDate": "1958-08-11", + "image": "https://robohash.org/aliquamcumqueiure.png", + "bloodGroup": "O+", + "height": 198, + "weight": 48, + "eyeColor": "Blue", + "hair": { + "color": "Chestnut", + "type": "Wavy" + }, + "domain": "alibaba.com", + "ip": "73.15.179.178", + "address": { + "address": "8821 West Myrtle Avenue", + "city": "Glendale", + "coordinates": { + "lat": 33.5404296, + "lng": -112.2488391 + }, + "postalCode": "85305", + "state": "AZ" + }, + "macAddress": "BC:A9:D8:98:CB:0B", + "university": "Melaka City Polytechnic", + "bank": { + "cardExpire": "01/24", + "cardNumber": "3557806620295254", + "cardType": "jcb", + "currency": "Ringgit", + "iban": "GT40 DWAD 9UHA VEOZ ZF4J 2Y0F OOFD" + }, + "company": { + "address": { + "address": "1649 Timberridge Court", + "city": "Fayetteville", + "coordinates": { + "lat": 36.084563, + "lng": -94.206082 + }, + "postalCode": "72704", + "state": "AR" + }, + "department": "Marketing", + "name": "Bins Group", + "title": "Senior Sales Associate" + }, + "ein": "21-5278484", + "ssn": "544-66-0745", + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.6 Safari/537.11" + }, + { + "id": 11, + "firstName": "Marcel", + "lastName": "Jones", + "maidenName": "Smith", + "age": 39, + "gender": "male", + "email": "acharlota@liveinternet.ru", + "phone": "+967 253 210 0344", + "username": "acharlota", + "password": "M9lbMdydMN", + "birthDate": "1961-09-12", + "image": "https://robohash.org/impeditautest.png", + "bloodGroup": "B−", + "height": 203, + "weight": 63.7, + "eyeColor": "Amber", + "hair": { + "color": "Black", + "type": "Straight" + }, + "domain": "feedburner.com", + "ip": "137.235.164.173", + "address": { + "address": "2203 7th Street Road", + "city": "Louisville", + "coordinates": { + "lat": 38.218107, + "lng": -85.779006 + }, + "postalCode": "40208", + "state": "KY" + }, + "macAddress": "59:E8:70:5A:E5:D6", + "university": "Hodeidah University", + "bank": { + "cardExpire": "05/24", + "cardNumber": "5893925889459720", + "cardType": "maestro", + "currency": "Rial", + "iban": "NL97 UWMY 2503 2999 43" + }, + "company": { + "address": { + "address": "308 Woodleaf Court", + "city": "Glen Burnie", + "coordinates": { + "lat": 39.1425931, + "lng": -76.6238441 + }, + "postalCode": "21061", + "state": "MD" + }, + "department": "Business Development", + "name": "Kuhn-Harber", + "title": "Account Executive" + }, + "ein": "09-3791007", + "ssn": "342-54-8422", + "userAgent": "Mozilla/5.0 (Windows NT 5.2) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.792.0 Safari/535.1" + }, + { + "id": 12, + "firstName": "Assunta", + "lastName": "Rath", + "maidenName": "Heller", + "age": 42, + "gender": "female", + "email": "rhallawellb@dropbox.com", + "phone": "+380 962 542 6549", + "username": "rhallawellb", + "password": "esTkitT1r", + "birthDate": "1990-12-14", + "image": "https://robohash.org/namquaerataut.png", + "bloodGroup": "O−", + "height": 168, + "weight": 96.8, + "eyeColor": "Gray", + "hair": { + "color": "Black", + "type": "Very curly" + }, + "domain": "123-reg.co.uk", + "ip": "74.80.53.208", + "address": { + "address": "6463 Vrain Street", + "city": "Arvada", + "coordinates": { + "lat": 39.814056, + "lng": -105.046913 + }, + "postalCode": "80003", + "state": "CO" + }, + "macAddress": "9B:DC:21:C2:30:A3", + "university": "Kiev Slavonic University", + "bank": { + "cardExpire": "09/22", + "cardNumber": "5602230671060360", + "cardType": "bankcard", + "currency": "Hryvnia", + "iban": "KW76 VNLA LX0Y DMDE PFS8 FVKP VMDF AV" + }, + "company": { + "address": { + "address": "388 East Main Street", + "coordinates": { + "lat": 43.9727945, + "lng": -73.1023187 + }, + "postalCode": "05753", + "state": "VT" + }, + "department": "Product Management", + "name": "Goodwin-Skiles", + "title": "Developer II" + }, + "ein": "14-1242349", + "ssn": "116-51-6131", + "userAgent": "Mozilla/5.0 (X11; CrOS i686 4319.74.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.57 Safari/537.36" + }, + { + "id": 13, + "firstName": "Trace", + "lastName": "Douglas", + "maidenName": "Lemke", + "age": 26, + "gender": "male", + "email": "lgribbinc@posterous.com", + "phone": "+1 609 937 3468", + "username": "lgribbinc", + "password": "ftGj8LZTtv9g", + "birthDate": "1967-07-23", + "image": "https://robohash.org/voluptatemsintnulla.png", + "bloodGroup": "O+", + "height": 181, + "weight": 56.5, + "eyeColor": "Amber", + "hair": { + "color": "Auburn", + "type": "Straight" + }, + "domain": "histats.com", + "ip": "163.245.232.27", + "address": { + "address": "87 Horseshoe Drive", + "city": "West Windsor", + "coordinates": { + "lat": 43.4731793, + "lng": -72.4967532 + }, + "postalCode": "05037", + "state": "VT" + }, + "macAddress": "B9:21:ED:9F:B8:9E", + "university": "Dallas Christian College", + "bank": { + "cardExpire": "01/23", + "cardNumber": "3556299106119514", + "cardType": "jcb", + "currency": "Dollar", + "iban": "AE47 4194 4544 3401 3419 286" + }, + "company": { + "address": { + "address": "310 Timrod Road", + "city": "Manchester", + "coordinates": { + "lat": 41.756758, + "lng": -72.493501 + }, + "postalCode": "06040", + "state": "CT" + }, + "department": "Research and Development", + "name": "Casper Inc", + "title": "Sales Associate" + }, + "ein": "94-0648182", + "ssn": "217-05-3082", + "userAgent": "Mozilla/5.0 (Windows; U; Windows NT 5.1; ru-RU) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4" + }, + { + "id": 14, + "firstName": "Enoch", + "lastName": "Lynch", + "maidenName": "Heidenreich", + "age": 21, + "gender": "male", + "email": "mturleyd@tumblr.com", + "phone": "+94 912 100 5118", + "username": "mturleyd", + "password": "GyLnCB8gNIp", + "birthDate": "1979-08-25", + "image": "https://robohash.org/quisequienim.png", + "bloodGroup": "O+", + "height": 150, + "weight": 100.3, + "eyeColor": "Green", + "hair": { + "color": "Auburn", + "type": "Strands" + }, + "domain": "icio.us", + "ip": "174.238.43.126", + "address": { + "address": "60 Desousa Drive", + "city": "Manchester", + "coordinates": { + "lat": 41.7409259, + "lng": -72.5619104 + }, + "postalCode": "06040", + "state": "CT" + }, + "macAddress": "52:11:E1:31:35:C1", + "university": "University of Sri Jayawardenapura", + "bank": { + "cardExpire": "11/23", + "cardNumber": "5339467937996728", + "cardType": "mastercard", + "currency": "Rupee", + "iban": "SI28 0812 7967 0952 944" + }, + "company": { + "address": { + "address": "21950 Arnold Center Road", + "city": "Carson", + "coordinates": { + "lat": 33.8272706, + "lng": -118.2302826 + }, + "postalCode": "90810", + "state": "CA" + }, + "department": "Sales", + "name": "Schoen Inc", + "title": "Professor" + }, + "ein": "61-8316825", + "ssn": "742-81-1714", + "userAgent": "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-en) AppleWebKit/533.16 (KHTML, like Gecko) Version/4.1 Safari/533.16" + }, + { + "id": 15, + "firstName": "Jeanne", + "lastName": "Halvorson", + "maidenName": "Cummerata", + "age": 26, + "gender": "female", + "email": "kminchelle@qq.com", + "phone": "+86 581 108 7855", + "username": "kminchelle", + "password": "0lelplR", + "birthDate": "1996-02-02", + "image": "https://robohash.org/autquiaut.png", + "bloodGroup": "A+", + "height": 176, + "weight": 45.7, + "eyeColor": "Amber", + "hair": { + "color": "Blond", + "type": "Straight" + }, + "domain": "google.co.uk", + "ip": "78.43.74.226", + "address": { + "address": "4 Old Colony Way", + "city": "Yarmouth", + "coordinates": { + "lat": 41.697168, + "lng": -70.189992 + }, + "postalCode": "02664", + "state": "MA" + }, + "macAddress": "D9:DB:D9:5A:01:09", + "university": "Donghua University, Shanghai", + "bank": { + "cardExpire": "10/23", + "cardNumber": "3588859507772914", + "cardType": "jcb", + "currency": "Yuan Renminbi", + "iban": "FO12 1440 0396 8902 56" + }, + "company": { + "address": { + "address": "22572 Toreador Drive", + "city": "Salinas", + "coordinates": { + "lat": 36.602449, + "lng": -121.699071 + }, + "postalCode": "93908", + "state": "CA" + }, + "department": "Marketing", + "name": "Hahn-MacGyver", + "title": "Software Test Engineer IV" + }, + "ein": "62-0561095", + "ssn": "855-43-8639", + "userAgent": "Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.14 Safari/534.24" + }, + { + "id": 16, + "firstName": "Trycia", + "lastName": "Fadel", + "maidenName": "Rosenbaum", + "age": 41, + "gender": "female", + "email": "dpierrof@vimeo.com", + "phone": "+420 833 708 0340", + "username": "dpierrof", + "password": "Vru55Y4tufI4", + "birthDate": "1963-07-03", + "image": "https://robohash.org/porronumquamid.png", + "bloodGroup": "B+", + "height": 166, + "weight": 87.2, + "eyeColor": "Gray", + "hair": { + "color": "Black", + "type": "Very curly" + }, + "domain": "tamu.edu", + "ip": "82.170.69.15", + "address": { + "address": "314 South 17th Street", + "city": "Nashville", + "coordinates": { + "lat": 36.1719075, + "lng": -86.740228 + }, + "postalCode": "37206", + "state": "TN" + }, + "macAddress": "3D:21:5B:9F:76:FF", + "university": "Technical University of Mining and Metallurgy Ostrava", + "bank": { + "cardExpire": "07/23", + "cardNumber": "6378941710246212", + "cardType": "instapayment", + "currency": "Koruna", + "iban": "CH94 4961 5QY1 VPV1 NGIP P" + }, + "company": { + "address": { + "address": "1407 Walden Court", + "city": "Crofton", + "coordinates": { + "lat": 39.019306, + "lng": -76.660653 + }, + "postalCode": "21114", + "state": "MD" + }, + "department": "Research and Development", + "name": "Steuber, Considine and Padberg", + "title": "Geological Engineer" + }, + "ein": "75-1816504", + "ssn": "677-73-1525", + "userAgent": "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.872.0 Safari/535.2" + }, + { + "id": 17, + "firstName": "Bradford", + "lastName": "Prohaska", + "maidenName": "Bins", + "age": 43, + "gender": "male", + "email": "vcholdcroftg@ucoz.com", + "phone": "+420 874 628 3710", + "username": "vcholdcroftg", + "password": "mSPzYZfR", + "birthDate": "1975-10-20", + "image": "https://robohash.org/accusantiumvoluptateseos.png", + "bloodGroup": "O−", + "height": 199, + "weight": 94.3, + "eyeColor": "Brown", + "hair": { + "color": "Black", + "type": "Curly" + }, + "domain": "wix.com", + "ip": "75.75.234.243", + "address": { + "address": "1649 Timberridge Court", + "city": "Fayetteville", + "coordinates": { + "lat": 36.084563, + "lng": -94.206082 + }, + "postalCode": "72704", + "state": "AR" + }, + "macAddress": "47:FA:F7:94:7B:5D", + "university": "Technical University of Mining and Metallurgy Ostrava", + "bank": { + "cardExpire": "05/24", + "cardNumber": "3574627048005672", + "cardType": "jcb", + "currency": "Koruna", + "iban": "SI81 7221 0344 9088 864" + }, + "company": { + "address": { + "address": "20930 Todd Valley Road", + "city": "Foresthill", + "coordinates": { + "lat": 38.989466, + "lng": -120.883108 + }, + "postalCode": "95631", + "state": "CA" + }, + "department": "Sales", + "name": "Bogisich and Sons", + "title": "Operator" + }, + "ein": "92-8837697", + "ssn": "795-36-7752", + "userAgent": "Mozilla/5.0 (Windows NT 5.2) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.813.0 Safari/535.1" + }, + { + "id": 18, + "firstName": "Arely", + "lastName": "Skiles", + "maidenName": "Monahan", + "age": 42, + "gender": "male", + "email": "sberminghamh@chron.com", + "phone": "+55 886 766 8617", + "username": "sberminghamh", + "password": "cAjfb8vg", + "birthDate": "1958-02-05", + "image": "https://robohash.org/nihilharumqui.png", + "bloodGroup": "AB−", + "height": 192, + "weight": 97, + "eyeColor": "Amber", + "hair": { + "color": "Brown", + "type": "Straight" + }, + "domain": "seesaa.net", + "ip": "29.82.54.30", + "address": { + "address": "5461 West Shades Valley Drive", + "city": "Montgomery", + "coordinates": { + "lat": 32.296422, + "lng": -86.34280299999999 + }, + "postalCode": "36108", + "state": "AL" + }, + "macAddress": "61:0C:8F:92:48:D5", + "university": "Universidade Estadual do Ceará", + "bank": { + "cardExpire": "09/24", + "cardNumber": "3578078357052002", + "cardType": "jcb", + "currency": "Real", + "iban": "FR79 7925 2903 77HF 2ZY6 TU4M T84" + }, + "company": { + "address": { + "address": "3162 Martin Luther King Junior Boulevard", + "city": "Fayetteville", + "coordinates": { + "lat": 36.05233310000001, + "lng": -94.2056987 + }, + "postalCode": "72704", + "state": "AR" + }, + "department": "Support", + "name": "Metz Group", + "title": "VP Accounting" + }, + "ein": "55-4062919", + "ssn": "551-74-1349", + "userAgent": "Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0" + }, + { + "id": 19, + "firstName": "Gust", + "lastName": "Purdy", + "maidenName": "Abshire", + "age": 46, + "gender": "male", + "email": "bleveragei@so-net.ne.jp", + "phone": "+86 886 889 0258", + "username": "bleveragei", + "password": "UZGAiqPqWQHQ", + "birthDate": "1989-10-15", + "image": "https://robohash.org/delenitipraesentiumvoluptatum.png", + "bloodGroup": "A−", + "height": 167, + "weight": 65.3, + "eyeColor": "Amber", + "hair": { + "color": "Black", + "type": "Straight" + }, + "domain": "homestead.com", + "ip": "90.202.216.39", + "address": { + "address": "629 Debbie Drive", + "city": "Nashville", + "coordinates": { + "lat": 36.208114, + "lng": -86.58621199999999 + }, + "postalCode": "37076", + "state": "TN" + }, + "macAddress": "22:98:8D:97:2D:AE", + "university": "Xinjiang University", + "bank": { + "cardExpire": "05/22", + "cardNumber": "5602214306858976", + "cardType": "bankcard", + "currency": "Yuan Renminbi", + "iban": "GB94 MOIU 1274 8449 9733 05" + }, + "company": { + "address": { + "address": "6463 Vrain Street", + "city": "Arvada", + "coordinates": { + "lat": 39.814056, + "lng": -105.046913 + }, + "postalCode": "80003", + "state": "CO" + }, + "department": "Sales", + "name": "Bahringer, Auer and Wehner", + "title": "Financial Analyst" + }, + "ein": "53-7190545", + "ssn": "809-93-2422", + "userAgent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.3 Safari/534.24" + }, + { + "id": 20, + "firstName": "Lenna", + "lastName": "Renner", + "maidenName": "Schumm", + "age": 41, + "gender": "female", + "email": "aeatockj@psu.edu", + "phone": "+1 904 601 7177", + "username": "aeatockj", + "password": "szWAG6hc", + "birthDate": "1980-01-19", + "image": "https://robohash.org/ipsumutofficiis.png", + "bloodGroup": "O−", + "height": 175, + "weight": 68, + "eyeColor": "Green", + "hair": { + "color": "Black", + "type": "Strands" + }, + "domain": "sourceforge.net", + "ip": "59.43.194.22", + "address": { + "address": "22572 Toreador Drive", + "city": "Salinas", + "coordinates": { + "lat": 36.602449, + "lng": -121.699071 + }, + "postalCode": "93908", + "state": "CA" + }, + "macAddress": "ED:64:AE:91:49:C9", + "university": "Moraine Valley Community College", + "bank": { + "cardExpire": "07/22", + "cardNumber": "3565173055875732", + "cardType": "jcb", + "currency": "Dollar", + "iban": "GT39 KL9Z CZYV XF26 UPYW SFPT H74U" + }, + "company": { + "address": { + "address": "491 Arabian Way", + "city": "Grand Junction", + "coordinates": { + "lat": 39.07548999999999, + "lng": -108.474785 + }, + "postalCode": "81504", + "state": "CO" + }, + "department": "Support", + "name": "Hoppe Group", + "title": "Geologist III" + }, + "ein": "88-6715551", + "ssn": "389-03-0381", + "userAgent": "Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Ubuntu/10.10 Chromium/12.0.702.0 Chrome/12.0.702.0 Safari/534.24" + }, + { + "id": 21, + "firstName": "Doyle", + "lastName": "Ernser", + "maidenName": "Feeney", + "age": 23, + "gender": "male", + "email": "ckensleyk@pen.io", + "phone": "+86 634 419 6839", + "username": "ckensleyk", + "password": "tq7kPXyf", + "birthDate": "1983-01-22", + "image": "https://robohash.org/providenttemporadelectus.png", + "bloodGroup": "A−", + "height": 173, + "weight": 69.9, + "eyeColor": "Brown", + "hair": { + "color": "Black", + "type": "Curly" + }, + "domain": "free.fr", + "ip": "87.213.156.73", + "address": { + "address": "3034 Mica Street", + "city": "Fayetteville", + "coordinates": { + "lat": 36.0807929, + "lng": -94.2066449 + }, + "postalCode": "72704", + "state": "AR" + }, + "macAddress": "E2:5A:A5:85:9B:6D", + "university": "Nanjing University of Traditional Chinese Medicine", + "bank": { + "cardExpire": "06/24", + "cardNumber": "30464640811198", + "cardType": "diners-club-carte-blanche", + "currency": "Yuan Renminbi", + "iban": "BE41 7150 0766 2980" + }, + "company": { + "address": { + "address": "5906 Milton Avenue", + "city": "Deale", + "coordinates": { + "lat": 38.784451, + "lng": -76.54125499999999 + }, + "postalCode": "20751", + "state": "MD" + }, + "department": "Product Management", + "name": "Brekke Group", + "title": "Programmer Analyst I" + }, + "ein": "23-4116115", + "ssn": "562-46-9709", + "userAgent": "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3" + }, + { + "id": 22, + "firstName": "Tressa", + "lastName": "Weber", + "maidenName": "Williamson", + "age": 41, + "gender": "female", + "email": "froachel@howstuffworks.com", + "phone": "+34 517 104 6248", + "username": "froachel", + "password": "rfVSKImC", + "birthDate": "1987-11-11", + "image": "https://robohash.org/temporarecusandaeest.png", + "bloodGroup": "B−", + "height": 164, + "weight": 87.1, + "eyeColor": "Green", + "hair": { + "color": "Black", + "type": "Strands" + }, + "domain": "indiatimes.com", + "ip": "71.57.235.192", + "address": { + "address": "3729 East Mission Boulevard", + "city": "Fayetteville", + "coordinates": { + "lat": 36.0919353, + "lng": -94.10654219999999 + }, + "postalCode": "72703", + "state": "AR" + }, + "macAddress": "A4:8B:56:BC:ED:98", + "university": "Universitat Rámon Llull", + "bank": { + "cardExpire": "12/21", + "cardNumber": "342220243660686", + "cardType": "americanexpress", + "currency": "Euro", + "iban": "CY09 2675 2653 QNEJ JNSA 0E2V ONMM" + }, + "company": { + "address": { + "address": "8800 Cordell Circle", + "city": "Anchorage", + "coordinates": { + "lat": 61.1409305, + "lng": -149.9437822 + }, + "postalCode": "99502", + "state": "AK" + }, + "department": "Research and Development", + "name": "Durgan Group", + "title": "VP Quality Control" + }, + "ein": "78-2846180", + "ssn": "155-87-0243", + "userAgent": "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_4_11; de-de) AppleWebKit/533.16 (KHTML, like Gecko) Version/4.1 Safari/533.16" + }, + { + "id": 23, + "firstName": "Felicity", + "lastName": "O'Reilly", + "maidenName": "Rosenbaum", + "age": 46, + "gender": "female", + "email": "beykelhofm@wikispaces.com", + "phone": "+63 919 564 1690", + "username": "beykelhofm", + "password": "zQwaHTHbuZyr", + "birthDate": "1967-10-05", + "image": "https://robohash.org/odioquivero.png", + "bloodGroup": "O−", + "height": 151, + "weight": 96.7, + "eyeColor": "Brown", + "hair": { + "color": "Brown", + "type": "Curly" + }, + "domain": "tamu.edu", + "ip": "141.14.53.176", + "address": { + "address": "5114 Greentree Drive", + "city": "Nashville", + "coordinates": { + "lat": 36.0618539, + "lng": -86.738508 + }, + "postalCode": "37211", + "state": "TN" + }, + "macAddress": "4D:AB:8D:9A:E5:02", + "university": "University of lloilo", + "bank": { + "cardExpire": "06/22", + "cardNumber": "6333837222395642", + "cardType": "switch", + "currency": "Peso", + "iban": "FR40 3929 7903 26S5 QL9A HUSV Z09" + }, + "company": { + "address": { + "address": "1770 Colony Way", + "city": "Fayetteville", + "coordinates": { + "lat": 36.0867, + "lng": -94.229754 + }, + "postalCode": "72704", + "state": "AR" + }, + "department": "Legal", + "name": "Romaguera, Williamson and Kessler", + "title": "Assistant Manager" + }, + "ein": "92-4814248", + "ssn": "441-72-1229", + "userAgent": "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.872.0 Safari/535.2" + }, + { + "id": 24, + "firstName": "Jocelyn", + "lastName": "Schuster", + "maidenName": "Dooley", + "age": 19, + "gender": "male", + "email": "brickeardn@fema.gov", + "phone": "+7 968 462 1292", + "username": "brickeardn", + "password": "bMQnPttV", + "birthDate": "1966-06-02", + "image": "https://robohash.org/odiomolestiaealias.png", + "bloodGroup": "O+", + "height": 166, + "weight": 93.3, + "eyeColor": "Brown", + "hair": { + "color": "Brown", + "type": "Curly" + }, + "domain": "pen.io", + "ip": "116.92.198.102", + "address": { + "address": "3466 Southview Avenue", + "city": "Montgomery", + "coordinates": { + "lat": 32.341227, + "lng": -86.2846859 + }, + "postalCode": "36111", + "state": "AL" + }, + "macAddress": "AF:AA:20:8E:CA:CD", + "university": "Bashkir State Medical University", + "bank": { + "cardExpire": "11/21", + "cardNumber": "5007666357943463", + "cardType": "mastercard", + "currency": "Ruble", + "iban": "NL22 YBPM 0101 6695 08" + }, + "company": { + "address": { + "address": "80 North East Street", + "city": "Holyoke", + "coordinates": { + "lat": 42.2041219, + "lng": -72.5977704 + }, + "postalCode": "01040", + "state": "MA" + }, + "department": "Product Management", + "name": "Wintheiser-Boehm", + "title": "Research Nurse" + }, + "ein": "77-6259466", + "ssn": "291-72-5526", + "userAgent": "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; ja-jp) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27" + }, + { + "id": 25, + "firstName": "Edwina", + "lastName": "Ernser", + "maidenName": "Kiehn", + "age": 21, + "gender": "female", + "email": "dfundello@amazon.co.jp", + "phone": "+86 376 986 8945", + "username": "dfundello", + "password": "k9zgV68UKw8m", + "birthDate": "2000-09-28", + "image": "https://robohash.org/doloremautdolores.png", + "bloodGroup": "O+", + "height": 180, + "weight": 102.1, + "eyeColor": "Blue", + "hair": { + "color": "Brown", + "type": "Wavy" + }, + "domain": "apple.com", + "ip": "48.30.193.203", + "address": { + "address": "1513 Cathy Street", + "city": "Savannah", + "coordinates": { + "lat": 32.067416, + "lng": -81.125331 + }, + "postalCode": "31415", + "state": "GA" + }, + "macAddress": "EC:59:D3:FC:65:92", + "university": "Wuhan University of Technology", + "bank": { + "cardExpire": "10/23", + "cardNumber": "3558628665594956", + "cardType": "jcb", + "currency": "Yuan Renminbi", + "iban": "RS85 6347 5884 2820 5764 23" + }, + "company": { + "address": { + "address": "125 John Street", + "city": "Santa Cruz", + "coordinates": { + "lat": 36.950901, + "lng": -122.046881 + }, + "postalCode": "95060", + "state": "CA" + }, + "department": "Marketing", + "name": "Volkman Group", + "title": "Cost Accountant" + }, + "ein": "14-6307509", + "ssn": "266-43-5297", + "userAgent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3" + }, + { + "id": 26, + "firstName": "Griffin", + "lastName": "Braun", + "maidenName": "Deckow", + "age": 35, + "gender": "male", + "email": "lgronaverp@cornell.edu", + "phone": "+62 511 790 0161", + "username": "lgronaverp", + "password": "4a1dAKDv9KB9", + "birthDate": "1965-09-06", + "image": "https://robohash.org/laboriosammollitiaut.png", + "bloodGroup": "O−", + "height": 146, + "weight": 65.5, + "eyeColor": "Blue", + "hair": { + "color": "Blond", + "type": "Wavy" + }, + "domain": "foxnews.com", + "ip": "93.246.47.59", + "address": { + "address": "600 West 19th Avenue", + "city": "Anchorage", + "coordinates": { + "lat": 61.203115, + "lng": -149.894107 + }, + "postalCode": "99503", + "state": "AK" + }, + "macAddress": "34:06:26:95:37:D6", + "university": "Universitas Bojonegoro", + "bank": { + "cardExpire": "07/24", + "cardNumber": "3587188969123346", + "cardType": "jcb", + "currency": "Rupiah", + "iban": "AD24 9240 6903 OD2X OW1Y WD1K" + }, + "company": { + "address": { + "address": "1508 Massachusetts Avenue Southeast", + "city": "Washington", + "coordinates": { + "lat": 38.887255, + "lng": -76.98318499999999 + }, + "postalCode": "20003", + "state": "DC" + }, + "department": "Engineering", + "name": "Boyle, Boyer and Lang", + "title": "Senior Cost Accountant" + }, + "ein": "38-0997138", + "ssn": "407-02-8915", + "userAgent": "Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25" + }, + { + "id": 27, + "firstName": "Piper", + "lastName": "Schowalter", + "maidenName": "Wuckert", + "age": 47, + "gender": "female", + "email": "fokillq@amazon.co.jp", + "phone": "+60 785 960 7918", + "username": "fokillq", + "password": "xZnWSWnqH", + "birthDate": "1983-06-07", + "image": "https://robohash.org/nequeodiosapiente.png", + "bloodGroup": "A−", + "height": 197, + "weight": 71.5, + "eyeColor": "Brown", + "hair": { + "color": "Black", + "type": "Curly" + }, + "domain": "toplist.cz", + "ip": "100.159.51.104", + "address": { + "address": "1208 Elkader Court North", + "city": "Nashville", + "coordinates": { + "lat": 36.080049, + "lng": -86.60116099999999 + }, + "postalCode": "37013", + "state": "TN" + }, + "macAddress": "1F:42:5D:8C:66:3D", + "university": "Sultanah Bahiyah Polytechnic", + "bank": { + "cardExpire": "09/22", + "cardNumber": "6762169351744592", + "cardType": "maestro", + "currency": "Ringgit", + "iban": "BH05 STDW HECU HD4S L8U1 C6" + }, + "company": { + "address": { + "address": "600 West 19th Avenue", + "city": "Anchorage", + "coordinates": { + "lat": 61.203115, + "lng": -149.894107 + }, + "postalCode": "99503", + "state": "AK" + }, + "department": "Human Resources", + "name": "O'Hara and Sons", + "title": "Sales Representative" + }, + "ein": "11-3129153", + "ssn": "408-90-5986", + "userAgent": "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36" + }, + { + "id": 28, + "firstName": "Kody", + "lastName": "Terry", + "maidenName": "Larkin", + "age": 28, + "gender": "male", + "email": "xisherwoodr@ask.com", + "phone": "+81 859 545 8951", + "username": "xisherwoodr", + "password": "HLDqN5vCF", + "birthDate": "1979-01-09", + "image": "https://robohash.org/consequunturabnon.png", + "bloodGroup": "B−", + "height": 172, + "weight": 90.2, + "eyeColor": "Blue", + "hair": { + "color": "Brown", + "type": "Wavy" + }, + "domain": "elpais.com", + "ip": "51.102.180.216", + "address": { + "address": "210 Green Road", + "city": "Manchester", + "coordinates": { + "lat": 41.7909099, + "lng": -72.51195129999999 + }, + "postalCode": "06042", + "state": "CT" + }, + "macAddress": "B4:B6:17:3C:41:E5", + "university": "Science University of Tokyo", + "bank": { + "cardExpire": "05/23", + "cardNumber": "201443655632569", + "cardType": "diners-club-enroute", + "currency": "Yen", + "iban": "GT70 4NNE RDSR 0AJV 6AQI 4XH1 RWOC" + }, + "company": { + "address": { + "address": "109 Summit Street", + "city": "Burlington", + "coordinates": { + "lat": 44.4729749, + "lng": -73.2026566 + }, + "postalCode": "05401", + "state": "VT" + }, + "department": "Support", + "name": "Leffler, Beatty and Kilback", + "title": "Recruiting Manager" + }, + "ein": "09-1129306", + "ssn": "389-74-9456", + "userAgent": "Mozilla/6.0 (Macintosh; I; Intel Mac OS X 11_7_9; de-LI; rv:1.9b4) Gecko/2012010317 Firefox/10.0a4" + }, + { + "id": 29, + "firstName": "Macy", + "lastName": "Greenfelder", + "maidenName": "Koepp", + "age": 45, + "gender": "female", + "email": "jissetts@hostgator.com", + "phone": "+81 915 649 2384", + "username": "jissetts", + "password": "ePawWgrnZR8L", + "birthDate": "1976-09-07", + "image": "https://robohash.org/amettemporeea.png", + "bloodGroup": "A−", + "height": 166, + "weight": 93.7, + "eyeColor": "Amber", + "hair": { + "color": "Black", + "type": "Straight" + }, + "domain": "ibm.com", + "ip": "197.37.13.163", + "address": { + "address": "49548 Road 200", + "city": "O'Neals", + "coordinates": { + "lat": 37.153463, + "lng": -119.648192 + }, + "postalCode": "93645", + "state": "CA" + }, + "macAddress": "D7:14:C5:45:69:C1", + "university": "Fuji Women's College", + "bank": { + "cardExpire": "04/24", + "cardNumber": "633413352570887921", + "cardType": "solo", + "currency": "Yen", + "iban": "IS23 8410 4605 1294 9479 5900 11" + }, + "company": { + "address": { + "address": "5403 Illinois Avenue", + "city": "Nashville", + "coordinates": { + "lat": 36.157077, + "lng": -86.853827 + }, + "postalCode": "37209", + "state": "TN" + }, + "department": "Product Management", + "name": "Bruen and Sons", + "title": "Structural Analysis Engineer" + }, + "ein": "31-6688179", + "ssn": "391-33-1550", + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.45 Safari/535.19" + }, + { + "id": 30, + "firstName": "Maurine", + "lastName": "Stracke", + "maidenName": "Abshire", + "age": 31, + "gender": "female", + "email": "kdulyt@umich.edu", + "phone": "+48 143 590 6847", + "username": "kdulyt", + "password": "5t6q4KC7O", + "birthDate": "1964-12-18", + "image": "https://robohash.org/perferendisideveniet.png", + "bloodGroup": "O−", + "height": 170, + "weight": 107.2, + "eyeColor": "Blue", + "hair": { + "color": "Blond", + "type": "Wavy" + }, + "domain": "ow.ly", + "ip": "97.11.116.84", + "address": { + "address": "81 Seaton Place Northwest", + "city": "Washington", + "coordinates": { + "lat": 38.9149499, + "lng": -77.01170259999999 + }, + "postalCode": "20001", + "state": "DC" + }, + "macAddress": "42:87:72:A1:4D:9A", + "university": "Poznan School of Banking", + "bank": { + "cardExpire": "02/24", + "cardNumber": "6331108070510590026", + "cardType": "switch", + "currency": "Zloty", + "iban": "MT70 MKRC 8244 59Z4 9UG1 1HY7 TKM6 1YX" + }, + "company": { + "address": { + "address": "816 West 19th Avenue", + "city": "Anchorage", + "coordinates": { + "lat": 61.203221, + "lng": -149.898655 + }, + "postalCode": "99503", + "state": "AK" + }, + "department": "Support", + "name": "Balistreri-Kshlerin", + "title": "Quality Engineer" + }, + "ein": "51-7727524", + "ssn": "534-76-0952", + "userAgent": "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11" + } + ], + "carts": [ + { + "id": 1, + "products": [ + { + "id": 59, + "title": "Spring and summershoes", + "price": 20, + "quantity": 3, + "total": 60, + "discountPercentage": 8.71, + "discountedPrice": 55 + }, + { + "id": 88, + "title": "TC Reusable Silicone Magic Washing Gloves", + "price": 29, + "quantity": 2, + "total": 58, + "discountPercentage": 3.19, + "discountedPrice": 56 + }, + { + "id": 18, + "title": "Oil Free Moisturizer 100ml", + "price": 40, + "quantity": 2, + "total": 80, + "discountPercentage": 13.1, + "discountedPrice": 70 + }, + { + "id": 95, + "title": "Wholesale cargo lashing Belt", + "price": 930, + "quantity": 1, + "total": 930, + "discountPercentage": 17.67, + "discountedPrice": 766 + }, + { + "id": 39, + "title": "Women Sweaters Wool", + "price": 600, + "quantity": 2, + "total": 1200, + "discountPercentage": 17.2, + "discountedPrice": 994 + } + ], + "total": 2328, + "discountedTotal": 1941, + "userId": 97, + "totalProducts": 5, + "totalQuantity": 10 + }, + { + "id": 2, + "products": [ + { + "id": 96, + "title": "lighting ceiling kitchen", + "price": 30, + "quantity": 2, + "total": 60, + "discountPercentage": 14.89, + "discountedPrice": 51 + }, + { + "id": 91, + "title": "Black Motorbike", + "price": 569, + "quantity": 3, + "total": 1707, + "discountPercentage": 13.63, + "discountedPrice": 1474 + }, + { + "id": 9, + "title": "Infinix INBOOK", + "price": 1099, + "quantity": 1, + "total": 1099, + "discountPercentage": 11.83, + "discountedPrice": 969 + }, + { + "id": 16, + "title": "Hyaluronic Acid Serum", + "price": 19, + "quantity": 1, + "total": 19, + "discountPercentage": 13.31, + "discountedPrice": 16 + }, + { + "id": 54, + "title": "Pubg Printed Graphic T-Shirt", + "price": 46, + "quantity": 3, + "total": 138, + "discountPercentage": 16.44, + "discountedPrice": 115 + } + ], + "total": 3023, + "discountedTotal": 2625, + "userId": 30, + "totalProducts": 5, + "totalQuantity": 10 + }, + { + "id": 3, + "products": [ + { + "id": 37, + "title": "ank Tops for Womens/Girls", + "price": 50, + "quantity": 2, + "total": 100, + "discountPercentage": 12.05, + "discountedPrice": 88 + }, + { + "id": 80, + "title": "Chain Pin Tassel Earrings", + "price": 45, + "quantity": 3, + "total": 135, + "discountPercentage": 17.75, + "discountedPrice": 111 + }, + { + "id": 68, + "title": "Stylish Luxury Digital Watch", + "price": 57, + "quantity": 3, + "total": 171, + "discountPercentage": 9.03, + "discountedPrice": 156 + }, + { + "id": 81, + "title": "Round Silver Frame Sun Glasses", + "price": 19, + "quantity": 1, + "total": 19, + "discountPercentage": 10.1, + "discountedPrice": 17 + }, + { + "id": 90, + "title": "Cycle Bike Glow", + "price": 35, + "quantity": 1, + "total": 35, + "discountPercentage": 11.08, + "discountedPrice": 31 + } + ], + "total": 460, + "discountedTotal": 403, + "userId": 63, + "totalProducts": 5, + "totalQuantity": 10 + }, + { + "id": 4, + "products": [ + { + "id": 36, + "title": "Sleeve Shirt Womens", + "price": 90, + "quantity": 1, + "total": 90, + "discountPercentage": 10.89, + "discountedPrice": 80 + }, + { + "id": 54, + "title": "Pubg Printed Graphic T-Shirt", + "price": 46, + "quantity": 1, + "total": 46, + "discountPercentage": 16.44, + "discountedPrice": 38 + }, + { + "id": 11, + "title": "perfume Oil", + "price": 13, + "quantity": 3, + "total": 39, + "discountPercentage": 8.4, + "discountedPrice": 36 + }, + { + "id": 47, + "title": "Sneaker shoes", + "price": 120, + "quantity": 2, + "total": 240, + "discountPercentage": 10.37, + "discountedPrice": 215 + }, + { + "id": 64, + "title": "Leather Strap Skeleton Watch", + "price": 46, + "quantity": 3, + "total": 138, + "discountPercentage": 10.2, + "discountedPrice": 124 + } + ], + "total": 553, + "discountedTotal": 493, + "userId": 83, + "totalProducts": 5, + "totalQuantity": 10 + }, + { + "id": 5, + "products": [ + { + "id": 23, + "title": "Orange Essence Food Flavou", + "price": 14, + "quantity": 3, + "total": 42, + "discountPercentage": 8.04, + "discountedPrice": 39 + }, + { + "id": 91, + "title": "Black Motorbike", + "price": 569, + "quantity": 1, + "total": 569, + "discountPercentage": 13.63, + "discountedPrice": 491 + }, + { + "id": 45, + "title": "Malai Maxi Dress", + "price": 50, + "quantity": 2, + "total": 100, + "discountPercentage": 5.07, + "discountedPrice": 95 + }, + { + "id": 84, + "title": "Square Sunglasses", + "price": 28, + "quantity": 1, + "total": 28, + "discountPercentage": 13.89, + "discountedPrice": 24 + }, + { + "id": 70, + "title": "Stainless Steel Women", + "price": 35, + "quantity": 3, + "total": 105, + "discountPercentage": 8.98, + "discountedPrice": 96 + } + ], + "total": 844, + "discountedTotal": 745, + "userId": 58, + "totalProducts": 5, + "totalQuantity": 10 + }, + { + "id": 6, + "products": [ + { + "id": 53, + "title": "printed high quality T shirts", + "price": 35, + "quantity": 3, + "total": 105, + "discountPercentage": 7.54, + "discountedPrice": 97 + }, + { + "id": 61, + "title": "Leather Straps Wristwatch", + "price": 120, + "quantity": 2, + "total": 240, + "discountPercentage": 7.14, + "discountedPrice": 223 + }, + { + "id": 92, + "title": "HOT SALE IN EUROPE electric racing motorcycle", + "price": 920, + "quantity": 1, + "total": 920, + "discountPercentage": 14.4, + "discountedPrice": 788 + }, + { + "id": 11, + "title": "perfume Oil", + "price": 13, + "quantity": 3, + "total": 39, + "discountPercentage": 8.4, + "discountedPrice": 36 + }, + { + "id": 37, + "title": "ank Tops for Womens/Girls", + "price": 50, + "quantity": 3, + "total": 150, + "discountPercentage": 12.05, + "discountedPrice": 132 + } + ], + "total": 1454, + "discountedTotal": 1276, + "userId": 26, + "totalProducts": 5, + "totalQuantity": 12 + }, + { + "id": 7, + "products": [ + { + "id": 61, + "title": "Leather Straps Wristwatch", + "price": 120, + "quantity": 1, + "total": 120, + "discountPercentage": 7.14, + "discountedPrice": 111 + }, + { + "id": 80, + "title": "Chain Pin Tassel Earrings", + "price": 45, + "quantity": 2, + "total": 90, + "discountPercentage": 17.75, + "discountedPrice": 74 + }, + { + "id": 99, + "title": "American Vintage Wood Pendant Light", + "price": 46, + "quantity": 3, + "total": 138, + "discountPercentage": 8.84, + "discountedPrice": 126 + }, + { + "id": 14, + "title": "Non-Alcoholic Concentrated Perfume Oil", + "price": 120, + "quantity": 1, + "total": 120, + "discountPercentage": 15.6, + "discountedPrice": 101 + }, + { + "id": 48, + "title": "Women Strip Heel", + "price": 40, + "quantity": 3, + "total": 120, + "discountPercentage": 10.83, + "discountedPrice": 107 + } + ], + "total": 588, + "discountedTotal": 519, + "userId": 56, + "totalProducts": 5, + "totalQuantity": 10 + }, + { + "id": 8, + "products": [ + { + "id": 45, + "title": "Malai Maxi Dress", + "price": 50, + "quantity": 1, + "total": 50, + "discountPercentage": 5.07, + "discountedPrice": 47 + }, + { + "id": 83, + "title": "Wiley X Night Vision Yellow Glasses", + "price": 30, + "quantity": 3, + "total": 90, + "discountPercentage": 6.33, + "discountedPrice": 84 + }, + { + "id": 96, + "title": "lighting ceiling kitchen", + "price": 30, + "quantity": 1, + "total": 30, + "discountPercentage": 14.89, + "discountedPrice": 26 + }, + { + "id": 21, + "title": "- Daal Masoor 500 grams", + "price": 20, + "quantity": 3, + "total": 60, + "discountPercentage": 4.81, + "discountedPrice": 57 + }, + { + "id": 2, + "title": "iphone X", + "price": 899, + "quantity": 1, + "total": 899, + "discountPercentage": 17.94, + "discountedPrice": 738 + } + ], + "total": 1129, + "discountedTotal": 952, + "userId": 1, + "totalProducts": 5, + "totalQuantity": 9 + }, + { + "id": 9, + "products": [ + { + "id": 74, + "title": "Leather Hand Bag", + "price": 57, + "quantity": 3, + "total": 171, + "discountPercentage": 11.19, + "discountedPrice": 152 + }, + { + "id": 10, + "title": "HP Pavilion 15-DK1056WM", + "price": 1099, + "quantity": 3, + "total": 3297, + "discountPercentage": 6.18, + "discountedPrice": 3093 + }, + { + "id": 19, + "title": "Skin Beauty Serum.", + "price": 46, + "quantity": 2, + "total": 92, + "discountPercentage": 10.68, + "discountedPrice": 82 + }, + { + "id": 53, + "title": "printed high quality T shirts", + "price": 35, + "quantity": 1, + "total": 35, + "discountPercentage": 7.54, + "discountedPrice": 32 + }, + { + "id": 13, + "title": "Fog Scent Xpressio Perfume", + "price": 13, + "quantity": 1, + "total": 13, + "discountPercentage": 8.14, + "discountedPrice": 12 + } + ], + "total": 3608, + "discountedTotal": 3371, + "userId": 91, + "totalProducts": 5, + "totalQuantity": 10 + }, + { + "id": 10, + "products": [ + { + "id": 75, + "title": "Seven Pocket Women Bag", + "price": 68, + "quantity": 1, + "total": 68, + "discountPercentage": 14.87, + "discountedPrice": 58 + }, + { + "id": 39, + "title": "Women Sweaters Wool", + "price": 600, + "quantity": 1, + "total": 600, + "discountPercentage": 17.2, + "discountedPrice": 497 + }, + { + "id": 3, + "title": "Samsung Universe 9", + "price": 1249, + "quantity": 3, + "total": 3747, + "discountPercentage": 15.46, + "discountedPrice": 3168 + }, + { + "id": 7, + "title": "Samsung Galaxy Book", + "price": 1499, + "quantity": 1, + "total": 1499, + "discountPercentage": 4.15, + "discountedPrice": 1437 + }, + { + "id": 93, + "title": "Automatic Motor Gas Motorcycles", + "price": 1050, + "quantity": 3, + "total": 3150, + "discountPercentage": 3.34, + "discountedPrice": 3045 + } + ], + "total": 9064, + "discountedTotal": 8205, + "userId": 13, + "totalProducts": 5, + "totalQuantity": 9 + }, + { + "id": 11, + "products": [ + { + "id": 71, + "title": "Women Shoulder Bags", + "price": 46, + "quantity": 3, + "total": 138, + "discountPercentage": 14.65, + "discountedPrice": 118 + }, + { + "id": 25, + "title": "Gulab Powder 50 Gram", + "price": 70, + "quantity": 2, + "total": 140, + "discountPercentage": 13.58, + "discountedPrice": 121 + }, + { + "id": 65, + "title": "Stainless Steel Wrist Watch", + "price": 47, + "quantity": 3, + "total": 141, + "discountPercentage": 17.79, + "discountedPrice": 116 + }, + { + "id": 58, + "title": "formal offices shoes", + "price": 57, + "quantity": 1, + "total": 57, + "discountPercentage": 12, + "discountedPrice": 50 + }, + { + "id": 53, + "title": "printed high quality T shirts", + "price": 35, + "quantity": 3, + "total": 105, + "discountPercentage": 7.54, + "discountedPrice": 97 + } + ], + "total": 581, + "discountedTotal": 502, + "userId": 66, + "totalProducts": 5, + "totalQuantity": 12 + }, + { + "id": 12, + "products": [ + { + "id": 32, + "title": "Sofa for Coffe Cafe", + "price": 50, + "quantity": 2, + "total": 100, + "discountPercentage": 15.59, + "discountedPrice": 84 + }, + { + "id": 41, + "title": "NIGHT SUIT", + "price": 55, + "quantity": 3, + "total": 165, + "discountPercentage": 15.05, + "discountedPrice": 140 + }, + { + "id": 69, + "title": "Golden Watch Pearls Bracelet Watch", + "price": 47, + "quantity": 1, + "total": 47, + "discountPercentage": 17.55, + "discountedPrice": 39 + }, + { + "id": 98, + "title": "3 lights lndenpant kitchen islang", + "price": 34, + "quantity": 3, + "total": 102, + "discountPercentage": 5.92, + "discountedPrice": 96 + }, + { + "id": 67, + "title": "Fashion Magnetic Wrist Watch", + "price": 60, + "quantity": 2, + "total": 120, + "discountPercentage": 16.69, + "discountedPrice": 100 + } + ], + "total": 534, + "discountedTotal": 459, + "userId": 52, + "totalProducts": 5, + "totalQuantity": 11 + }, + { + "id": 13, + "products": [ + { + "id": 81, + "title": "Round Silver Frame Sun Glasses", + "price": 19, + "quantity": 1, + "total": 19, + "discountPercentage": 10.1, + "discountedPrice": 17 + }, + { + "id": 42, + "title": "Stiched Kurta plus trouser", + "price": 80, + "quantity": 2, + "total": 160, + "discountPercentage": 15.37, + "discountedPrice": 135 + }, + { + "id": 29, + "title": "Handcraft Chinese style", + "price": 60, + "quantity": 3, + "total": 180, + "discountPercentage": 15.34, + "discountedPrice": 152 + }, + { + "id": 64, + "title": "Leather Strap Skeleton Watch", + "price": 46, + "quantity": 2, + "total": 92, + "discountPercentage": 10.2, + "discountedPrice": 83 + }, + { + "id": 54, + "title": "Pubg Printed Graphic T-Shirt", + "price": 46, + "quantity": 1, + "total": 46, + "discountPercentage": 16.44, + "discountedPrice": 38 + } + ], + "total": 497, + "discountedTotal": 425, + "userId": 79, + "totalProducts": 5, + "totalQuantity": 9 + }, + { + "id": 14, + "products": [ + { + "id": 64, + "title": "Leather Strap Skeleton Watch", + "price": 46, + "quantity": 2, + "total": 92, + "discountPercentage": 10.2, + "discountedPrice": 83 + }, + { + "id": 76, + "title": "Silver Ring Set Women", + "price": 70, + "quantity": 2, + "total": 140, + "discountPercentage": 13.57, + "discountedPrice": 121 + }, + { + "id": 30, + "title": "Key Holder", + "price": 30, + "quantity": 2, + "total": 60, + "discountPercentage": 2.92, + "discountedPrice": 58 + }, + { + "id": 88, + "title": "TC Reusable Silicone Magic Washing Gloves", + "price": 29, + "quantity": 1, + "total": 29, + "discountPercentage": 3.19, + "discountedPrice": 28 + }, + { + "id": 94, + "title": "new arrivals Fashion motocross goggles", + "price": 900, + "quantity": 2, + "total": 1800, + "discountPercentage": 3.85, + "discountedPrice": 1731 + } + ], + "total": 2121, + "discountedTotal": 2021, + "userId": 76, + "totalProducts": 5, + "totalQuantity": 9 + }, + { + "id": 15, + "products": [ + { + "id": 4, + "title": "OPPOF19", + "price": 280, + "quantity": 1, + "total": 280, + "discountPercentage": 17.91, + "discountedPrice": 230 + }, + { + "id": 100, + "title": "Crystal chandelier maria theresa for 12 light", + "price": 47, + "quantity": 3, + "total": 141, + "discountPercentage": 16, + "discountedPrice": 118 + }, + { + "id": 1, + "title": "iPhone 9", + "price": 549, + "quantity": 2, + "total": 1098, + "discountPercentage": 12.96, + "discountedPrice": 956 + }, + { + "id": 48, + "title": "Women Strip Heel", + "price": 40, + "quantity": 3, + "total": 120, + "discountPercentage": 10.83, + "discountedPrice": 107 + }, + { + "id": 94, + "title": "new arrivals Fashion motocross goggles", + "price": 900, + "quantity": 3, + "total": 2700, + "discountPercentage": 3.85, + "discountedPrice": 2596 + } + ], + "total": 4339, + "discountedTotal": 4007, + "userId": 47, + "totalProducts": 5, + "totalQuantity": 12 + }, + { + "id": 16, + "products": [ + { + "id": 3, + "title": "Samsung Universe 9", + "price": 1249, + "quantity": 3, + "total": 3747, + "discountPercentage": 15.46, + "discountedPrice": 3168 + }, + { + "id": 50, + "title": "Women Shoes", + "price": 36, + "quantity": 3, + "total": 108, + "discountPercentage": 16.87, + "discountedPrice": 90 + }, + { + "id": 67, + "title": "Fashion Magnetic Wrist Watch", + "price": 60, + "quantity": 2, + "total": 120, + "discountPercentage": 16.69, + "discountedPrice": 100 + }, + { + "id": 86, + "title": "Bluetooth Aux", + "price": 25, + "quantity": 1, + "total": 25, + "discountPercentage": 10.56, + "discountedPrice": 22 + }, + { + "id": 12, + "title": "Brown Perfume", + "price": 40, + "quantity": 1, + "total": 40, + "discountPercentage": 15.66, + "discountedPrice": 34 + } + ], + "total": 4040, + "discountedTotal": 3414, + "userId": 15, + "totalProducts": 5, + "totalQuantity": 10 + }, + { + "id": 17, + "products": [ + { + "id": 12, + "title": "Brown Perfume", + "price": 40, + "quantity": 3, + "total": 120, + "discountPercentage": 15.66, + "discountedPrice": 101 + }, + { + "id": 25, + "title": "Gulab Powder 50 Gram", + "price": 70, + "quantity": 1, + "total": 70, + "discountPercentage": 13.58, + "discountedPrice": 60 + }, + { + "id": 24, + "title": "cereals muesli fruit nuts", + "price": 46, + "quantity": 2, + "total": 92, + "discountPercentage": 16.8, + "discountedPrice": 77 + }, + { + "id": 90, + "title": "Cycle Bike Glow", + "price": 35, + "quantity": 1, + "total": 35, + "discountPercentage": 11.08, + "discountedPrice": 31 + }, + { + "id": 97, + "title": "Metal Ceramic Flower", + "price": 35, + "quantity": 1, + "total": 35, + "discountPercentage": 10.94, + "discountedPrice": 31 + } + ], + "total": 352, + "discountedTotal": 300, + "userId": 56, + "totalProducts": 5, + "totalQuantity": 8 + }, + { + "id": 18, + "products": [ + { + "id": 23, + "title": "Orange Essence Food Flavou", + "price": 14, + "quantity": 2, + "total": 28, + "discountPercentage": 8.04, + "discountedPrice": 26 + }, + { + "id": 25, + "title": "Gulab Powder 50 Gram", + "price": 70, + "quantity": 2, + "total": 140, + "discountPercentage": 13.58, + "discountedPrice": 121 + }, + { + "id": 9, + "title": "Infinix INBOOK", + "price": 1099, + "quantity": 2, + "total": 2198, + "discountPercentage": 11.83, + "discountedPrice": 1938 + }, + { + "id": 78, + "title": "Rhinestone Korean Style Open Rings", + "price": 30, + "quantity": 2, + "total": 60, + "discountPercentage": 8.02, + "discountedPrice": 55 + }, + { + "id": 63, + "title": "Royal Blue Premium Watch", + "price": 50, + "quantity": 1, + "total": 50, + "discountPercentage": 2.56, + "discountedPrice": 49 + } + ], + "total": 2476, + "discountedTotal": 2189, + "userId": 42, + "totalProducts": 5, + "totalQuantity": 9 + }, + { + "id": 19, + "products": [ + { + "id": 43, + "title": "frock gold printed", + "price": 600, + "quantity": 3, + "total": 1800, + "discountPercentage": 15.55, + "discountedPrice": 1520 + }, + { + "id": 77, + "title": "Rose Ring", + "price": 100, + "quantity": 3, + "total": 300, + "discountPercentage": 3.22, + "discountedPrice": 290 + }, + { + "id": 50, + "title": "Women Shoes", + "price": 36, + "quantity": 3, + "total": 108, + "discountPercentage": 16.87, + "discountedPrice": 90 + }, + { + "id": 31, + "title": "Mornadi Velvet Bed", + "price": 40, + "quantity": 2, + "total": 80, + "discountPercentage": 17, + "discountedPrice": 66 + }, + { + "id": 75, + "title": "Seven Pocket Women Bag", + "price": 68, + "quantity": 3, + "total": 204, + "discountPercentage": 14.87, + "discountedPrice": 174 + } + ], + "total": 2492, + "discountedTotal": 2140, + "userId": 5, + "totalProducts": 5, + "totalQuantity": 14 + }, + { + "id": 20, + "products": [ + { + "id": 66, + "title": "Steel Analog Couple Watches", + "price": 35, + "quantity": 3, + "total": 105, + "discountPercentage": 3.23, + "discountedPrice": 102 + }, + { + "id": 59, + "title": "Spring and summershoes", + "price": 20, + "quantity": 1, + "total": 20, + "discountPercentage": 8.71, + "discountedPrice": 18 + }, + { + "id": 29, + "title": "Handcraft Chinese style", + "price": 60, + "quantity": 1, + "total": 60, + "discountPercentage": 15.34, + "discountedPrice": 51 + }, + { + "id": 32, + "title": "Sofa for Coffe Cafe", + "price": 50, + "quantity": 1, + "total": 50, + "discountPercentage": 15.59, + "discountedPrice": 42 + }, + { + "id": 46, + "title": "women's shoes", + "price": 40, + "quantity": 2, + "total": 80, + "discountPercentage": 16.96, + "discountedPrice": 66 + } + ], + "total": 315, + "discountedTotal": 279, + "userId": 75, + "totalProducts": 5, + "totalQuantity": 8 + } + ], + "total": 100, + "skip": 0, + "limit": 30 +} \ No newline at end of file diff --git a/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/test/resources/usercarts/mapping.ttl b/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/test/resources/usercarts/mapping.ttl new file mode 100644 index 000000000..cc186c486 --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/src/test/resources/usercarts/mapping.ttl @@ -0,0 +1,131 @@ +@prefix rr: . +@prefix rml: . +@prefix rdf: . +@prefix rdfs: . +@prefix ql: . +@prefix map: . +@prefix ma: . +@prefix foaf: . +@prefix schema: . +@prefix ex: . + +map:gm_000 rdf:type rr:GraphMap ; + rr:template "http://example.com/cart/{id}" . + +map:jc_000 rr:child "userId" ; + rr:parent "id" . + +map:map_cart_000 rml:logicalSource map:source_001 ; + rdf:type rr:TriplesMap ; + rdfs:label "cart" ; + rr:predicateObjectMap map:pom_000, map:pom_001, map:pom_002 ; + rr:subjectMap map:s_000 . + +map:map_people_000 rml:logicalSource map:source_002 ; + rdf:type rr:TriplesMap ; + rdfs:label "people" ; + rr:predicateObjectMap map:pom_003, map:pom_004, map:pom_005, map:pom_006 ; + rr:subjectMap map:s_001 . + +map:om_000 rdf:type rr:ObjectMap ; + rr:constant "http://xmlns.com/foaf/0.1/Cart" ; + rr:termType rr:IRI . + +map:om_001 rml:reference "totalQuantity" ; + rdf:type rr:ObjectMap ; + rr:termType rr:Literal . + +map:om_002 rdf:type rr:ObjectMap ; + rr:joinCondition map:jc_000 ; + rr:parentTriplesMap map:map_people_000 . + +map:om_003 rdf:type rr:ObjectMap ; + rr:constant "http://xmlns.com/foaf/0.1/Person" ; + rr:termType rr:IRI . + +map:om_004 rml:reference "firstName" ; + rdf:type rr:ObjectMap ; + rr:termType rr:Literal . + +map:om_005 rml:reference "lastName" ; + rdf:type rr:ObjectMap ; + rr:termType rr:Literal . + +map:om_006 rml:reference "age" ; + rdf:type rr:ObjectMap ; + rr:termType rr:Literal . + +map:pm_000 rdf:type rr:PredicateMap ; + rr:constant rdf:type . + +map:pm_001 rdf:type rr:PredicateMap ; + rr:constant ex:totalQuantity . + +map:pm_002 rdf:type rr:PredicateMap ; + rr:constant ex:cartOwner . + +map:pm_003 rdf:type rr:PredicateMap ; + rr:constant rdf:type . + +map:pm_004 rdf:type rr:PredicateMap ; + rr:constant schema:firstName . + +map:pm_005 rdf:type rr:PredicateMap ; + rr:constant schema:lastName . + +map:pm_006 rdf:type rr:PredicateMap ; + rr:constant ex:age . + +map:pom_000 rdf:type rr:PredicateObjectMap ; + rr:objectMap map:om_000 ; + rr:predicateMap map:pm_000 . + +map:pom_001 rdf:type rr:PredicateObjectMap ; + rr:objectMap map:om_001 ; + rr:predicateMap map:pm_001 . + +map:pom_002 rdf:type rr:PredicateObjectMap ; + rr:objectMap map:om_002 ; + rr:predicateMap map:pm_002 . + +map:pom_003 rdf:type rr:PredicateObjectMap ; + rr:objectMap map:om_003 ; + rr:predicateMap map:pm_003 . + +map:pom_004 rdf:type rr:PredicateObjectMap ; + rr:objectMap map:om_004 ; + rr:predicateMap map:pm_004 . + +map:pom_005 rdf:type rr:PredicateObjectMap ; + rr:objectMap map:om_005 ; + rr:predicateMap map:pm_005 . + +map:pom_006 rdf:type rr:PredicateObjectMap ; + rr:objectMap map:om_006 ; + rr:predicateMap map:pm_006 . + +map:rules_000 map:map_cart_000, map:map_people_000 ; + rdf:type . + +map:s_000 rdf:type rr:SubjectMap ; + rr:graphMap map:gm_000 ; + rr:template "http://example.com/cart/{id}" . + +map:s_001 rdf:type rr:SubjectMap ; + rr:template "http://example.com/person/{username}" . + +map:source_000 rml:referenceFormulation ql:JSONPath ; + rml:source "persons.json" ; + rdf:type rml:LogicalSource ; + rdfs:label "personcart-source" . + +map:source_001 rml:iterator "$.carts[*]" ; + rml:referenceFormulation ql:JSONPath ; + rml:source "persons.json" ; + rdf:type rml:LogicalSource . + +map:source_002 rml:iterator "$.users[*]" ; + rml:referenceFormulation ql:JSONPath ; + rml:source "persons.json" ; + rdf:type rml:LogicalSource . + diff --git a/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml index c00bb30d7..0893c078e 100644 --- a/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml index 8d4001879..49a6ef38d 100644 --- a/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/pom.xml b/ldi-nifi/pom.xml index 5cde7384f..8472159eb 100644 --- a/ldi-nifi/pom.xml +++ b/ldi-nifi/pom.xml @@ -3,7 +3,7 @@ linked-data-interactions be.vlaanderen.informatievlaanderen.ldes - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 pom diff --git a/ldi-orchestrator/ldio-application/pom.xml b/ldi-orchestrator/ldio-application/pom.xml index c0c3aa22a..dbba854b1 100644 --- a/ldi-orchestrator/ldio-application/pom.xml +++ b/ldi-orchestrator/ldio-application/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-orchestrator - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/components/AdapterDebugger.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/components/AdapterDebugger.java index bef666e9f..bfc11ab0c 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/components/AdapterDebugger.java +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/components/AdapterDebugger.java @@ -7,6 +7,9 @@ import java.util.stream.Stream; +/** + * Wrapper around any LdiAdapter for debugging purposes + */ public class AdapterDebugger implements LdiAdapter { private final Logger log; private final LdiAdapter adapter; diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/components/LdiOutputLogger.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/components/LdiOutputLogger.java index e3eedcbeb..1d26bbce1 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/components/LdiOutputLogger.java +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/components/LdiOutputLogger.java @@ -8,6 +8,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Wrapper around any LdiOutput for logging purposes + */ public class LdiOutputLogger implements LdiOutput { private final LdiOutput ldiOutput; private final ObservationRegistry observationRegistry; diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/components/LdioSender.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/components/LdioSender.java index 55b91dcf4..168299404 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/components/LdioSender.java +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/components/LdioSender.java @@ -9,13 +9,17 @@ import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.PipelineConfig.PIPELINE_NAME; +/** + * Important implementation of the LdioTransformer which must be added at the end of every transformer chains of + * a pipeline to make sure the output of that chain can be sent to the provided outputs of that pipeline + */ public class LdioSender extends LdioTransformer { private final List ldiOutputs; private final String pipelineName; private static final String LDIO_DATA_OUT = "ldio_data_out"; public LdioSender(String pipelineName, - List ldiOutputs) { + List ldiOutputs) { this.ldiOutputs = ldiOutputs; this.pipelineName = pipelineName; Metrics.counter(LDIO_DATA_OUT, PIPELINE_NAME, pipelineName).increment(0); diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/components/OutputDebugger.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/components/OutputDebugger.java index 6fa590208..f44d4573e 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/components/OutputDebugger.java +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/components/OutputDebugger.java @@ -7,6 +7,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Wrapper around any LdiOutput for debugging purposes + */ public class OutputDebugger implements LdiOutput { private final Logger log; private final LdiOutput ldiOutput; diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/components/TransformerDebugger.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/components/TransformerDebugger.java index 0ec70ec55..0ec97f874 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/components/TransformerDebugger.java +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/components/TransformerDebugger.java @@ -7,6 +7,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Wrapper around any LdioTransformer for debugging purposes + */ public class TransformerDebugger extends LdioTransformer { private final Logger log; diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/WebConfig.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/WebConfig.java index a8362f029..34a600830 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/WebConfig.java +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/WebConfig.java @@ -11,6 +11,9 @@ import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; +/** + * Configurer that adds the right HttpRequest and HttpResponse body converters to the application + */ @Configuration public class WebConfig implements WebMvcConfigurer { @Override diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/converters/FlattenDeserializer.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/converters/FlattenDeserializer.java index f08cde7f9..b49c6e780 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/converters/FlattenDeserializer.java +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/converters/FlattenDeserializer.java @@ -1,15 +1,15 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.converters; -import java.io.IOException; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; - import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.JsonNode; +import java.io.IOException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + /** * A custom deserializer that flattens a JSON object into a Map of Strings. * The keys in the map are the JSON paths of the values in the input JSON. @@ -19,15 +19,15 @@ public class FlattenDeserializer extends JsonDeserializer> { /** * Deserializes a JSON object into a flattened map. * - * @param p The JSON parser. - * @param ctxt The deserialization context. + * @param parser The JSON parser. + * @param context The deserialization context. * @return A map where the keys are JSON paths and the values are the corresponding values from the input JSON. * @throws IOException If an input or output exception occurred. */ @Override - public Map deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { + public Map deserialize(JsonParser parser, DeserializationContext context) throws IOException { Map result = new HashMap<>(); - JsonNode node = p.getCodec().readTree(p); + JsonNode node = parser.getCodec().readTree(parser); flatten("", node, result); return result; } @@ -36,7 +36,7 @@ public Map deserialize(JsonParser p, DeserializationContext ctxt * Recursively flattens a JSON node. * * @param prefix The current path prefix. - * @param node The current JSON node. + * @param node The current JSON node. * @param result The result map. */ private void flatten(String prefix, JsonNode node, Map result) { diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/management/PipelineController.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/management/PipelineController.java index a65950ed8..fe4eae20d 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/management/PipelineController.java +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/management/PipelineController.java @@ -1,7 +1,7 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.management; import be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioMediaType; -import be.vlaanderen.informatievlaanderen.ldes.ldio.services.PipelineService; +import be.vlaanderen.informatievlaanderen.ldes.ldio.services.PipelineServiceImpl; import be.vlaanderen.informatievlaanderen.ldes.ldio.services.PipelineStatusService; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineConfigTO; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineTO; @@ -16,10 +16,10 @@ @RequestMapping(path = "/admin/api/v1/pipeline") public class PipelineController implements OpenApiPipelineController { - private final PipelineService pipelineService; + private final PipelineServiceImpl pipelineService; private final PipelineStatusService pipelineStatusService; - public PipelineController(PipelineService pipelineService, PipelineStatusService pipelineStatusService) { + public PipelineController(PipelineServiceImpl pipelineService, PipelineStatusService pipelineStatusService) { this.pipelineService = pipelineService; this.pipelineStatusService = pipelineStatusService; } diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineCreatorService.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineCreatorService.java index 59795e14c..d63da6ef4 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineCreatorService.java +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineCreatorService.java @@ -54,7 +54,15 @@ public PipelineCreatorService(OrchestratorConfig orchestratorConfig, Configurabl this.beanFactory = (DefaultListableBeanFactory) configContext.getBeanFactory(); } - public void initialisePipeline(PipelineConfig config) throws InvalidComponentException, InvalidPipelineNameException, LdiAdapterMissingException { + /** + * Initializes a pipeline with the provided pipeline config + * + * @param config Definition of the pipeline + * @throws InvalidComponentException when no configurator could be found for the defined component name + * @throws InvalidPipelineNameException when the pipeline name does not match RegEx {@link PipelineConfig#NAME_PATTERN} + * @throws LdiAdapterMissingException when a ldi adapter is expected, but not configured + */ + public void initialisePipeline(PipelineConfig config) { try { String pipeLineName = config.getName(); validateName(pipeLineName); @@ -84,17 +92,21 @@ public void initialisePipeline(PipelineConfig config) throws InvalidComponentExc private static void verifyAdapter(PipelineConfig config, LdioInputConfigurator configurator) { final ComponentDefinition adapter = config.getInput().getAdapter(); - if(configurator.isAdapterRequired() && adapter == null) { + if (configurator.isAdapterRequired() && adapter == null) { throw new LdiAdapterMissingException(config.getName(), config.getInput().getName()); } - if(!configurator.isAdapterRequired() && adapter != null) { + if (!configurator.isAdapterRequired() && adapter != null) { log.warn("Pipeline \"{}\": Input: \"{}\": \"{}\" ignored", config.getName(), config.getInput().getName(), adapter.getName()); } } + /** + * Removes the pipeline from the spring bean registry + * + * @param pipeline name of the pipeline to delete + */ public void removePipeline(String pipeline) { - DefaultListableBeanFactory beanRegistry = (DefaultListableBeanFactory) configContext.getBeanFactory(); - LdioInput ldioInput = (LdioInput) beanRegistry.getBean(pipeline); + LdioInput ldioInput = beanFactory.getBean(pipeline, LdioInput.class); ldioInput.shutdown(); beanFactory.destroyBean(pipeline); } diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineService.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineService.java index 050b0cbe1..a5653f792 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineService.java +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineService.java @@ -1,72 +1,35 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.services; import be.vlaanderen.informatievlaanderen.ldes.ldio.config.PipelineConfig; -import be.vlaanderen.informatievlaanderen.ldes.ldio.exception.PipelineAlreadyExistsException; import be.vlaanderen.informatievlaanderen.ldes.ldio.exception.PipelineException; -import be.vlaanderen.informatievlaanderen.ldes.ldio.repositories.PipelineRepository; -import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineTO; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; import java.io.File; -import java.util.List; -@Service -public class PipelineService { - private static final Logger log = LoggerFactory.getLogger(PipelineService.class); - private final PipelineCreatorService pipelineCreatorService; - private final PipelineStatusService pipelineStatusService; - private final PipelineRepository pipelineRepository; - - public PipelineService(PipelineCreatorService pipelineCreatorService, PipelineStatusService pipelineStatusService, - PipelineRepository pipelineRepository) { - this.pipelineCreatorService = pipelineCreatorService; - this.pipelineStatusService = pipelineStatusService; - this.pipelineRepository = pipelineRepository; - } - - public PipelineConfig addPipeline(PipelineConfig pipeline) throws PipelineException { - if (pipelineRepository.exists(pipeline.getName())) { - throw new PipelineAlreadyExistsException(pipeline.getName()); - } else { - pipelineCreatorService.initialisePipeline(pipeline); - pipelineRepository.activateNewPipeline(pipeline); - log.atInfo().log("CREATION of pipeline '{}' successfully finished", pipeline.getName().replaceAll("[\n\r]", "_")); - return pipeline; - } - } - - public PipelineConfig addPipeline(PipelineConfig pipeline, File persistedFile) throws PipelineException { - if (pipelineRepository.exists(pipeline.getName())) { - throw new PipelineAlreadyExistsException(pipeline.getName()); - } else { - pipelineCreatorService.initialisePipeline(pipeline); - pipelineRepository.activateExistingPipeline(pipeline, persistedFile); - return pipeline; - } - } - - public List getPipelines() { - return pipelineRepository.getActivePipelines() - .stream() - .map(config -> PipelineTO.build(config, pipelineStatusService.getPipelineStatus(config.name()), pipelineStatusService.getPipelineStatusChangeSource(config.name()))) - .toList(); - } - - public boolean requestDeletion(String pipeline) { - if (pipelineRepository.exists(pipeline)) { - pipelineStatusService.stopPipeline(pipeline); - deletePipelineFromServices(pipeline); - log.atInfo().log("DELETION of pipeline '{}' successfully finished", pipeline.replaceAll("[\n\r]", "_")); - return true; - } else { - return false; - } - } - - private void deletePipelineFromServices(String pipeline) { - pipelineRepository.delete(pipeline); - pipelineCreatorService.removePipeline(pipeline); - } +public interface PipelineService { + /** + * Adds a pipeline to the application + * + * @param pipeline configuration of a pipeline to add + * @return the pipeline configuration + * @throws PipelineException when something goes wrong while creating or adding the pipeline + */ + PipelineConfig addPipeline(PipelineConfig pipeline) throws PipelineException; + + /** + * Adds a pipeline to the application and persists the config to a file + * + * @param pipeline configuration of a pipeline to add + * @param persistedFile file whereto the pipeline configuration must be persisted + * @return the pipeline configuration + * @throws PipelineException when something goes wrong while creating or adding the pipeline + */ + PipelineConfig addPipeline(PipelineConfig pipeline, File persistedFile) throws PipelineException; + + /** + * Deletes a pipeline when it exists + * + * @param pipeline name of the pipeline to be deleted + * @return true if deletion succeeded, otherwise false + */ + boolean requestDeletion(String pipeline); } diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineServiceImpl.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineServiceImpl.java new file mode 100644 index 000000000..29ab68229 --- /dev/null +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineServiceImpl.java @@ -0,0 +1,82 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.services; + +import be.vlaanderen.informatievlaanderen.ldes.ldio.config.PipelineConfig; +import be.vlaanderen.informatievlaanderen.ldes.ldio.events.PipelineShutdownEvent; +import be.vlaanderen.informatievlaanderen.ldes.ldio.exception.PipelineAlreadyExistsException; +import be.vlaanderen.informatievlaanderen.ldes.ldio.exception.PipelineException; +import be.vlaanderen.informatievlaanderen.ldes.ldio.repositories.PipelineRepository; +import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineTO; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.util.List; + +@Service +public class PipelineServiceImpl implements PipelineService { + private static final Logger log = LoggerFactory.getLogger(PipelineServiceImpl.class); + private final PipelineCreatorService pipelineCreatorService; + private final PipelineStatusService pipelineStatusService; + private final PipelineRepository pipelineRepository; + + public PipelineServiceImpl(PipelineCreatorService pipelineCreatorService, PipelineStatusService pipelineStatusService, + PipelineRepository pipelineRepository) { + this.pipelineCreatorService = pipelineCreatorService; + this.pipelineStatusService = pipelineStatusService; + this.pipelineRepository = pipelineRepository; + } + + @EventListener + public void handlePipelineShutdown(PipelineShutdownEvent event) { + this.requestDeletion(event.pipelineId()); + } + + @Override + public PipelineConfig addPipeline(PipelineConfig pipeline) throws PipelineException { + if (pipelineRepository.exists(pipeline.getName())) { + throw new PipelineAlreadyExistsException(pipeline.getName()); + } else { + pipelineCreatorService.initialisePipeline(pipeline); + pipelineRepository.activateNewPipeline(pipeline); + log.atInfo().log("CREATION of pipeline '{}' successfully finished", pipeline.getName().replaceAll("[\n\r]", "_")); + return pipeline; + } + } + + @Override + public PipelineConfig addPipeline(PipelineConfig pipeline, File persistedFile) throws PipelineException { + if (pipelineRepository.exists(pipeline.getName())) { + throw new PipelineAlreadyExistsException(pipeline.getName()); + } else { + pipelineCreatorService.initialisePipeline(pipeline); + pipelineRepository.activateExistingPipeline(pipeline, persistedFile); + return pipeline; + } + } + + public List getPipelines() { + return pipelineRepository.getActivePipelines() + .stream() + .map(config -> PipelineTO.build(config, pipelineStatusService.getPipelineStatus(config.name()), pipelineStatusService.getPipelineStatusChangeSource(config.name()))) + .toList(); + } + + @Override + public boolean requestDeletion(String pipeline) { + if (pipelineRepository.exists(pipeline)) { + pipelineStatusService.stopPipeline(pipeline); + deletePipelineFromServices(pipeline); + log.atInfo().log("DELETION of pipeline '{}' successfully finished", pipeline.replaceAll("[\n\r]", "_")); + return true; + } else { + return false; + } + } + + private void deletePipelineFromServices(String pipeline) { + pipelineRepository.delete(pipeline); + pipelineCreatorService.removePipeline(pipeline); + } +} diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusService.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusService.java index 78e079aa7..5b27c91be 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusService.java +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusService.java @@ -1,126 +1,51 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.services; -import be.vlaanderen.informatievlaanderen.ldes.ldio.events.InputCreatedEvent; -import be.vlaanderen.informatievlaanderen.ldes.ldio.events.PipelineDeletedEvent; -import be.vlaanderen.informatievlaanderen.ldes.ldio.events.PipelineStatusEvent; -import be.vlaanderen.informatievlaanderen.ldes.ldio.exception.PipelineDoesNotExistException; -import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.StatusChangeSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.context.ApplicationEventPublisher; -import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Component; -import java.util.HashMap; import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.*; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatusTrigger.*; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.StatusChangeSource.AUTO; - -@Component -public class PipelineStatusService { - private final ApplicationEventPublisher eventPublisher; - private final Logger logger = LoggerFactory.getLogger(PipelineStatusService.class); - private final Map savedPipelines; - - public PipelineStatusService(ApplicationEventPublisher eventPublisher) { - this.eventPublisher = eventPublisher; - this.savedPipelines = new HashMap<>(); - } - - public PipelineStatus getPipelineStatus(String pipelineName) { - return Optional.ofNullable(savedPipelines.get(pipelineName)) - .map(SavedPipeline::getStatus) - .orElseThrow(() -> new PipelineDoesNotExistException(pipelineName)); - } - - public StatusChangeSource getPipelineStatusChangeSource(String pipelineName) { - return Optional.ofNullable(savedPipelines.get(pipelineName)) - .map(SavedPipeline::getLastStatusChangeSource) - .orElseThrow(() -> new PipelineDoesNotExistException(pipelineName)); - } - - @EventListener - public void handlePipelineStatusResponse(PipelineStatusEvent statusEvent) { - SavedPipeline currentSavedPipeline = savedPipelines.get(statusEvent.pipelineId()); - if (currentSavedPipeline == null) { - logger.warn("Non initialized pipeline received status update: {}", statusEvent.pipelineId()); - return; - } - currentSavedPipeline.updateStatus(statusEvent.status(), statusEvent.statusChangeSource()); - } - - @EventListener - public void handlePipelineCreated(InputCreatedEvent event) { - savedPipelines.put(event.pipelineName(), new SavedPipeline(event.ldioInput(), event.ldioInput().getStatus())); - } - - public PipelineStatus resumeHaltedPipeline(String pipelineId) { - var pipelineStatus = getPipelineStatus(pipelineId); - - return switch (pipelineStatus) { - case HALTED -> savedPipelines.get(pipelineId).getLdioInput().updateStatus(RESUME); - case INIT -> INIT; - case RUNNING -> RUNNING; - case STOPPED -> STOPPED; - }; - } - - public PipelineStatus haltRunningPipeline(String pipelineId) { - PipelineStatus pipelineStatus = getPipelineStatus(pipelineId); - - return switch (pipelineStatus) { - case RUNNING -> savedPipelines.get(pipelineId).getLdioInput().updateStatus(HALT); - case INIT -> INIT; - case HALTED -> HALTED; - case STOPPED -> STOPPED; - }; - } - - public PipelineStatus stopPipeline(String pipelineId) { - PipelineStatus newPipelineStatus = savedPipelines.get(pipelineId).getLdioInput().updateStatus(STOP); - this.savedPipelines.remove(pipelineId); - eventPublisher.publishEvent(new PipelineDeletedEvent(pipelineId)); - return newPipelineStatus; - } - - public Map getPipelineStatusOverview() { - return savedPipelines.entrySet() - .stream() - .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getStatus())); - } - - static class SavedPipeline { - private final LdioInput ldioInput; - private PipelineStatus status; - private StatusChangeSource lastStatusChangeSource; - - public SavedPipeline(LdioInput ldioInput, PipelineStatus status) { - this.ldioInput = ldioInput; - this.status = status; - lastStatusChangeSource = AUTO; - } - - public void updateStatus(PipelineStatus status, StatusChangeSource statusChangeSource) { - this.status = status; - this.lastStatusChangeSource = statusChangeSource; - } - - public PipelineStatus getStatus() { - return status; - } - - public LdioInput getLdioInput() { - return ldioInput; - } - - public StatusChangeSource getLastStatusChangeSource() { - return lastStatusChangeSource; - } - } +public interface PipelineStatusService { + /** + * @param pipelineName name of the pipeline of which the status must be fetched + * @return the status of the pipeline + */ + PipelineStatus getPipelineStatus(String pipelineName); + + /** + * @param pipelineName name of the pipeline of which the source of the status change must be fetched + * @return the last status change source of the pipeline + */ + StatusChangeSource getPipelineStatusChangeSource(String pipelineName); + + /** + * Resume the pipeline if halted + * + * @param pipelineId name of the halted pipeline that must be resumed + * @return the updated pipeline status + */ + PipelineStatus resumeHaltedPipeline(String pipelineId); + + /** + * Halt the pipeline if running + * + * @param pipelineId name of the pipeline that must be paused + * @return the updated pipeline status + */ + PipelineStatus haltRunningPipeline(String pipelineId); + + /** + * Stops the pipeline completely + * + * @param pipelineId name of the pipeline that must be stopped + * @return the updated pipeline status + */ + PipelineStatus stopPipeline(String pipelineId); + + /** + * Fetches an overview of the existing pipelines with their status + * + * @return a map with the pipeline name as key and the belonging pipeline status as value + */ + Map getPipelineStatusOverview(); } diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusServiceImpl.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusServiceImpl.java new file mode 100644 index 000000000..ea6aae526 --- /dev/null +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusServiceImpl.java @@ -0,0 +1,132 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.services; + +import be.vlaanderen.informatievlaanderen.ldes.ldio.events.InputCreatedEvent; +import be.vlaanderen.informatievlaanderen.ldes.ldio.events.PipelineDeletedEvent; +import be.vlaanderen.informatievlaanderen.ldes.ldio.events.PipelineStatusEvent; +import be.vlaanderen.informatievlaanderen.ldes.ldio.exception.PipelineDoesNotExistException; +import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; +import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus; +import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.StatusChangeSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.*; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatusTrigger.*; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.StatusChangeSource.AUTO; + +@Component +public class PipelineStatusServiceImpl implements PipelineStatusService { + private final ApplicationEventPublisher eventPublisher; + private final Logger logger = LoggerFactory.getLogger(PipelineStatusServiceImpl.class); + private final Map savedPipelines; + + public PipelineStatusServiceImpl(ApplicationEventPublisher eventPublisher) { + this.eventPublisher = eventPublisher; + this.savedPipelines = new HashMap<>(); + } + + @Override + public PipelineStatus getPipelineStatus(String pipelineName) { + return Optional.ofNullable(savedPipelines.get(pipelineName)) + .map(SavedPipeline::getStatus) + .orElseThrow(() -> new PipelineDoesNotExistException(pipelineName)); + } + + @Override + public StatusChangeSource getPipelineStatusChangeSource(String pipelineName) { + return Optional.ofNullable(savedPipelines.get(pipelineName)) + .map(SavedPipeline::getLastStatusChangeSource) + .orElseThrow(() -> new PipelineDoesNotExistException(pipelineName)); + } + + @EventListener + public void handlePipelineStatusResponse(PipelineStatusEvent statusEvent) { + SavedPipeline currentSavedPipeline = savedPipelines.get(statusEvent.pipelineId()); + if (currentSavedPipeline == null) { + logger.warn("Non initialized pipeline received status update: {}", statusEvent.pipelineId()); + return; + } + currentSavedPipeline.updateStatus(statusEvent.status(), statusEvent.statusChangeSource()); + } + + @EventListener + public void handlePipelineCreated(InputCreatedEvent event) { + savedPipelines.put(event.pipelineName(), new SavedPipeline(event.ldioInput(), event.ldioInput().getStatus())); + } + + @Override + public PipelineStatus resumeHaltedPipeline(String pipelineId) { + var pipelineStatus = getPipelineStatus(pipelineId); + + return switch (pipelineStatus) { + case HALTED -> savedPipelines.get(pipelineId).getLdioInput().updateStatus(RESUME); + case INIT -> INIT; + case RUNNING -> RUNNING; + case STOPPED -> STOPPED; + }; + } + + @Override + public PipelineStatus haltRunningPipeline(String pipelineId) { + PipelineStatus pipelineStatus = getPipelineStatus(pipelineId); + + return switch (pipelineStatus) { + case RUNNING -> savedPipelines.get(pipelineId).getLdioInput().updateStatus(HALT); + case INIT -> INIT; + case HALTED -> HALTED; + case STOPPED -> STOPPED; + }; + } + + @Override + public PipelineStatus stopPipeline(String pipelineId) { + PipelineStatus newPipelineStatus = savedPipelines.get(pipelineId).getLdioInput().updateStatus(STOP); + this.savedPipelines.remove(pipelineId); + eventPublisher.publishEvent(new PipelineDeletedEvent(pipelineId)); + return newPipelineStatus; + } + + @Override + public Map getPipelineStatusOverview() { + return savedPipelines.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getStatus())); + } + + static class SavedPipeline { + private final LdioInput ldioInput; + private PipelineStatus status; + private StatusChangeSource lastStatusChangeSource; + + public SavedPipeline(LdioInput ldioInput, PipelineStatus status) { + this.ldioInput = ldioInput; + this.status = status; + lastStatusChangeSource = AUTO; + } + + public void updateStatus(PipelineStatus status, StatusChangeSource statusChangeSource) { + this.status = status; + this.lastStatusChangeSource = statusChangeSource; + } + + public PipelineStatus getStatus() { + return status; + } + + public LdioInput getLdioInput() { + return ldioInput; + } + + public StatusChangeSource getLastStatusChangeSource() { + return lastStatusChangeSource; + } + } +} diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/initializer/ConfigPipelineInitializer.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/initializer/ConfigPipelineInitializer.java index 5a2bbe4e0..c8c5d45ae 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/initializer/ConfigPipelineInitializer.java +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/initializer/ConfigPipelineInitializer.java @@ -12,6 +12,10 @@ import java.util.List; import java.util.Objects; +/** + * @deprecated since the implementation of dynamic pipelines, pipelines should either be added in the folder self or by the admin api + */ +@Deprecated(forRemoval = true) @Service public class ConfigPipelineInitializer implements PipelineInitializer { private static final String NAME = "LDI Orchestrator Spring Config Initializer"; diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineConfigTO.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineConfigTO.java index eab7b3826..47c235933 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineConfigTO.java +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineConfigTO.java @@ -15,6 +15,12 @@ public PipelineConfigTO(String name, String description, InputComponentDefinitio this.outputs = outputs; } + /** + * Maps a pipeline config to a data transfer object + * + * @param config the pipeline config to be mapped to a TO + * @return the data transfer object + */ public static PipelineConfigTO fromPipelineConfig(PipelineConfig config) { var adapter = config.getInput().getAdapter() == null ? null : new ComponentDefinitionTO(config.getInput().getAdapter().getName(), config.getInput().getAdapter().getConfigMap()); @@ -25,6 +31,11 @@ public static PipelineConfigTO fromPipelineConfig(PipelineConfig config) { return new PipelineConfigTO(config.getName(), config.getDescription(), input, transformers, outputs); } + /** + * Maps the instance of the data transfer object to a pipeline config object + * + * @return the pipeline config + */ public PipelineConfig toPipelineConfig() { var config = new PipelineConfig(); config.setName(name); diff --git a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineTO.java b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineTO.java index cbf52726f..489d8c00b 100644 --- a/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineTO.java +++ b/ldi-orchestrator/ldio-application/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/PipelineTO.java @@ -4,8 +4,23 @@ import java.util.List; import java.util.Optional; -public record PipelineTO(String name, PipelineStatus status, StatusChangeSource updateSource, String description, InputComponentDefinitionTO input, - List transformers, List outputs) { +public record PipelineTO( + String name, + PipelineStatus status, + StatusChangeSource updateSource, + String description, + InputComponentDefinitionTO input, + List transformers, + List outputs +) { + /** + * Build a pipeline data transfer object based on the provided information + * + * @param config transfer object of the pipeline configuration + * @param status the current status of the pipeline + * @param statusChangeSource the source where the pipeline has last been updated + * @return a data transfer object of the pipeline + */ public static PipelineTO build(PipelineConfigTO config, PipelineStatus status, StatusChangeSource statusChangeSource) { var input = new InputComponentDefinitionTO(config.input().getName(), config.input().getAdapter().map(adapter -> new ComponentDefinitionTO(adapter.name(), adapter.config())).orElse(null), diff --git a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineServiceTest.java b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineServiceTest.java index d18f06235..c4f993417 100644 --- a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineServiceTest.java +++ b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineServiceTest.java @@ -1,22 +1,24 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.services; +import be.vlaanderen.informatievlaanderen.ldes.ldio.events.PipelineShutdownEvent; import be.vlaanderen.informatievlaanderen.ldes.ldio.repositories.PipelineFileRepository; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.*; class PipelineServiceTest { private final String pipelineName = "pipeline"; private final PipelineCreatorService pipelineCreatorService = mock(PipelineCreatorService.class); - private final PipelineStatusService pipelineStatusService = mock(PipelineStatusService.class); + private final PipelineStatusService pipelineStatusService = mock(PipelineStatusServiceImpl.class); private final PipelineFileRepository pipelineRepository = mock(PipelineFileRepository.class); - private PipelineService pipelineService; + private PipelineServiceImpl pipelineService; @BeforeEach void setup() { - pipelineService = new PipelineService(pipelineCreatorService, pipelineStatusService, pipelineRepository); + pipelineService = new PipelineServiceImpl(pipelineCreatorService, pipelineStatusService, pipelineRepository); } @Test @@ -28,6 +30,7 @@ void when_StoppingPipeline_Then_MethodsAreCalled() { assertTrue(result); verify(pipelineStatusService).stopPipeline(pipelineName); } + @Test void when_StoppingNonExistingPipeline_Then_NoMethodsAreCalled() { when(pipelineRepository.exists(pipelineName)).thenReturn(false); @@ -38,4 +41,15 @@ void when_StoppingNonExistingPipeline_Then_NoMethodsAreCalled() { verifyNoInteractions(pipelineStatusService); } + @Test + void when_PipelineShutdown_Then_RemovePipeline() { + when(pipelineRepository.exists(pipelineName)).thenReturn(true); + PipelineShutdownEvent pipelineShutdownEvent = new PipelineShutdownEvent(pipelineName); + + pipelineService.handlePipelineShutdown(pipelineShutdownEvent); + + verify(pipelineStatusService).stopPipeline(pipelineName); + verify(pipelineRepository).delete(pipelineName); + verify(pipelineCreatorService).removePipeline(pipelineName); + } } \ No newline at end of file diff --git a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusServiceTest.java b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusServiceTest.java index 03189b5fc..917416591 100644 --- a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusServiceTest.java +++ b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/services/PipelineStatusServiceTest.java @@ -1,6 +1,7 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.services; import be.vlaanderen.informatievlaanderen.ldes.ldio.events.InputCreatedEvent; +import be.vlaanderen.informatievlaanderen.ldes.ldio.events.PipelineDeletedEvent; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatusTrigger; @@ -15,11 +16,11 @@ class PipelineStatusServiceTest { private final String pipelineName = "pipeline"; private final LdioInput input = mock(LdioInput.class); private final ApplicationEventPublisher eventPublisher = mock(ApplicationEventPublisher.class); - private PipelineStatusService pipelineStatusService; + private PipelineStatusServiceImpl pipelineStatusService; @BeforeEach void setup() { - pipelineStatusService = new PipelineStatusService(eventPublisher); + pipelineStatusService = new PipelineStatusServiceImpl(eventPublisher); pipelineStatusService.handlePipelineCreated(new InputCreatedEvent(pipelineName, input)); } @@ -30,6 +31,7 @@ void when_StoppingPipeline_Then_MethodsAreCalled() { assertEquals(PipelineStatus.STOPPED, result); verify(input).updateStatus(PipelineStatusTrigger.STOP); + verify(eventPublisher).publishEvent(new PipelineDeletedEvent(pipelineName)); } } diff --git a/ldi-orchestrator/ldio-common/pom.xml b/ldi-orchestrator/ldio-common/pom.xml index d530058f5..c8c08c06b 100644 --- a/ldi-orchestrator/ldio-common/pom.xml +++ b/ldi-orchestrator/ldio-common/pom.xml @@ -5,7 +5,7 @@ ldi-orchestrator be.vlaanderen.informatievlaanderen.ldes.ldi - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/configurator/LdioAdapterConfigurator.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/configurator/LdioAdapterConfigurator.java index e3030e9dc..c1f904108 100644 --- a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/configurator/LdioAdapterConfigurator.java +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/configurator/LdioAdapterConfigurator.java @@ -1,4 +1,12 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.configurator; +import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; + +/** + * Interface to manage the configuration of the {@link LdiAdapter} + * in LDIO. Implementations will typically be declared as a bean in a "Ldio[AdapterName]AutoConfig" class that will be + * annotated as {@link org.springframework.context.annotation.Configuration} + */ +@FunctionalInterface public interface LdioAdapterConfigurator extends LdioConfigurator { } diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/configurator/LdioConfigurator.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/configurator/LdioConfigurator.java index 1340a9dd0..938939e1c 100644 --- a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/configurator/LdioConfigurator.java +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/configurator/LdioConfigurator.java @@ -3,6 +3,13 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiComponent; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; +/** + * Base interface to configure all LDIO components except for the {@link be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput} + */ +@FunctionalInterface public interface LdioConfigurator { + /** + * Configures an LdiComponent based on the provided ComponentProperties + */ LdiComponent configure(ComponentProperties properties); } diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/configurator/LdioInputConfigurator.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/configurator/LdioInputConfigurator.java index 2aa920005..15471db13 100644 --- a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/configurator/LdioInputConfigurator.java +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/configurator/LdioInputConfigurator.java @@ -6,6 +6,11 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import org.springframework.context.ApplicationEventPublisher; +/** + * Interface to manage the configuration of the {@link LdioInput} + * in LDIO. Implementations will typically be declared as a bean in a "Ldio[OutputName]AutoConfig" class that will be + * annotated as {@link org.springframework.context.annotation.Configuration} + */ public interface LdioInputConfigurator { LdioInput configure(LdiAdapter adapter, ComponentExecutor executor, ApplicationEventPublisher eventPublisher, ComponentProperties properties); diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/configurator/LdioOutputConfigurator.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/configurator/LdioOutputConfigurator.java index f750cebf2..57c1153e2 100644 --- a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/configurator/LdioOutputConfigurator.java +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/configurator/LdioOutputConfigurator.java @@ -1,4 +1,12 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.configurator; +import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiOutput; + +/** + * Interface to manage the configuration of the {@link LdiOutput} + * in LDIO. Implementations will typically be declared as a bean in a "Ldio[OutputName]AutoConfig" class that will be + * annotated as {@link org.springframework.context.annotation.Configuration} + */ +@FunctionalInterface public interface LdioOutputConfigurator extends LdioConfigurator { } diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/configurator/LdioTransformerConfigurator.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/configurator/LdioTransformerConfigurator.java index 591346b53..734d12862 100644 --- a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/configurator/LdioTransformerConfigurator.java +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/configurator/LdioTransformerConfigurator.java @@ -3,6 +3,13 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioTransformer; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; -public interface LdioTransformerConfigurator { +/** + * Interface to manage the configuration of the {@link LdioTransformer} + * in LDIO. Implementations will typically be declared as a bean in a "Ldio[OutputName]AutoConfig" class that will be + * annotated as {@link org.springframework.context.annotation.Configuration} + */ +@FunctionalInterface +public interface LdioTransformerConfigurator extends LdioConfigurator { + @Override LdioTransformer configure(ComponentProperties properties); } diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/events/PipelineShutdownEvent.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/events/PipelineShutdownEvent.java new file mode 100644 index 000000000..67d3882d6 --- /dev/null +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/events/PipelineShutdownEvent.java @@ -0,0 +1,4 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.events; + +public record PipelineShutdownEvent(String pipelineId) { +} diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/exception/InvalidConfigException.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/exception/InvalidConfigException.java index 44f7c3a75..ddc8b2971 100644 --- a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/exception/InvalidConfigException.java +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/exception/InvalidConfigException.java @@ -1,6 +1,6 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.exception; -public class InvalidConfigException extends RuntimeException { +public class InvalidConfigException extends PipelineException { private final String cause; public InvalidConfigException(String cause) { diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/persistence/PersistenceProperties.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/persistence/PersistenceProperties.java index 0f58feea2..ace50c39c 100644 --- a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/persistence/PersistenceProperties.java +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/persistence/PersistenceProperties.java @@ -1,5 +1,8 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.persistence; +/** + * LDIO properties for managing the persistence + */ public class PersistenceProperties { private PersistenceProperties() {} diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioInput.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioInput.java index 1e440a65a..11e6ebe72 100644 --- a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioInput.java +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioInput.java @@ -28,7 +28,7 @@ public abstract class LdioInput implements LdiComponent { private final ComponentExecutor executor; private final LdiAdapter adapter; private final LdioObserver ldioObserver; - private final ApplicationEventPublisher applicationEventPublisher; + protected final ApplicationEventPublisher applicationEventPublisher; private final Logger log = LoggerFactory.getLogger(this.getClass()); private PipelineStatus pipelineStatus; diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioObserver.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioObserver.java index d90252910..1aacc1450 100644 --- a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioObserver.java +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioObserver.java @@ -12,6 +12,9 @@ import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.PipelineConfig.PIPELINE_NAME; +/** + * Observer that is dealing with errors and metrics from ldio actions + */ public class LdioObserver { private static final String LDIO_DATA_IN = "ldio_data_in"; private static final String LDIO_COMPONENT_NAME = "ldio_type"; @@ -20,14 +23,29 @@ public class LdioObserver { private final String pipelineName; private final ObservationRegistry observationRegistry; + /** + * @param componentName name of the component that must be observed + * @param pipelineName name of the pipeline where the component under observation must be observed + * @param observationRegistry the registry where the observations are part of + */ private LdioObserver(String componentName, String pipelineName, ObservationRegistry observationRegistry) { this.componentName = componentName; this.pipelineName = pipelineName; this.observationRegistry = observationRegistry; } + /** + * Wraps and runs a runnable function inside an observation that is handled by an observation registry and the + * exceptions that are thrown by the runnable are handled in a consistent way + * + * @param observable runnable process that must be observed + * @param location string representation from where the runnable comes + * @param additionalErrorLoggingContent, additional logging content when an error occurs + * @see Observation + * @see ObservationRegistry + */ @SafeVarargs - public final void observe(Runnable observable, String location, Supplier... additionalLoggingContent) { + public final void observe(Runnable observable, String location, Supplier... additionalErrorLoggingContent) { final String errorLocation = pipelineName + ":" + location; Observation.createNotStarted(this.componentName, observationRegistry) .observe(() -> { @@ -35,7 +53,7 @@ public final void observe(Runnable observable, String location, Supplier observable.run(); } catch (Exception e) { log.atError().log(ObserveConfiguration.ERROR_TEMPLATE, errorLocation, e.getMessage()); - Arrays.stream(additionalLoggingContent) + Arrays.stream(additionalErrorLoggingContent) .forEach(content -> log.atError().log(ObserveConfiguration.ERROR_TEMPLATE, errorLocation, content.get()) ); @@ -44,10 +62,20 @@ public final void observe(Runnable observable, String location, Supplier }); } + /** + * Increments a metric for the ldio component + */ public void increment() { Metrics.counter(LDIO_DATA_IN, PIPELINE_NAME, pipelineName, LDIO_COMPONENT_NAME, componentName).increment(); } + /** + * Registers a pipeline and the component and initializes the metrics + * + * @param componentName name of the component that must be observed + * @param pipelineName name of the pipeline where the component under observation must be observed + * @param observationRegistry the registry where the observations are part of + */ public static LdioObserver register(String componentName, String pipelineName, ObservationRegistry observationRegistry) { Metrics.counter(LDIO_DATA_IN, PIPELINE_NAME, pipelineName, LDIO_COMPONENT_NAME, componentName).increment(0); return new LdioObserver(componentName, pipelineName, observationRegistry); diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioPipelineEventsListener.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioPipelineEventsListener.java index 954796678..070cd2b43 100644 --- a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioPipelineEventsListener.java +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioPipelineEventsListener.java @@ -10,6 +10,11 @@ import java.util.Map; import java.util.Objects; +/** + * EventListener that listens that pipeline status changes + * + * @param Generic type that can be any LdiComponent + */ public class LdioPipelineEventsListener { private final Map ldioComponents = new HashMap<>(); private final PipelineStatusChangedBehavior startBehavior; diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioTransformer.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioTransformer.java index 10a763fe1..2912e78f1 100644 --- a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioTransformer.java +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioTransformer.java @@ -9,6 +9,11 @@ import java.util.List; +/** + * Ldio Wrapper that will be used to wrap any + * {@link be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiOneToOneTransformer} or + * {@link be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiOneToManyTransformer} + */ public abstract class LdioTransformer implements LdiComponent { private LdioTransformer nextProcessor; private final Logger log = LoggerFactory.getLogger(this.getClass()); diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/PipelineStatusChangedBehavior.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/PipelineStatusChangedBehavior.java index 5809569ed..9f9670d7a 100644 --- a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/PipelineStatusChangedBehavior.java +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/PipelineStatusChangedBehavior.java @@ -2,6 +2,9 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiComponent; +/** + * Behavior interface that can be implemented for each pipeline status change + */ @FunctionalInterface public interface PipelineStatusChangedBehavior { void applyNewStatus(T ldioComponent); diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/StatusChangeSource.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/StatusChangeSource.java index 3b8c86c49..7e4eab38d 100644 --- a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/StatusChangeSource.java +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/StatusChangeSource.java @@ -1,5 +1,15 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects; +/** + * Representation where the pipeline status has changed + */ public enum StatusChangeSource { - AUTO, MANUAL + /** + * When a pipeline status has changed by the application itself, e.g. when an error occurs + */ + AUTO, + /** + * When a pipeline status has changed by the user, e.g. by calling an REST endpoint + */ + MANUAL } diff --git a/ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml index 1398afef2..669d201a8 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml @@ -6,7 +6,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT ldio-amqp diff --git a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml index 1920e4e94..172ddfbf6 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT ldio-archive-file-in diff --git a/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml index 81c04bc36..50bc9d216 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT diff --git a/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/pom.xml index 810826f1e..ac6a13b5f 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT ldio-change-detection-filter diff --git a/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml index 86dafa6a7..547c6e2a4 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml index efa0fe0ff..bf18e09d4 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml @@ -6,7 +6,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT ldio-file-out diff --git a/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml index 0e503e773..0a73625f7 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT ldio-geojson-to-wkt diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml index 09d2e8a9c..421f75b37 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml index 2b72dd65c..55c38bf0a 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT ldio-http-in-poller diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml index 13bcc1742..54a3f3d15 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml @@ -4,7 +4,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml index e5256ba97..c9db4720f 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml @@ -3,7 +3,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml index 17a34f56d..774198530 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml index 213d8425f..c95aa4db5 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml index 46ab4b4d7..8b6c88338 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/collection/LdioLdesClientConnectorApiCollection.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/collection/LdioLdesClientConnectorApiCollection.java index 89a74585e..ce3fb5f47 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/collection/LdioLdesClientConnectorApiCollection.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/collection/LdioLdesClientConnectorApiCollection.java @@ -4,10 +4,32 @@ import java.util.Optional; +/** + * Keeps references to LdioLdesClientConnectorApis that are added by the configured pipelines + */ public interface LdioLdesClientConnectorApiCollection { + /** + * Gets the LdioLdesClientConnectorApi that belongs to the pipeline + * + * @param pipeline name of the pipeline + * @return the connector api wrapped in an optional if present, otherwise an empty optional + */ + Optional get(String pipeline); + /** + * Adds a LdioLdesClientConnectorApi with a pipeline name to the collection + * + * @param pipeline name of the pipeline + * @param ldioLdesClientConnectorApi api to add + */ void add(String pipeline, LdioLdesClientConnectorApi ldioLdesClientConnectorApi); + /** + * Removes the LdioLdesClientConnectorApi when it belongs to a pipeline + * + * @param pipeline the name of the pipeline + * @return the found connector api wrapped in an optional if present, otherwise an empty optional + */ Optional remove(String pipeline); } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml index c105791f4..d68148628 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java index f256616e8..fb0bf0d79 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java @@ -1,11 +1,12 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio; import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; +import be.vlaanderen.informatievlaanderen.ldes.ldio.events.PipelineShutdownEvent; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioObserver; -import ldes.client.treenodesupplier.membersuppliers.MemberSupplier; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatusTrigger; import ldes.client.treenodesupplier.domain.valueobject.EndOfLdesException; +import ldes.client.treenodesupplier.membersuppliers.MemberSupplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationEventPublisher; @@ -24,6 +25,7 @@ public class LdioLdesClient extends LdioInput { private boolean threadRunning = true; private boolean paused = false; private final boolean keepState; + private final String pipelineName; public LdioLdesClient(ComponentExecutor componentExecutor, LdioObserver ldioObserver, @@ -31,6 +33,7 @@ public LdioLdesClient(ComponentExecutor componentExecutor, ApplicationEventPublisher applicationEventPublisher, boolean keepState) { super(componentExecutor, null, ldioObserver, applicationEventPublisher); + this.pipelineName = ldioObserver.getPipelineName(); this.memberSupplier = memberSupplier; this.keepState = keepState; } @@ -59,7 +62,7 @@ private synchronized void run() { processModel(memberSupplier.get().getModel()); } } catch (EndOfLdesException e) { - log.warn(e.getMessage()); + shutdownPipeline(); } catch (Exception e) { log.error("LdesClientRunner FAILURE: {}", e.getMessage()); } @@ -94,4 +97,9 @@ protected synchronized void resume() { protected void pause() { this.paused = true; } + + private void shutdownPipeline() { + log.info("SHUTTING DOWN pipeline {} because end of LDES has been reached", pipelineName); + applicationEventPublisher.publishEvent(new PipelineShutdownEvent(pipelineName)); + } } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientTest.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientTest.java new file mode 100644 index 000000000..52d5dad91 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientTest.java @@ -0,0 +1,80 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; +import be.vlaanderen.informatievlaanderen.ldes.ldio.events.PipelineShutdownEvent; +import be.vlaanderen.informatievlaanderen.ldes.ldio.events.PipelineStatusEvent; +import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioObserver; +import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus; +import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.StatusChangeSource; +import ldes.client.treenodesupplier.domain.valueobject.EndOfLdesException; +import ldes.client.treenodesupplier.membersuppliers.MemberSupplier; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.context.ApplicationEventPublisher; + +import java.time.Duration; + +import static org.awaitility.Awaitility.await; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class LdioLdesClientTest { + + @Mock + private ComponentExecutor componentExecutor; + + @Mock + private LdioObserver observer; + + @Mock + private MemberSupplier supplier; + + @Mock + private ApplicationEventPublisher eventPublisher; + + private LdioLdesClient client; + private final String pipelineName = "pipeline"; + + @BeforeEach + void setUp() { + when(observer.getPipelineName()).thenReturn(pipelineName); + client = new LdioLdesClient( + componentExecutor, + observer, + supplier, + eventPublisher, + false + ); + } + + @AfterEach + void tearDown() { + client.shutdown(); + } + + @Test + void when_EndOfLdesException_ShutdownPipeline() { + when(supplier.get()).thenThrow(EndOfLdesException.class); + + client.start(); + + verify(eventPublisher).publishEvent(new PipelineShutdownEvent(pipelineName)); + } + + @Test + void when_RuntimeException_StopPipeline() { + doThrow(RuntimeException.class).when(supplier).init(); + + client.start(); + + await().atMost(Duration.ofSeconds(20)).untilAsserted(() -> { + verify(eventPublisher).publishEvent(new PipelineStatusEvent(pipelineName, PipelineStatus.RUNNING, StatusChangeSource.MANUAL)); + verify(eventPublisher).publishEvent(new PipelineStatusEvent(pipelineName, PipelineStatus.HALTED, StatusChangeSource.MANUAL)); + }); + + } +} \ No newline at end of file diff --git a/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml index d325a4ea5..29adabc65 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml @@ -6,7 +6,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT ldio-ngsiv2-to-ld-adapter @@ -15,7 +15,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ngsiv2-to-ld-adapter - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT compile diff --git a/ldi-orchestrator/ldio-connectors/ldio-noop-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-noop-out/pom.xml index 5c323206c..c26b9ea0e 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-noop-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-noop-out/pom.xml @@ -4,7 +4,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml index f05a5ffb9..d68daa460 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/pom.xml index 4c7b5282f..e3d94d870 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT ldio-repository-materialiser diff --git a/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml index 47394bb2a..47a26960a 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT ldio-request-executor diff --git a/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml index c511d5cc7..6c593306b 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml index 650eb8caa..a64129e36 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml index e97c0e046..2c1fa82cc 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml @@ -3,7 +3,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml index af936acd3..beb12f2a7 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml @@ -3,7 +3,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/pom.xml b/ldi-orchestrator/ldio-connectors/pom.xml index 97aaa5688..4ba82346c 100644 --- a/ldi-orchestrator/ldio-connectors/pom.xml +++ b/ldi-orchestrator/ldio-connectors/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-orchestrator - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/pom.xml b/ldi-orchestrator/pom.xml index 3dce0d4ea..f8db05e4e 100644 --- a/ldi-orchestrator/pom.xml +++ b/ldi-orchestrator/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT 4.0.0 diff --git a/pom.xml b/pom.xml index 2defd8ccf..a98faaec1 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions pom - 2.6.0-SNAPSHOT + 2.7.0-SNAPSHOT ldi-api