ci: finalise 2.13.0 (#750)

Author: Yalz

.github/workflows/deploy-documentation-old.yml

@@ -34,7 +34,7 @@ jobs:
           export VERSION=$(mvn help:evaluate -Dexpression="project.version" -q -DforceStdout)
           echo "version=$VERSION" >> $>> $GITHUB_OUTPUT
       - name: Setup Ruby
-        uses: ruby/setup-ruby@0a29871fe2b0200a17a4497bae54fe5df0d973aa
+        uses: ruby/setup-ruby@v1
         with:
           ruby-version: '3.0' # Not needed with a .ruby-version file
           bundler-cache: true # runs 'bundle install' and caches installed gems automatically

.github/workflows/deploy-documentation.yml

@@ -15,28 +15,61 @@ The processors can be imported into a NiFi docker instance via volume binding:
 1. Create a `docker-compose.yml` file with the following content in a new directory
     ````yaml
     services:
-      nifi:
-        image: apache/nifi:2.0.0
-        environment:
-          SINGLE_USER_CREDENTIALS_USERNAME: admin
-          SINGLE_USER_CREDENTIALS_PASSWORD: ctsBtRBKHRAx69EqUghvvgEvjnaLjFEB
-        ports:
-          - 8443:8443
-        volumes:
-          - ./nifi-ext:/opt/nifi/nifi-current/nar_extensions:rw
+        nifi:
+    image: apache/nifi:2.0.0
+    environment:
+      SINGLE_USER_CREDENTIALS_USERNAME: admin
+      SINGLE_USER_CREDENTIALS_PASSWORD: ctsBtRBKHRAx69EqUghvvgEvjnaLjFEB
+    ports:
+      - 8443:8443
+    volumes:
+      - ./nifi-ext:/opt/nifi/nifi-current/nar_extensions:rw
+      - ./nifi-drivers:/opt/nifi/nifi-current/lib_temp:rw
+    entrypoint: [ "/bin/bash",
+                  "-c",
+                  "cp -r /opt/nifi/nifi-current/lib_temp/* /opt/nifi/nifi-current/lib/ && exec /opt/nifi/scripts/start.sh" ]
     ````
 2. Create a directory `nifi-ext` in your current directory.
 3. Download either the `...-nar-bundle.jar` and unpack this or download the individual required processors (.nar extension) from the [nexus repository].
    Next, place the required processors in the `nifi-ext` directory.
-4. Finally, start your instance.
+4. Additionally, for using any processor with a persistence layer, you will need to download the needed drivers.
+   Place these in the `nifi-drivers` directory.
+   Possible drivers are:
+   - [PostgreSQL driver](https://jdbc.postgresql.org/download.html)
+   - [SQLite driver](https://mvnrepository.com/artifact/org.xerial/sqlite-jdbc)
+5. Finally, start your instance.
     ````shell
     docker compose up
     ````
-5. Log in at `https://localhost:8443/nifi` with the credentials mentioned in step 1
-6. All downloaded extensions are available under the ``be.vlaanderen.informatievlaanderen.ldes.ldi.nifi`` group.
+6. Log in at `https://localhost:8443/nifi` with the credentials mentioned in step 1
+7. All downloaded extensions are available under the ``be.vlaanderen.informatievlaanderen.ldes.ldi.nifi`` group.
 
 {: .note }
 All documentation and notes about configuration are available in the NiFi component itself.
 
+## Setting up Database Connection Pools
+
+As of LDI 2.13.0, some processors require a database connection pool to be set up.
+Although this is a pure NiFi configuration, a minimal explanation will be mentioned here. \
+For more details, please refer to
+the [NiFi documentation on DBCP](https://nifi.apache.org/components/org.apache.nifi.dbcp.DBCPConnectionPool/).
+This can be done by following these steps:
+
+1. Go to the `Controller Services` tab in the NiFi UI.
+2. Click on the `Create new service` button.
+3. Select the `DBCPConnectionPool` service.
+4. Configure the service with the required properties:
+   - `Database Connection URL`,
+      - `jdbc:sqlite:/path/to/database.db` for SQLite
+      - `jdbc:postgresql://localhost:5432/database` for PostgreSQL,
+   - `Database Driver Class Name`
+      - `org.sqlite.JDBC` for SQLite
+      - `org.postgresql.Driver` for PostgreSQL,
+   - `Database User`,
+   - `Password`,
+5. Save the service.
+6. Enable the service.
+7. Use the service in the processors that require it.
+
 [Apache NiFi]: https://nifi.apache.org/
 [nexus repository]: https://s01.oss.sonatype.org/#nexus-search;quick~be.vlaanderen.informatievlaanderen.ldes.ldi.nifi

docs/_ldi-nifi/index.md

@@ -3,7 +3,7 @@
     <parent>
         <groupId>be.vlaanderen.informatievlaanderen.ldes</groupId>
         <artifactId>linked-data-interactions</artifactId>
-        <version>2.12.0</version>
+        <version>2.13.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

ldi-api/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>be.vlaanderen.informatievlaanderen.ldes.ldi</groupId>
         <artifactId>ldi-core</artifactId>
-        <version>2.12.0</version>
+        <version>2.13.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>change-detection-filter</artifactId>

ldi-core/change-detection-filter/pom.xml

@@ -27,11 +27,9 @@ public class ChangeDetectionFilter implements LdiOneToOneTransformer {
 	private static final Lang NORMALIZING_LANG = Lang.NQUADS;
 	private static final MediaType NORMALIZING_MEDIA_TYPE = MediaType.N_QUADS;
 	private final HashedStateMemberRepository hashedStateMemberRepository;
-	private final boolean keepState;
 
-	public ChangeDetectionFilter(HashedStateMemberRepository hashedStateMemberRepository, boolean keepState) {
+	public ChangeDetectionFilter(HashedStateMemberRepository hashedStateMemberRepository) {
 		this.hashedStateMemberRepository = hashedStateMemberRepository;
-		this.keepState = keepState;
 	}
 
 	/**
@@ -54,16 +52,10 @@ public Model transform(Model model) {
 		return model;
 	}
 
-	/**
-	 * Clean up the resources, should be called when the component is not used anymore
-	 */
-	public void destroyState() {
-		if (!keepState) {
-			hashedStateMemberRepository.destroyState();
-		}
+	public void close() {
+		hashedStateMemberRepository.close();
 	}
 
-
 	private Resource getSingleNamedNodeFromStateObject(Model model) {
 		final List<Resource> namedNodes = model.listSubjects().filterDrop(RDFNode::isAnon).toList();
 		if (namedNodes.size() != 1) {

ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/ChangeDetectionFilter.java

@@ -18,5 +18,5 @@ public interface HashedStateMemberRepository {
 	/**
 	 * Clean up the resources, should be called when the repository is not used anymore
 	 */
-	void destroyState();
+	void close();
 }

ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/repositories/HashedStateMemberRepository.java

@@ -3,7 +3,8 @@
 import be.vlaanderen.informatievlaanderen.ldes.ldi.entities.HashedStateMember;
 import be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.HashedStateMemberRepository;
 
-import java.util.*;
+import java.util.HashMap;
+import java.util.Map;
 
 public class InMemoryHashedStateMemberRepository implements HashedStateMemberRepository {
 	private final Map<String, HashedStateMember> members;
@@ -24,7 +25,7 @@ public void saveHashedStateMember(HashedStateMember hashedStateMember) {
 	}
 
 	@Override
-	public void destroyState() {
+	public void close() {
 		members.clear();
 	}
 }

ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/repositories/inmemory/InMemoryHashedStateMemberRepository.java

@@ -1,6 +1,5 @@
 package be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.sql;
 
-import be.vlaanderen.informatievlaanderen.ldes.ldi.EntityManagerFactory;
 import be.vlaanderen.informatievlaanderen.ldes.ldi.entities.HashedStateMember;
 import be.vlaanderen.informatievlaanderen.ldes.ldi.entities.HashedStateMemberEntity;
 import be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.HashedStateMemberRepository;
@@ -9,14 +8,10 @@
 import javax.persistence.EntityManager;
 
 public class SqlHashedStateMemberRepository implements HashedStateMemberRepository {
-	private final EntityManagerFactory entityManagerFactory;
 	private final EntityManager entityManager;
-	private final String instanceName;
 
-	public SqlHashedStateMemberRepository(EntityManagerFactory entityManagerFactory, String instanceName) {
-		this.entityManagerFactory = entityManagerFactory;
-		this.entityManager = entityManagerFactory.getEntityManager();
-		this.instanceName = instanceName;
+	public SqlHashedStateMemberRepository(EntityManager entityManager) {
+		this.entityManager = entityManager;
 	}
 
 	@Override
@@ -38,7 +33,7 @@ public void saveHashedStateMember(HashedStateMember hashedStateMember) {
 	}
 
 	@Override
-	public void destroyState() {
-		entityManagerFactory.destroyState(instanceName);
+	public void close() {
+		entityManager.close();
 	}
 }

ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/repositories/sql/SqlHashedStateMemberRepository.java

@@ -1,11 +1,9 @@
 package be.vlaanderen.informatievlaanderen.ldes.ldi;
 
-import be.vlaanderen.informatievlaanderen.ldes.ldi.postgres.PostgresEntityManagerFactory;
 import be.vlaanderen.informatievlaanderen.ldes.ldi.postgres.PostgresProperties;
 import be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.HashedStateMemberRepository;
 import be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.inmemory.InMemoryHashedStateMemberRepository;
 import be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.sql.SqlHashedStateMemberRepository;
-import be.vlaanderen.informatievlaanderen.ldes.ldi.sqlite.SqliteEntityManagerFactory;
 import be.vlaanderen.informatievlaanderen.ldes.ldi.sqlite.SqliteProperties;
 import io.cucumber.java.en.Given;
 import io.cucumber.java.en.Then;
@@ -34,14 +32,14 @@ public void teardown() {
 	@Given("A ChangeDetectionFilter with state persistence strategy MEMORY")
 	public void aChangeDetectionFilterWithStatePersistenceStrategyMEMORY() {
 		HashedStateMemberRepository hashedStateMemberRepository = new InMemoryHashedStateMemberRepository();
-		changeDetectionFilter = new ChangeDetectionFilter(hashedStateMemberRepository, false);
+		changeDetectionFilter = new ChangeDetectionFilter(hashedStateMemberRepository);
 	}
 
 	@Given("A ChangeDetectionFilter with state persistence strategy SQLITE")
 	public void aChangeDetectionFilterWithStatePersistenceStrategySQLITE() {
-		final var emf = SqliteEntityManagerFactory.getInstance(new SqliteProperties(DATABASE_INSTANCE_NAME, false));
-		HashedStateMemberRepository hashedStateMemberRepository = new SqlHashedStateMemberRepository(emf, DATABASE_INSTANCE_NAME);
-		changeDetectionFilter = new ChangeDetectionFilter(hashedStateMemberRepository, false);
+		var entityManager = HibernateUtil.createEntityManagerFromProperties(new SqliteProperties("target", DATABASE_INSTANCE_NAME, false).getProperties());
+		HashedStateMemberRepository hashedStateMemberRepository = new SqlHashedStateMemberRepository(entityManager);
+		changeDetectionFilter = new ChangeDetectionFilter(hashedStateMemberRepository);
 	}
 
 	@Given("A ChangeDetectionFilter with state persistence strategy POSTGRES")
@@ -55,9 +53,9 @@ public void aChangeDetectionFilterWithStatePersistenceStrategyPOSTGRES() {
 		PostgresProperties postgresProperties = new PostgresProperties(postgreSQLContainer.getJdbcUrl(),
 				postgreSQLContainer.getUsername(), postgreSQLContainer.getPassword(), false);
 
-		final var emf = PostgresEntityManagerFactory.getInstance(DATABASE_INSTANCE_NAME, postgresProperties.getProperties());
-		HashedStateMemberRepository hashedStateMemberRepository = new SqlHashedStateMemberRepository(emf, DATABASE_INSTANCE_NAME);
-		changeDetectionFilter = new ChangeDetectionFilter(hashedStateMemberRepository, false);
+		var entityManager = HibernateUtil.createEntityManagerFromProperties(postgresProperties.getProperties());
+		HashedStateMemberRepository hashedStateMemberRepository = new SqlHashedStateMemberRepository(entityManager);
+		changeDetectionFilter = new ChangeDetectionFilter(hashedStateMemberRepository);
 	}
 
 	@Then("The filtered member is not empty")
@@ -85,6 +83,6 @@ public void theFilteredMemberIsIsomorphicWith(String fileName) {
 
 	@Then("The filter is destroyed")
 	public void theFilterIsDestroyed() {
-		changeDetectionFilter.destroyState();
+		changeDetectionFilter.close();
 	}
 }