🚧 Refactor to adapt to latest version (WIP)

fundacionctic · Feb 20, 2024 · 13f7468 · 13f7468
1 parent 3df8df9
commit 13f7468
Show file tree

Hide file tree

Showing 41 changed files with 708 additions and 575 deletions.
diff --git a/README.md b/README.md
@@ -27,7 +27,7 @@ The approach taken here is that **any data space participant component can expos
 The repository is organized as follows:
 
 * The `connector` folder contains a Java project with a very early draft version of the _Core Connector_ extension. This extension is responsible for creating the assets in the data space based on the OpenAPI schema of the participant component.
-* The `mock-component` folder contains an example data space participant that exposes both an HTTP API and an event-driven API based on RabbitMQ. These APIs are described by [OpenAPI](https://learn.openapis.org/) and [AsyncAPI](https://www.asyncapi.com/docs) documents, respectively. The logic of the component itself does not hold any value; its purpose is to demonstrate where each partner should contribute.
+* The `mock-backend` folder contains an example data space participant that exposes both an HTTP API and an event-driven API based on RabbitMQ. These APIs are described by [OpenAPI](https://learn.openapis.org/) and [AsyncAPI](https://www.asyncapi.com/docs) documents, respectively. The logic of the component itself does not hold any value; its purpose is to demonstrate where each partner should contribute.
 
 > Support for AsyncAPI and event-driven APIs is a nice-to-have that is not currently being prioritized. It will be addressed at a later stage if time permits and there are no technological roadblocks.
 
@@ -41,7 +41,7 @@ This repository publishes two software artifacts for convenience:
 * The `edcpy` Python package, which is [published to PyPI](https://pypi.org/project/edcpy/).
 * The `agmangas/edc-connector` Docker image for the _Core Connector_, which is [published to Docker Hub](https://hub.docker.com/r/agmangas/edc-connector).
 
-### Configuration of the Connector Image 
+### Configuration of the Connector Image
 
 Although the later examples go into more detail about how to configure the connector, it is relevant to note that the `agmangas/edc-connector` image expects the following environment variables:
 

diff --git a/Taskfile.yml b/Taskfile.yml
@@ -10,16 +10,14 @@ env:
   EXAMPLE_PROVIDER_CERTS_DIR: "{{.ROOT_DIR}}/example/certs-provider"
   KEY_ALIAS: datacellar
   KEY_PASSW: datacellar
-  EXAMPLE_CONNECTOR_PROVIDER_ID: example-provider
-  EXAMPLE_CONNECTOR_CONSUMER_ID: example-consumer
-  EXAMPLE_KEYCLOAK_REALM: edc-example-realm
 
 tasks:
   clean:
     desc: Performs cleanup by removing generated artifacts, including files, directories, and containers resulting from task execution
     cmds:
       - docker compose -f {{.ROOT_DIR}}/docker-compose-consumer.yml down -v
       - docker compose -f {{.ROOT_DIR}}/docker-compose-provider.yml down -v
+      - docker compose -f {{.ROOT_DIR}}/mock-backend/docker-compose.yml down -v
       - rm -fr {{.ROOT_DIR}}/example/certs-*
       - rm -fr {{.ROOT_DIR}}/example/*.jar
       - cd {{.CONNECTOR_DIR}} && gradle clean
@@ -49,74 +47,51 @@ tasks:
     cmds:
       - docker build -t {{.EDCPY_IMAGE}} -f Dockerfile.edcpy .
 
-  create-example-certs-consumer:
-    desc: Creates the certificates used by the example consumer
+  create-certs:
     deps:
       - build-scripts-image
+    requires:
+      vars: [CERTS_DIR]
     cmds:
-      - mkdir -p {{.EXAMPLE_CONSUMER_CERTS_DIR}}
+      - mkdir -p {{.CERTS_DIR}}
       - >
-        docker run --rm 
-        -v {{.EXAMPLE_CONSUMER_CERTS_DIR}}:/out 
+        docker run --rm
+        -v {{.CERTS_DIR}}:/out
         {{.SCRIPTS_IMAGE}}
         /bin/bash -c "OUT_DIR=/out KEY_ALIAS={{.KEY_ALIAS}} KEY_PASSW={{.KEY_PASSW}} ./create-certs.sh"
     status:
-      - test -f {{.EXAMPLE_CONSUMER_CERTS_DIR}}/*.pfx
+      - test -f {{.CERTS_DIR}}/*.pfx
+
+  create-example-certs-consumer:
+    desc: Creates the certificates used by the example consumer
+    cmds:
+      - task: create-certs
+        vars:
+          CERTS_DIR: "{{.EXAMPLE_CONSUMER_CERTS_DIR}}"
 
   create-example-certs-provider:
     desc: Creates the certificates used by the example provider
-    deps:
-      - build-scripts-image
     cmds:
-      - mkdir -p {{.EXAMPLE_PROVIDER_CERTS_DIR}}
-      - >
-        docker run --rm 
-        -v {{.EXAMPLE_PROVIDER_CERTS_DIR}}:/out 
-        {{.SCRIPTS_IMAGE}}
-        /bin/bash -c "OUT_DIR=/out KEY_ALIAS={{.KEY_ALIAS}} KEY_PASSW={{.KEY_PASSW}} ./create-certs.sh"
-    status:
-      - test -f {{.EXAMPLE_PROVIDER_CERTS_DIR}}/*.pfx
+      - task: create-certs
+        vars:
+          CERTS_DIR: "{{.EXAMPLE_PROVIDER_CERTS_DIR}}"
 
-  move-connector-jar:
-    desc: Moves the connector JAR to the example directory
-    deps:
-      - build-connector
+  provision-example-provider:
     cmds:
+      - task: create-example-certs-provider
       - >
-        cp -f 
-        {{.ROOT_DIR}}/{{.CONNECTOR_DIR}}/core-connector/build/libs/core-connector.jar 
-        {{.ROOT_DIR}}/example/
-
-  create-keycloak-client-example-consumer:
-    desc: Creates the OAuth2 client for the example consumer in Keycloak
-    deps:
-      - build-edcpy-image
-      - create-example-certs-consumer
-    cmds:
+        docker compose
+        -f {{.ROOT_DIR}}/mock-backend/docker-compose.yml
+        up -d --build --wait
       - >
-        docker run --rm 
-        -v {{.EXAMPLE_CONSUMER_CERTS_DIR}}:/certs 
-        -v /var/run/dbus:/var/run/dbus 
-        -v /var/run/avahi-daemon/socket:/var/run/avahi-daemon/socket 
-        {{.EDCPY_IMAGE}}
-        configure-keycloak-connector 
-        --connector-id {{.EXAMPLE_CONNECTOR_CONSUMER_ID}} 
-        --connector-cert-path /certs/cert.pem 
-        --keycloak-realm {{.EXAMPLE_KEYCLOAK_REALM}}
+        docker compose
+        -f {{.ROOT_DIR}}/docker-compose-provider.yml
+        up -d --build --wait
 
-  create-keycloak-client-example-provider:
-    desc: Creates the OAuth2 client for the example provider in Keycloak
-    deps:
-      - build-edcpy-image
-      - create-example-certs-provider
+  provision-example-consumer:
     cmds:
+      - task: create-example-certs-consumer
       - >
-        docker run --rm 
-        -v {{.EXAMPLE_PROVIDER_CERTS_DIR}}:/certs 
-        -v /var/run/dbus:/var/run/dbus 
-        -v /var/run/avahi-daemon/socket:/var/run/avahi-daemon/socket 
-        {{.EDCPY_IMAGE}}
-        configure-keycloak-connector 
-        --connector-id {{.EXAMPLE_CONNECTOR_PROVIDER_ID}} 
-        --connector-cert-path /certs/cert.pem 
-        --keycloak-realm {{.EXAMPLE_KEYCLOAK_REALM}}
+        docker compose
+        -f {{.ROOT_DIR}}/docker-compose-consumer.yml
+        up -d --build --wait
diff --git a/Vagrantfile b/Vagrantfile
@@ -6,7 +6,7 @@ Vagrant.configure("2") do |config|
     v.cpus = 1
   end
 
-  config.vm.provision "shell", path: "scripts/provision-common.sh"
+  config.vm.provision "shell", path: "example/provision-boxes.sh"
 
   # Install Avahi to enable mDNS resolution between boxes
   # https://stackoverflow.com/a/30780347
@@ -20,30 +20,14 @@ Vagrant.configure("2") do |config|
   # The provider must be provisioned before the consumer
   config.vm.define "provider", primary: true do |c|
     c.vm.hostname = "provider"
-    c.vm.provision "shell", path: "scripts/provision-provider.sh"
+    c.vm.provision "shell", inline: "cd /vagrant && task clean && task provision-example-provider"
     c.vm.network "private_network", type: "dhcp"
   end
 
   config.vm.define "consumer" do |c|
     c.vm.hostname = "consumer"
-    c.vm.provision "shell", path: "scripts/provision-consumer.sh"
+    c.vm.provision "shell", inline: "cd /vagrant && task provision-example-consumer"
     c.vm.network "private_network", type: "dhcp"
     c.vm.network "forwarded_port", guest: 15672, host: 30200
   end
-
-  config.vm.define "keycloak" do |c|
-    c.vm.hostname = "keycloak"
-
-    # This is a quick and dirty fix to avoid issues with 
-    # the "iat" claim validation in the EDC connector.
-    c.vm.provision "shell", inline: <<-SHELL
-      set -ex
-      tm=$(python3 -c "import datetime; now = datetime.datetime.now(); ago = now - datetime.timedelta(seconds=2); print(ago);") \
-      && timedatectl set-time "$tm"
-    SHELL
-
-    c.vm.provision "shell", path: "scripts/provision-keycloak.sh"
-    c.vm.network "private_network", type: "dhcp"
-    c.vm.network "forwarded_port", guest: 8080, host: 30100
-  end
 end
diff --git a/connector/core-connector/build.gradle.kts b/connector/core-connector/build.gradle.kts
@@ -5,21 +5,25 @@ plugins {
 }
 
 dependencies {
+    implementation(libs.edc.control.plane.api.client)
+    implementation(libs.edc.control.plane.api)
     implementation(libs.edc.control.plane.core)
     implementation(libs.edc.dsp)
-    implementation(libs.edc.iam.mock)
-    implementation(libs.edc.management.api)
-    implementation(libs.edc.data.plane.core)
-    implementation(libs.edc.data.plane.api)
-    implementation(libs.edc.data.plane.http)
     implementation(libs.edc.configuration.filesystem)
     implementation(libs.edc.vault.filesystem)
+    implementation(libs.edc.iam.mock)
+    implementation(libs.edc.management.api)
     implementation(libs.edc.transfer.data.plane)
     implementation(libs.edc.transfer.pull.http.receiver)
+
     implementation(libs.edc.data.plane.selector.api)
     implementation(libs.edc.data.plane.selector.core)
     implementation(libs.edc.data.plane.selector.client)
 
+    implementation(libs.edc.data.plane.api)
+    implementation(libs.edc.data.plane.core)
+    implementation(libs.edc.data.plane.http)
+
     implementation(libs.swaggerParser)
     implementation(libs.slugify)
 

diff --git a/docker-compose-consumer-hub-image.yml → docker-compose-consumer-openjdk.yml b/docker-compose-consumer-hub-image.yml → docker-compose-consumer-openjdk.yml
@@ -1,25 +1,35 @@
 services:
   consumer:
-    image: agmangas/edc-connector:latest-oauth2
+    build:
+      context: .
+      dockerfile: Dockerfile.openjdk
     container_name: consumer
     restart: on-failure
+    command:
+      [
+        "java",
+        "-Dedc.fs.config=/opt/src/example/consumer.properties",
+        "-Dedc.keystore=/opt/src/example/certs-consumer/cert.pfx",
+        "-Dedc.keystore.password=datacellar",
+        "-Dedc.vault=/opt/src/example/certs-consumer/vault.properties",
+        "-jar",
+        "/opt/src/example/core-connector.jar"
+      ]
     ports:
-      - 9191:9191
-      - 9193:9193
-      - 9194:9194
-      - 9291:9291
-      - 9192:9192
-    environment:
-      PROPERTIES_FILE_PATH: /opt/src/example/consumer.properties
-      KEYSTORE_PATH: /opt/src/example/certs-consumer/cert.pfx
-      KEYSTORE_PASSWORD: datacellar
+      - "9191:9191"
+      - "9193:9193"
+      - "9194:9194"
+      - "9291:9291"
+      - "9192:9192"
     volumes:
       - .:/opt/src
       # To ensure that mDNS resolution is enabled in the container
       - /var/run/dbus:/var/run/dbus
       - /var/run/avahi-daemon/socket:/var/run/avahi-daemon/socket
   consumer_backend:
-    image: agmangas/edc-connector:latest-oauth2
+    build:
+      context: .
+      dockerfile: Dockerfile.edcpy
     container_name: consumer_backend
     restart: on-failure
     command: [ "run-http-backend" ]
@@ -30,12 +40,14 @@ services:
     environment:
       # The public key of the provider is needed to verify the signature of the JWT
       EDC_CERT_PATH: /opt/src/example/certs-provider/cert.pem
-      EDC_RABBIT_URL: amqp://guest:guest@broker:5672
+      EDC_RABBIT_URL: "amqp://guest:guest@broker:5672"
       EDC_HTTP_API_PORT: 8000
     ports:
-      - 8000:8000
+      - "8000:8000"
   consumer_sandbox:
-    image: agmangas/edc-connector:latest-oauth2
+    build:
+      context: .
+      dockerfile: Dockerfile.edcpy
     container_name: consumer_sandbox
     restart: on-failure
     command: [ "sleep", "infinity" ]
@@ -45,15 +57,16 @@ services:
       - /var/run/avahi-daemon/socket:/var/run/avahi-daemon/socket
     environment:
       EDC_CERT_PATH: /opt/src/example/certs-provider/cert.pem
-      EDC_RABBIT_URL: amqp://guest:guest@broker:5672
+      EDC_RABBIT_URL: "amqp://guest:guest@broker:5672"
     env_file: ./example/.env.example
   broker:
     image: rabbitmq:3.11-management
     container_name: consumer_broker
     restart: on-failure
     ports:
-      - 5672:5672
-      - 15672:15672
+      - "5672:5672"
+      - "15672:15672"
     environment:
       RABBITMQ_DEFAULT_USER: guest
       RABBITMQ_DEFAULT_PASS: guest
+
diff --git a/docker-compose-consumer.yml b/docker-compose-consumer.yml
@@ -1,72 +1,50 @@
 services:
   consumer:
-    build:
-      context: .
-      dockerfile: Dockerfile.openjdk
+    build: .
+    image: edc-connector
+    pull_policy: never
     container_name: consumer
     restart: on-failure
-    command:
-      [
-        "java",
-        "-Dedc.fs.config=/opt/src/example/consumer.properties",
-        "-Dedc.keystore=/opt/src/example/certs-consumer/cert.pfx",
-        "-Dedc.keystore.password=datacellar",
-        "-Dedc.vault=/opt/src/example/certs-consumer/vault.properties",
-        "-jar",
-        "/opt/src/example/core-connector.jar"
-      ]
     ports:
-      - "9191:9191"
-      - "9193:9193"
-      - "9194:9194"
-      - "9291:9291"
-      - "9192:9192"
+      - 9191:9191
+      - 9193:9193
+      - 9194:9194
+      - 9291:9291
+      - 9192:9192
+    environment:
+      PROPERTIES_FILE_PATH: /opt/src/example/consumer.properties
+      KEYSTORE_PATH: /opt/src/example/certs-consumer/cert.pfx
+      KEYSTORE_PASSWORD: datacellar
     volumes:
       - .:/opt/src
       # To ensure that mDNS resolution is enabled in the container
       - /var/run/dbus:/var/run/dbus
       - /var/run/avahi-daemon/socket:/var/run/avahi-daemon/socket
   consumer_backend:
-    build:
-      context: .
-      dockerfile: Dockerfile.edcpy
+    build: .
+    image: edc-connector
+    pull_policy: never
     container_name: consumer_backend
     restart: on-failure
-    command: [ "run-http-backend" ]
+    command: ["run-http-backend"]
     volumes:
       - .:/opt/src
       - /var/run/dbus:/var/run/dbus
       - /var/run/avahi-daemon/socket:/var/run/avahi-daemon/socket
     environment:
       # The public key of the provider is needed to verify the signature of the JWT
       EDC_CERT_PATH: /opt/src/example/certs-provider/cert.pem
-      EDC_RABBIT_URL: "amqp://guest:guest@broker:5672"
+      EDC_RABBIT_URL: amqp://guest:guest@broker:5672
       EDC_HTTP_API_PORT: 8000
     ports:
-      - "8000:8000"
-  consumer_sandbox:
-    build:
-      context: .
-      dockerfile: Dockerfile.edcpy
-    container_name: consumer_sandbox
-    restart: on-failure
-    command: [ "sleep", "infinity" ]
-    volumes:
-      - .:/opt/src
-      - /var/run/dbus:/var/run/dbus
-      - /var/run/avahi-daemon/socket:/var/run/avahi-daemon/socket
-    environment:
-      EDC_CERT_PATH: /opt/src/example/certs-provider/cert.pem
-      EDC_RABBIT_URL: "amqp://guest:guest@broker:5672"
-    env_file: ./example/.env.example
+      - 8000:8000
   broker:
     image: rabbitmq:3.11-management
     container_name: consumer_broker
     restart: on-failure
     ports:
-      - "5672:5672"
-      - "15672:15672"
+      - 5672:5672
+      - 15672:15672
     environment:
       RABBITMQ_DEFAULT_USER: guest
       RABBITMQ_DEFAULT_PASS: guest
-