Merge pull request #987 from assemblee-virtuelle/698_ldp_patch_import…

…_resource_one_by_one

698 ldp patch import resource one by one

src/jena/docker-compose.yaml

@@ -25,3 +25,13 @@ services:
     container_name: fuseki_compact
     volumes:
       - ./data/rdf_data:/fuseki
+
+  fuseki_migrate:
+    build:
+      context: fuseki-docker
+      dockerfile: Dockerfile
+    entrypoint: /docker-migration-entrypoint.sh
+    image: semapps/jena-fuseki-webacl
+    container_name: fuseki_migration
+    volumes:
+      - ./data/rdf_data:/fuseki

src/jena/fuseki-docker/Dockerfile

@@ -17,7 +17,7 @@
 #FROM alpine:3.4
 #RUN apk add --update openjdk8-jre pwgen bash wget ca-certificates && rm -rf /var/cache/apk/*
 FROM openjdk:8-jre-alpine
-RUN apk add --update pwgen bash curl ca-certificates procps && rm -rf /var/cache/apk/*
+RUN apk add --update nodejs npm pwgen bash curl ca-certificates procps && rm -rf /var/cache/apk/*
 
 MAINTAINER Niko PLP <[email protected]>
 
@@ -82,15 +82,18 @@ RUN chmod 755 $FUSEKI_HOME/bin/tdb2.*
 # (which we'll generate on start-up)
 
 COPY shiro.ini /jena-fuseki/
+COPY migration /jena-fuseki/migration
 COPY extra/commons-collections4-4.4.jar /jena-fuseki/extra/commons-collections4-4.4.jar
 COPY extra/jena-permissions-3.17.0.jar /jena-fuseki/extra/jena-permissions-3.17.0.jar
 COPY extra/semapps-jena-permissions-1.0.0.jar /jena-fuseki/extra/semapps-jena-permissions-1.0.0.jar
 #COPY configuration/localData.ttl /jena-fuseki/configuration/localData.ttl
 #COPY configuration/testData.ttl /jena-fuseki/configuration/testData.ttl
 COPY docker-entrypoint.sh /
 COPY docker-compact-entrypoint.sh /
+COPY docker-migration-entrypoint.sh /
 RUN chmod 755 /docker-entrypoint.sh
 RUN chmod 755 /docker-compact-entrypoint.sh
+RUN chmod 755 /docker-migration-entrypoint.sh
 
 COPY load.sh /jena-fuseki/
 COPY tdbloader /jena-fuseki/

src/jena/fuseki-docker/docker-migration-entrypoint.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+#   Licensed to the Apache Software Foundation (ASF) under one or more
+#   contributor license agreements.  See the NOTICE file distributed with
+#   this work for additional information regarding copyright ownership.
+#   The ASF licenses this file to You under the Apache License, Version 2.0
+#   (the "License"); you may not use this file except in compliance with
+#   the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+set -e
+
+cd /jena-fuseki/migration
+
+npm i 
+
+cd /fuseki
+
+node /jena-fuseki/migration/migrate.js > /fuseki/configuration/migration.log
+

src/jena/fuseki-docker/migration/migrate.js

@@ -0,0 +1,50 @@
+const fs = require('fs');
+const path = require('path');
+const format = require('string-template');
+
+const matchAll = require("match-all");
+
+//const folder = path.join(__dirname,'configuration');
+const folder = path.join(path.parse(__dirname).root,'fuseki','configuration');
+
+const rdfsLabelRegex = new RegExp('^\\s*rdfs:label\\s*"([\\w\\s]+)"\\s*[;.]\\s*$','gm')
+const fusekiNameRegex = new RegExp('^\\s*fuseki:name\\s*"([\\w]+)"\\s*[;.]\\s*$','gm')
+const tdb2LocationRegex = new RegExp('^\\s*tdb2:location\\s*"([/\\\\:\\w]+)"\\s*[;.]\\s*$','gm')
+
+let template = fs.readFileSync(path.join(__dirname,'templates','dataset.ttl'), {encoding: 'utf8'} )
+let templateAcl = fs.readFileSync(path.join(__dirname,'templates','secure-dataset.ttl'), {encoding: 'utf8'} )
+
+
+fs.readdirSync(folder).forEach(file => {
+  if (file.endsWith('.ttl')) {
+    console.log(file);
+    const filename = path.join(folder,file)
+    let content = fs.readFileSync(filename, {encoding: 'utf8'} )
+    if (!content.includes('mirror')) {
+      console.log('> migrating...')
+      fs.writeFileSync(filename+'.bak',content)
+      // extract names
+      const rdfsLabel = matchAll(content,rdfsLabelRegex).toArray()[0]
+      const fusekiName =  matchAll(content,fusekiNameRegex).toArray()[0]
+      console.log('> rdfsLabel = ',rdfsLabel)
+      console.log('> fusekiName = ',fusekiName)
+      const locations = matchAll(content,tdb2LocationRegex).toArray();
+      if (content.includes('webacl')) {
+        let aclLocation = locations.filter(l => l.includes('acl') || l.includes('Acl'))
+        aclLocation = aclLocation.length == 1 && aclLocation[0]
+        if (!aclLocation) { console.log('> error: cannot find acl location'); }
+        else {
+          console.log('> aclLocation = ',aclLocation)  
+          const newContent = format(templateAcl, { dataset: fusekiName, aclLocation, rdfsLabel });
+          fs.writeFileSync(filename,newContent)
+        }
+      } else {
+        const newContent = format(template, { dataset: fusekiName, rdfsLabel });
+        fs.writeFileSync(filename,newContent)
+      }
+
+    }
+    else  console.log('> already done.')
+    console.log('')
+  }
+});

src/jena/fuseki-docker/migration/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "migration_mirror",
+  "version": "0.1.0",
+  "description": "Migrates the configuration files of jena fuseki for Dataset, to include the mirror namedGraph",
+  "license": "Apache-2.0",
+  "author": "Virtual Assembly",
+  "dependencies": {
+    "string-template": "^1.0.0",
+    "match-all": "^1.0.0"
+  }
+}

src/jena/fuseki-docker/migration/templates/dataset.ttl

@@ -0,0 +1,64 @@
+@prefix :      <http://base/#> .
+@prefix rdf:   <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
+@prefix tdb2:  <http://jena.apache.org/2016/tdb#> .
+@prefix ja:    <http://jena.hpl.hp.com/2005/11/Assembler#> .
+@prefix rdfs:  <http://www.w3.org/2000/01/rdf-schema#> .
+@prefix fuseki: <http://jena.apache.org/fuseki#> .
+
+tdb2:DatasetTDB  rdfs:subClassOf  ja:RDFDataset .
+
+ja:DatasetTxnMem  rdfs:subClassOf  ja:RDFDataset .
+
+<http://jena.hpl.hp.com/2008/tdb#DatasetTDB>
+        rdfs:subClassOf  ja:RDFDataset .
+
+ja:ViewGraph  rdfs:subClassOf  ja:Model .
+
+<http://jena.hpl.hp.com/2008/tdb#GraphTDB>
+        rdfs:subClassOf  ja:Model .
+
+tdb2:GraphTDB2  rdfs:subClassOf  ja:Model .
+
+ja:MemoryDataset  rdfs:subClassOf  ja:RDFDataset .
+
+ja:RDFDatasetZero  rdfs:subClassOf  ja:RDFDataset .
+
+<http://jena.apache.org/text#TextDataset>
+        rdfs:subClassOf  ja:RDFDataset .
+
+:service_tdb_all  a                   fuseki:Service ;
+        rdfs:label                    "{rdfsLabel}" ;
+        fuseki:dataset                :combinedDataset ;
+        fuseki:name                   "{dataset}" ;
+        fuseki:serviceQuery           "query" , "" , "sparql" ;
+        fuseki:serviceReadGraphStore  "get" ;
+        fuseki:serviceReadWriteGraphStore
+                "data" ;
+        fuseki:serviceUpdate          "" , "update" ;
+        fuseki:serviceUpload          "upload" .
+
+:tdb_dataset_readwrite
+        a              tdb2:DatasetTDB2 ;
+        tdb2:location  "/fuseki/databases/{dataset}" .
+
+:baseModel rdf:type tdb2:GraphTDB ;
+    tdb2:dataset :tdb_dataset_readwrite.
+
+:mirrorModel rdf:type tdb2:GraphTDB ;
+    tdb2:graphName <http://semapps.org/mirror> ;
+    tdb2:dataset :tdb_dataset_readwrite.
+
+:combinedDataset rdf:type ja:RDFDataset ;
+   ja:defaultGraph :baseModel;
+   ja:namedGraph 
+        [ ja:graphName      <http://semapps.org/mirror> ;
+          ja:graph          :mirrorModel ] ;
+   .
+
+tdb2:GraphTDB  rdfs:subClassOf  ja:Model .
+
+ja:RDFDatasetOne  rdfs:subClassOf  ja:RDFDataset .
+
+ja:RDFDatasetSink  rdfs:subClassOf  ja:RDFDataset .
+
+tdb2:DatasetTDB2  rdfs:subClassOf  ja:RDFDataset .

src/jena/fuseki-docker/migration/templates/secure-dataset.ttl

@@ -0,0 +1,99 @@
+# Licensed under the terms of http://www.apache.org/licenses/LICENSE-2.0
+
+PREFIX :        <#>
+PREFIX fuseki:  <http://jena.apache.org/fuseki#>
+PREFIX rdf:     <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+PREFIX rdfs:    <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX tdb2:    <http://jena.apache.org/2016/tdb#>
+PREFIX ja:      <http://jena.hpl.hp.com/2005/11/Assembler#>
+PREFIX sec:     <http://apache.org/jena/permissions/Assembler#>
+PREFIX sa:      <http://semapps.org/#>
+## ---------------------------------------------------------------
+## Updatable TDB2 dataset with all services enabled.
+
+<#tdb_dataset_readwrite_acl> rdf:type      tdb2:DatasetTDB2 ;
+    tdb2:location "{aclLocation}" ;
+    ##tdb2:unionDefaultGraph true ;
+    .
+
+<#tdb_dataset_readwrite> rdf:type      tdb2:DatasetTDB2 ;
+    tdb2:location "/fuseki/databases/{dataset}" ;
+    ##tdb2:unionDefaultGraph true ;
+    .
+
+<#tdb_dataset_readwrite_mirror> rdf:type      tdb2:DatasetTDB2 ;
+    tdb2:location "/fuseki/databases/{dataset}Mirror" ;
+    .
+
+sa:baseModel rdf:type tdb2:GraphTDB ;
+    tdb2:dataset <#tdb_dataset_readwrite> .
+
+sa:mirrorModel rdf:type tdb2:GraphTDB ;
+    tdb2:dataset <#tdb_dataset_readwrite_mirror> .
+
+[] ja:loadClass    "org.apache.jena.permissions.SecuredAssembler" .
+sec:Model       rdfs:subClassOf  ja:NamedModel .
+sec:evaluator   rdfs:domain sec:Model ;
+                rdfs:range sec:Evaluator .
+
+sa:securedModel rdf:type sec:Model ;
+    sec:baseModel sa:baseModel ;
+    ja:modelName "http://semapps.org/securedModel" ;
+    sec:evaluatorImpl sa:secEvaluator .
+
+sa:unprotectedDataset rdf:type ja:RDFDataset ;
+   ja:defaultGraph sa:baseModel;
+   ja:namedGraph 
+        [ ja:graphName      <http://semapps.org/webacl> ;
+          ja:graph          sa:baseACLModel ] ;
+   .
+
+sa:secEvaluator rdf:type sec:Evaluator ;
+    sec:args [  
+      rdf:_1 sa:baseModel ;
+	    rdf:_2 sa:baseACLModel ;
+	    rdf:_3 sa:unprotectedDataset ;
+    ] ;
+    sec:evaluatorClass "org.semapps.jena.permissions.ShiroEvaluator" .
+
+sa:baseACLModel rdf:type tdb2:GraphTDB ;
+    tdb2:graphName <http://semapps.org/webacl> ;
+    tdb2:dataset <#tdb_dataset_readwrite_acl> .
+
+sa:securedACLModel rdf:type sec:Model ;
+    sec:baseModel sa:baseACLModel ;
+    ja:modelName "http://semapps.org/securedWebacl" ;
+    sec:evaluatorImpl sa:secACLEvaluator .
+
+sa:secACLEvaluator rdf:type sec:Evaluator ;
+    sec:args [ 
+ 	    rdf:_1 sa:baseModel ;
+      rdf:_2 sa:baseACLModel ;
+	    rdf:_3 sa:unprotectedDataset ;
+    ] ;
+    sec:evaluatorClass "org.semapps.jena.permissions.ShiroEvaluator" .
+
+
+sa:securedDataset rdf:type ja:RDFDataset ;
+   ja:defaultGraph sa:securedModel;
+   ja:namedGraph 
+        [ ja:graphName      <http://semapps.org/webacl> ;
+          ja:graph          sa:securedACLModel ] ;
+   ja:namedGraph 
+        [ ja:graphName      <http://semapps.org/mirror> ;
+          ja:graph          sa:mirrorModel ] ;
+   .
+
+<#service_tdb_all> rdf:type fuseki:Service ;
+    rdfs:label                      "{rdfsLabel}" ;
+    fuseki:name                     "{dataset}" ;
+    fuseki:serviceQuery             "query" ;
+    fuseki:serviceQuery             "sparql" ;
+    fuseki:serviceUpdate            "update" ;
+    fuseki:serviceUpload            "upload" ;
+    fuseki:serviceReadWriteGraphStore      "data" ;
+    # A separate read-only graph store endpoint:
+    fuseki:serviceReadGraphStore       "get" ;
+    fuseki:dataset          sa:securedDataset  ;
+
+    .

src/middleware/packages/activitypub/service.js

@@ -19,7 +19,7 @@ const ActivityPubService = {
     jsonContext: ['https://www.w3.org/ns/activitystreams', 'https://w3id.org/security/v1'],
     podProvider: false,
     selectActorData: resource => ({
-      '@type': ACTOR_TYPES.PERSON,
+      '@type': resource.type || resource['@type'] || ACTOR_TYPES.PERSON,
       name: undefined,
       preferredUsername: getSlugFromUri(resource.id || resource['@id'])
     }),

src/middleware/packages/activitypub/services/activity.js

@@ -16,6 +16,7 @@ const ActivityService = {
     permissions: {},
     newResourcesPermissions: {},
     readOnly: true,
+    excludeFromMirror: true,
     controlledActions: {
       // Activities shouldn't be handled manually
       patch: 'activitypub.activity.forbidden',

src/middleware/packages/activitypub/services/actor.js

@@ -11,7 +11,7 @@ const ActorService = {
     baseUri: null,
     jsonContext: ['https://www.w3.org/ns/activitystreams', 'https://w3id.org/security/v1'],
     selectActorData: resource => ({
-      '@type': ACTOR_TYPES.PERSON,
+      '@type': resource.type || resource['@type'] || ACTOR_TYPES.PERSON,
       name: undefined,
       preferredUsername: getSlugFromUri(resource.id || resource['@id'])
     }),
@@ -125,7 +125,9 @@ const ActorService = {
       return uri.startsWith(this.settings.baseUri);
     },
     isActor(resource) {
-      return defaultToArray(resource['@type'] || resource.type).some(type => Object.values(ACTOR_TYPES).includes(type));
+      return defaultToArray(resource['@type'] || resource.type || []).some(type =>
+        Object.values(ACTOR_TYPES).includes(type)
+      );
     }
   },
   events: {