#39, resolve relative uris, but save them as relative

dbpedia · Jun 6, 2024 · 7121ca3 · 7121ca3
1 parent 5ac7c47
commit 7121ca3
Show file tree

Hide file tree

Showing 6 changed files with 91 additions and 40 deletions.
diff --git a/src/main/scala/org/dbpedia/databus/ApiImpl.scala b/src/main/scala/org/dbpedia/databus/ApiImpl.scala
@@ -35,13 +35,13 @@ class ApiImpl(config: Config) extends DatabusApi {
 
   def stop() = JenaSystem.shutdown()
 
-
+  // todo NOTICE! this may fail with relative URIS, NOT TESTED!
   override def dataidSubgraph(body: String)(request: HttpServletRequest): Try[String] =
     readModel(
       body.getBytes,
       defaultLang,
       contextUrl(body.getBytes, defaultLang)
-        .map(jenaJsonLdContextWithFallbackForLocalhost(_, request.getRemoteHost).get)
+        .map(jenaJsonLdContextWithFallbackForLocalhost(_, request.getRemoteHost, None).get)
     )
       .flatMap(m => Tractate.extract(m._1.getGraph, TractateV1.Version))
       .map(_.stringForSigning)
@@ -67,8 +67,8 @@ class ApiImpl(config: Config) extends DatabusApi {
       )
   }
 
-  override def getFile(repo: String, path: String)(request: HttpServletRequest): Try[String] =
-    readFile(repo, path)(request)
+  override def getFile(repo: String, path: String, prefix: Option[String])(request: HttpServletRequest): Try[String] =
+    readFile(repo, path, prefix)(request)
 
 
   override def saveFile(repo: String,
@@ -86,12 +86,12 @@ class ApiImpl(config: Config) extends DatabusApi {
       .getOrElse("")
     val lang = mapContentType(ct, defaultLang)
     val ctxU = contextUrl(body.getBytes, lang)
-    val ctx = ctxU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, request.getRemoteHost).get)
+    val ctx = ctxU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, request.getRemoteHost, Some(graphId)).get)
     validateEmail(author_email).flatMap(email =>
       readModel(body.getBytes, lang, ctx)
         .flatMap(model => {
           saveToVirtuoso(model._1, graphId)({
-            graphToBytes(model._1.getGraph, defaultLang, ctxU)
+            graphToBytes(model._1.getGraph, defaultLang, ctx, ctxU)
               .flatMap(a => saveFiles(
                 repo,
                 Map(
@@ -118,18 +118,20 @@ class ApiImpl(config: Config) extends DatabusApi {
     val lang = getLangFromAcceptHeader(request)
     setResponseHeaders(Map("Content-Type" -> lang.getContentType.toHeaderString))(request)
     val ctxU = contextUrl(dataid.getBytes, lang)
-    val ctx = ctxU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, request.getRemoteHost).get)
+    val ctx = ctxU.map(cu =>
+      jenaJsonLdContextWithFallbackForLocalhost(cu, request.getRemoteHost, None).get)
 
     val shaclU = contextUrl(shacl.getBytes, RdfConversions.DefaultShaclLang)
-    val shaclCtx = shaclU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, request.getRemoteHost).get)
+    val shaclCtx = shaclU.map(cu =>
+      jenaJsonLdContextWithFallbackForLocalhost(cu, request.getRemoteHost, None).get)
 
     RdfConversions.validateWithShacl(
       dataid.getBytes,
       shacl.getBytes,
       ctx,
       shaclCtx,
       defaultLang
-    ).flatMap(r => RdfConversions.graphToBytes(r.getGraph, lang, None))
+    ).flatMap(r => RdfConversions.graphToBytes(r.getGraph, lang, shaclCtx, None))
       .map(new String(_))
   }
 
@@ -177,21 +179,22 @@ class ApiImpl(config: Config) extends DatabusApi {
     s"${url.getProtocol}://${url.getHost}:${url.getPort}${config.defaultGraphIdPrefix}/"
   }
 
-  private def readFile(username: String, path: String)(request: HttpServletRequest): Try[String] = {
+  private def readFile(username: String, path: String, prefix: Option[String])(request: HttpServletRequest): Try[String] = {
     val p = gitPath(path)
     val lang = getLangFromAcceptHeader(request)
+    val graphId = generateGraphId(prefix.getOrElse(getPrefix(request)), username, path)
     setResponseHeaders(Map("Content-Type" -> lang.getContentType.toHeaderString))(request)
     client.readFile(username, p)
       .flatMap(body => {
         val ctxUri = contextUrl(body, defaultLang)
+        val ctx = ctxUri.map(jenaJsonLdContextWithFallbackForLocalhost(_, request.getRemoteHost, Some(graphId)).get)
         readModel(
           body,
           defaultLang,
-          contextUrl(body, defaultLang)
-            .map(jenaJsonLdContextWithFallbackForLocalhost(_, request.getRemoteHost).get)
+          ctx
         )
           .flatMap(m =>
-            graphToBytes(m._1.getGraph, lang, ctxUri)
+            graphToBytes(m._1.getGraph, lang, ctx, ctxUri)
           )
       })
       .map(new String(_))

diff --git a/src/main/scala/org/dbpedia/databus/SparqlClient.scala b/src/main/scala/org/dbpedia/databus/SparqlClient.scala
@@ -14,9 +14,10 @@ import org.apache.jena.rdf.model.{Model, ModelFactory}
 import org.apache.jena.riot.lang.LangJSONLD10
 import org.apache.jena.riot.system.{ErrorHandler, ErrorHandlerFactory, StreamRDFLib}
 import org.apache.jena.riot.writer.JsonLD10Writer
-import org.apache.jena.riot.{Lang, RDFDataMgr, RDFFormat, RDFLanguages, RDFParserBuilder, RDFWriter, RIOT}
+import org.apache.jena.riot.{Lang, RDFDataMgr, RDFFormat, RDFLanguages, RDFParser, RDFWriter, RIOT}
 import org.apache.jena.shacl.{ShaclValidator, Shapes, ValidationReport}
 import org.apache.jena.sparql.util
+import org.apache.jena.sparql.util.Context
 import org.dbpedia.databus.ApiImpl.Config
 import org.slf4j.LoggerFactory
 import sttp.client3.{DigestAuthenticationBackend, HttpURLConnectionBackend, basicRequest}
@@ -163,36 +164,34 @@ object RdfConversions {
 
   def readModel(data: Array[Byte], lang: Lang, context: Option[util.Context]): Try[(Model, List[Warning])] = Try {
     val model = ModelFactory.createDefaultModel()
+    val eh = newErrorHandlerWithWarnings
     val dataStream = new ByteArrayInputStream(data)
     val dest = StreamRDFLib.graph(model.getGraph)
-    val parser = RDFParserBuilder.create()
+
+    val parser = RDFParser.create()
       .source(dataStream)
-      .base(null)
+      .errorHandler(eh)
       .lang(lang)
 
-    val eh = newErrorHandlerWithWarnings
-    parser.errorHandler(eh)
-
     context.foreach(cs =>
       parser.context(cs))
 
     parser.parse(dest)
     (model, eh.warningsList)
   }
 
-  def graphToBytes(model: Graph, outputLang: Lang, context: Option[URL]): Try[Array[Byte]] = Try {
+  def graphToBytes(model: Graph, outputLang: Lang, context: Option[Context], contextURL: Option[URL]): Try[Array[Byte]] = Try {
     val str = new ByteArrayOutputStream()
-    val builder = RDFWriter.create.format(langToFormat(outputLang))
+    val builder = RDFWriter.create()
       .source(model)
+      .format(langToFormat(outputLang))
 
-    context.foreach(ctx => {
-      val jctx = jenaContext(CachingContext.parse(ctx.toString))
-      builder.context(jctx)
-      builder.set(JsonLD10Writer.JSONLD_CONTEXT_SUBSTITUTION, new JsonString(ctx.toString))
-    })
+    context.foreach(ctx =>
+      builder.context(ctx))
+    contextURL.foreach(ctx =>
+      builder.set(JsonLD10Writer.JSONLD_CONTEXT_SUBSTITUTION, new JsonString(ctx.toString)))
 
     builder
-      .build()
       .output(str)
     str.toByteArray
   }
@@ -224,8 +223,8 @@ object RdfConversions {
   def langToFormat(lang: Lang): RDFFormat = lang match {
     case RDFLanguages.TURTLE => RDFFormat.TURTLE_PRETTY
     case RDFLanguages.TTL => RDFFormat.TTL
-    case RDFLanguages.JSONLD => RDFFormat.JSONLD10
-    case RDFLanguages.JSONLD10 => RDFFormat.JSONLD10
+    case RDFLanguages.JSONLD => RDFFormat.JSONLD10_COMPACT_PRETTY
+    case RDFLanguages.JSONLD10 => RDFFormat.JSONLD10_COMPACT_PRETTY
     case RDFLanguages.JSONLD11 => RDFFormat.JSONLD11
     case RDFLanguages.TRIG => RDFFormat.TRIG_PRETTY
     case RDFLanguages.RDFXML => RDFFormat.RDFXML_PRETTY
@@ -313,8 +312,17 @@ object RdfConversions {
       None
     }
 
-  def jenaJsonLdContextWithFallbackForLocalhost(jsonLdContextUrl: URL, requestHost: String): Try[util.Context] =
+  def jenaJsonLdContextWithFallbackForLocalhost(jsonLdContextUrl: URL, requestHost: String, baseUrl: Option[String]): Try[util.Context] =
     jsonLdContextWithFallbackForLocalhost(jsonLdContextUrl, requestHost)
+      .map(ctx =>
+        baseUrl
+          .map(bu => {
+            val c = ctx.clone()
+            c.put("@base", bu)
+            c
+          })
+          .getOrElse(ctx)
+      )
       .map(jenaContext)
 
   private def jsonLdContextUrl(data: Array[Byte]): Try[Option[URL]] =
@@ -359,12 +367,14 @@ object RdfConversions {
     ctx
   }
 
-  private def initCachingContext() = {
-    val opts = new JsonLdOptions(null)
+  def defaultJsonLdOpts(base: String) = {
+    val opts = new JsonLdOptions(base)
     opts.useNamespaces = true
-    new CachingJsonldContext(30, opts)
+    opts
   }
 
+  private def initCachingContext() = new CachingJsonldContext(30, defaultJsonLdOpts(null))
+
   private def escapeString(s: String) = {
     val sb = new StringBuilder(s.length())
     val slen = s.length()

diff --git a/src/test/resources/test-relative.jsonld b/src/test/resources/test-relative.jsonld
@@ -0,0 +1,13 @@
+{
+  "@context" : "https://raw.githubusercontent.com/dbpedia/databus-moss/dev/devenv/context2.jsonld",
+  "@id" : "#mod",
+  "wasGeneratedBy" :  {
+      "@id" : "#layer",
+      "@type" : "DatabusMetadataLayer",
+      "version" : "1.0.0",
+      "name": "simple",
+      "created" : "2024-03-01 14:37:32",
+      "used" : "https://databus.dbpedia.org/lrec2024/linguistics/wordnet/2023#wordnet_lang=en.ttl.gz"
+  },
+  "subject" : [ "oeo:OEO_00020033" ]
+}
diff --git a/src/test/scala/org/dbpedia/databus/DatabusScalatraTest.scala b/src/test/scala/org/dbpedia/databus/DatabusScalatraTest.scala
@@ -3,7 +3,7 @@ package org.dbpedia.databus
 
 import org.apache.jena.iri.ViolationCodes
 
-import java.io.ByteArrayInputStream
+import java.io.{ByteArrayInputStream}
 import java.nio.file.{Files, Paths}
 import org.apache.jena.rdf.model.ModelFactory
 import org.apache.jena.riot.{Lang, RDFDataMgr}
@@ -98,6 +98,25 @@ class DatabusScalatraTest extends ScalatraFlatSpec with BeforeAndAfter {
 
   }
 
+  "File save" should "save and retrieve jsonlds with relative uris" in {
+
+    val file = "test-relative.jsonld"
+    val bytes = Files.readAllBytes(Paths.get(getClass.getClassLoader.getResource(file).getFile))
+
+    post("/databus/graph/save?repo=kuckuck&path=pa/rel_test.jsonld", bytes) {
+      status should equal(200)
+    }
+
+    get("/databus/graph/read?repo=kuckuck&path=pa/rel_test.jsonld") {
+      status should equal(200)
+      val respCtx = RdfConversions.contextUrl(bodyBytes, Lang.JSONLD10)
+      respCtx should equal(RdfConversions.contextUrl(bytes, Lang.JSONLD10))
+      respCtx.get.toString.nonEmpty should equal(true)
+      body.contains(" \"generated\" : \"#mod\",") should equal(true)
+    }
+
+  }
+
   "File save" should "report problems in input" in {
 
 

diff --git a/src/test/scala/org/dbpedia/databus/ValidationTest.scala b/src/test/scala/org/dbpedia/databus/ValidationTest.scala
@@ -23,7 +23,7 @@ class ValidationTest extends FlatSpec with Matchers with BeforeAndAfter {
     val bytes = Files.readAllBytes(Paths.get(getClass.getClassLoader.getResource(file).getFile))
 
     val ctxU = contextUrl(bytes, lang)
-    val ctx = ctxU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, "random").get)
+    val ctx = ctxU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, "random", None).get)
 
     val re = RdfConversions.validateWithShacl(bytes, ctx, shacl, lang)
     re.get.conforms() should be(true)
@@ -35,7 +35,7 @@ class ValidationTest extends FlatSpec with Matchers with BeforeAndAfter {
     val bytes = Files.readAllBytes(Paths.get(getClass.getClassLoader.getResource(file).getFile))
 
     val ctxU = contextUrl(bytes, lang)
-    val ctx = ctxU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, "random").get)
+    val ctx = ctxU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, "random", None).get)
 
     val re = RdfConversions.validateWithShacl(bytes, ctx, shacl, lang)
     re.get.conforms() should be(false)
@@ -47,7 +47,7 @@ class ValidationTest extends FlatSpec with Matchers with BeforeAndAfter {
     val bytes = Files.readAllBytes(Paths.get(getClass.getClassLoader.getResource(file).getFile))
 
     val ctxU = contextUrl(bytes, lang)
-    val ctx = ctxU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, "random").get)
+    val ctx = ctxU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, "random", None).get)
 
     val re = RdfConversions.validateWithShacl(bytes, ctx, shacl, lang)
     re.get.conforms() should be(true)
@@ -59,7 +59,7 @@ class ValidationTest extends FlatSpec with Matchers with BeforeAndAfter {
     val bytes = Files.readAllBytes(Paths.get(getClass.getClassLoader.getResource(file).getFile))
 
     val ctxU = contextUrl(bytes, lang)
-    val ctx = ctxU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, "random").get)
+    val ctx = ctxU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, "random", None).get)
 
     val re = RdfConversions.validateWithShacl(bytes, ctx, shacl, lang)
     re.get.conforms() should be(true)
@@ -72,10 +72,10 @@ class ValidationTest extends FlatSpec with Matchers with BeforeAndAfter {
     val bytes = Files.readAllBytes(Paths.get(getClass.getClassLoader.getResource(file).getFile))
 
     val ctxU = contextUrl(bytes, lang)
-    val ctx = ctxU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, "random").get)
+    val ctx = ctxU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, "random", None).get)
 
     val shaclU = contextUrl(shacl, RdfConversions.DefaultShaclLang)
-    val shaclCtx = shaclU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, "random").get)
+    val shaclCtx = shaclU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, "random", None).get)
 
     val re = RdfConversions.validateWithShacl(bytes, shacl, ctx, shaclCtx, lang)
     re.get.conforms() should be(true)

diff --git a/swagger.yaml b/swagger.yaml
@@ -140,6 +140,12 @@ paths:
           type: string
           required: true
           example: "testgroup/test.jsonld"
+        - in: query
+          name: prefix
+          description: "Prefix for graphid URL if it is not default. NOTE! should be without slash in the end."
+          type: string
+          required: false
+          example: "http://foreighhost/api"
       responses:
         "200":
           description: "A file data."