fix bug with expanded context

dbpedia · Aug 5, 2024 · 69b1eb4 · 69b1eb4
1 parent d68d26b
commit 69b1eb4
Show file tree

Hide file tree

Showing 4 changed files with 39,876 additions and 34 deletions.
diff --git a/src/main/scala/org/dbpedia/databus/ApiImpl.scala b/src/main/scala/org/dbpedia/databus/ApiImpl.scala
@@ -22,7 +22,9 @@ import scala.xml.Node
 import collection.JavaConverters._
 
 
-class ApiImpl(config: Config) extends DatabusApi {
+class ApiImpl(config: Config, batchSize: Int = 1000) extends DatabusApi {
+
+  val SPARQLBatchSize = batchSize
 
   import ApiImpl._
 
@@ -227,7 +229,7 @@ class ApiImpl(config: Config) extends DatabusApi {
 
   private[databus] def saveToVirtuoso[T](model: Model, graphId: String)(execInTransaction: => Try[T]): Try[T] = {
     val rqsts = model.getGraph.find().asScala
-      .grouped(1000)
+      .grouped(SPARQLBatchSize)
       .map(tpls => RdfConversions.makeInsertSparqlQuery(tpls, graphId))
       .toSeq
     // NOTE! here the order of concatenation is important!

diff --git a/src/main/scala/org/dbpedia/databus/SparqlClient.scala b/src/main/scala/org/dbpedia/databus/SparqlClient.scala
@@ -360,7 +360,7 @@ object RdfConversions {
     import JsonLDSerialiser._
 
     // NOTE! Not thread safe!
-    private var remoteContext: Option[Try[(URL, util.Context)]] = None
+    private var remoteContext: Option[Try[(Option[URL], util.Context)]] = None
 
     override def graph: Try[(Model, List[Warning])] = {
       parseContext(data, base)
@@ -374,17 +374,15 @@ object RdfConversions {
     override def modifyWriter(writer: RDFWriterBuilder) =
       remoteContext.get.map(ctx => {
         writer.context(ctx._2)
-        writer.set(JsonLD10Writer.JSONLD_CONTEXT_SUBSTITUTION, new JsonString(ctx._1.toString))
+        ctx._1.foreach(url => writer.set(JsonLD10Writer.JSONLD_CONTEXT_SUBSTITUTION, new JsonString(url.toString)))
       })
 
 
-    private def parseContext(body: Array[Byte], base: Option[String]): Option[Try[(URL, util.Context)]] =
+    private def parseContext(body: Array[Byte], base: Option[String]): Option[Try[(Option[URL], util.Context)]] =
       remoteContext match {
         case None | Some(Failure(_)) =>
-          val c = jsonLdContextUrl(body)
-            .map(c => jenaJsonLdContext(c, base).map((c, _)))
-          remoteContext = c
-          c
+          remoteContext = Some(jsonLdContext(body, base))
+          remoteContext
         case other => other
       }
 
@@ -411,33 +409,40 @@ object RdfConversions {
       opts
     }
 
-    private def jenaJsonLdContext(jsonLdContextUrl: URL, baseUrl: Option[String]): Try[util.Context] =
-      Try(CachingContext.parse(jsonLdContextUrl.toString))
-        .map(ctx =>
-          baseUrl
-            .map(bu => {
-              val c = ctx.clone()
-              c.put("@base", bu)
-              c
-            })
-            .getOrElse(ctx)
-        )
-        .map(jenaContext)
-
+    /*
+    * Only for testing, do not use this in the app
+    *
+    * todo, remove in the future
+    * */
     private[databus] def jsonLdContextUrl(data: Array[Byte]): Option[URL] =
+      jsonLdContext(data, None)
+        .map(_._1)
+        .toOption
+        .flatten
+
+    private def wrapWithBase(ctx: core.Context, baseUrl: Option[String]): core.Context =
+      baseUrl
+        .map(bu => {
+          val c = ctx.clone()
+          c.put("@base", bu)
+          c
+        })
+        .getOrElse(ctx)
+
+    private[databus] def jsonLdContext(data: Array[Byte], base: Option[String]): Try[(Option[URL], util.Context)] =
       Try(
-        JsonUtils.fromString(new String(data))
+        JsonUtils.fromInputStream(new ByteArrayInputStream(data))
       )
-        .map(j =>
-          Try(j.asInstanceOf[java.util.Map[String, Object]]).toOption)
-        .map(_.flatMap(c =>
-          Option(c.get(JsonLdConsts.CONTEXT))
-            .map(_.toString)
-            .flatMap(ctx =>
-              Try(new URL(ctx)) match {
-                case Failure(_) => None
-                case Success(uri) => Some(uri)
-              }))).toOption.flatten
+        .flatMap(j =>
+          Try(j.asInstanceOf[java.util.Map[String, Object]]))
+        .flatMap(c =>
+          Try(c.get(JsonLdConsts.CONTEXT)).flatMap {
+            case ctxs: String => Try(new URL(ctxs))
+              .flatMap(uri => Try(CachingContext.parse(uri.toString))
+                .map((Some(uri), _)))
+            case ctxo => Try(CachingContext.parse(ctxo))
+              .map(c => (None, c))
+          }).map(p => (p._1, jenaContext(wrapWithBase(p._2, base))))
 
     private def jenaContext(jsonLdCtx: core.Context) = {
       val context: util.Context = RIOT.getContext.copy()