diff --git a/build.sbt b/build.sbt index 9332727..fb08e23 100644 --- a/build.sbt +++ b/build.sbt @@ -1,14 +1,14 @@ import Keys._ import org.dbpedia.sbt.Codegen -scalaVersion := "2.12.6" +scalaVersion := "2.12.13" organization := "org.dbpedia" name := "gstore" version := "0.2.0-SNAPSHOT" val ScalatraVersion = "2.6.3" -val jenaVersion = "4.10.0" +val jenaVersion = "5.2.0" val jettyVersion = "9.4.9.v20180320" libraryDependencies ++= Seq( @@ -19,7 +19,6 @@ libraryDependencies ++= Seq( "org.apache.jena" % "apache-jena-libs" % jenaVersion, "org.apache.jena" % "jena-shacl" % jenaVersion, - "org.apache.jena" % "jena-jdbc-driver-remote" % jenaVersion, "com.openlink.virtuoso" % "virtjdbc4" % "x.x.x" from "http://download3.openlinksw.com/uda/virtuoso/jdbc/virtjdbc4.jar", "c3p0" % "c3p0" % "0.9.1.2", diff --git a/src/main/scala/com/apicatalog/jsonld/loader/JsonLdInit.scala b/src/main/scala/com/apicatalog/jsonld/loader/JsonLdInit.scala new file mode 100644 index 0000000..68ca282 --- /dev/null +++ b/src/main/scala/com/apicatalog/jsonld/loader/JsonLdInit.scala @@ -0,0 +1,12 @@ +package com.apicatalog.jsonld.loader + +import com.apicatalog.jsonld.http.media.MediaType + +object JsonLdInit { + def initLoader: DocumentLoader = { + val dl = HttpLoader.defaultInstance() + dl.fallbackContentType(MediaType.JSON_LD) + dl + } + +} diff --git a/src/main/scala/org/dbpedia/databus/ApiImpl.scala b/src/main/scala/org/dbpedia/databus/ApiImpl.scala index 10a203e..a5beed5 100644 --- a/src/main/scala/org/dbpedia/databus/ApiImpl.scala +++ b/src/main/scala/org/dbpedia/databus/ApiImpl.scala @@ -121,6 +121,7 @@ class ApiImpl(config: Config, batchSize: Int = 1000) extends DatabusApi { override def getGraphMapException404(e: Throwable)(request: javax.servlet.http.HttpServletRequest): Option[org.dbpedia.databus.swagger.model.OperationFailure] = getFileMapException404(e)(request) + override def shaclValidate(dataid: Array[Byte], shacl: Array[Byte])(request: HttpServletRequest): Try[String] = { val outLang = getLangFromAcceptHeader(request).flatMap(rdf).getOrElse(DefaultFormat) setResponseHeaders(Map("Content-Type" -> outLang.lang.getContentType.toHeaderString))(request) @@ -186,8 +187,9 @@ class ApiImpl(config: Config, batchSize: Int = 1000) extends DatabusApi { .collect { case c: RDFContentFormat => c } .getOrElse(DefaultFormat) val graphId = generateGraphId(prefix.getOrElse(getPrefix(request)), username, path) - wrapWithUnsupportedException(formatFromPath(path) - .flatMap(rdfExtractor), + wrapWithUnsupportedException( + formatFromPath(path) + .flatMap(rdfExtractor), path) .flatMap(format => { setResponseHeaders(Map("Content-Type" -> outFormat.lang.getContentType.toHeaderString))(request) @@ -326,7 +328,6 @@ object ApiImpl { defaultJsonldLocalhostContextLocation: Option[String]) - object Config { def default: Config = fromMapper(SystemMapper) diff --git a/src/main/scala/org/dbpedia/databus/CachingJsonldContext.scala b/src/main/scala/org/dbpedia/databus/CachingJsonldContexts.scala similarity index 51% rename from src/main/scala/org/dbpedia/databus/CachingJsonldContext.scala rename to src/main/scala/org/dbpedia/databus/CachingJsonldContexts.scala index aa0242d..5403789 100644 --- a/src/main/scala/org/dbpedia/databus/CachingJsonldContext.scala +++ b/src/main/scala/org/dbpedia/databus/CachingJsonldContexts.scala @@ -1,34 +1,27 @@ package org.dbpedia.databus -import java.util.concurrent.ConcurrentHashMap +import com.apicatalog.jsonld.context.cache.Cache +import com.apicatalog.jsonld.document.Document +import com.apicatalog.jsonld.loader.{DocumentLoader, DocumentLoaderOptions, JsonLdInit} -import com.github.jsonldjava.core.{Context, JsonLdOptions} -import org.dbpedia.databus.CachingJsonldContext.ApproxSizeStringKeyCache +import java.util.concurrent.ConcurrentHashMap +import org.dbpedia.databus.CachingJsonldContexts.ApproxSizeStringKeyCache +import java.net.URI import scala.collection.JavaConverters._ -class CachingJsonldContext(sizeLimit: Int, opts: JsonLdOptions) extends Context(opts) { - - private val cache = new ApproxSizeStringKeyCache[Context](sizeLimit) - - override def parse(ctx: Object): Context = - ctx match { - case s: String => - cache.get(s) - .map(c => super.parse(c)) - .getOrElse({ - val re = super.parse(ctx) - cache.put(s, re) - re - }) - case _ => super.parse(ctx) - } - def putInCache(contextUri: String, ctx: Context) = - cache.put(contextUri, ctx) +class CachingJsonldContexts(sizeLimit: Int) extends Cache[String, Document] { + + private val cache = new ApproxSizeStringKeyCache[Document](sizeLimit) + override def containsKey(key: String): Boolean = cache.get(key).isDefined + + override def get(key: String): Document = cache.get(key).orNull + + override def put(key: String, value: Document): Unit = cache.put(key, value) } -object CachingJsonldContext { +object CachingJsonldContexts { // not the most efficient impl, but should work for now :) class ApproxSizeStringKeyCache[T](sizeLimit: Int) { @@ -65,3 +58,19 @@ object CachingJsonldContext { } } + +class LRUDocumentCacheLoader(private val cache: Cache[String, Document], private val documentLoader: DocumentLoader) extends DocumentLoader { + + override def loadDocument(url: URI, options: DocumentLoaderOptions): Document = { + val k = url.toString + var result = cache.get(k) + if (result == null) { + result = documentLoader.loadDocument(url, options) + cache.put(k, result) + } + result + } + + def put(k: String, v: Document) = cache.put(k, v) + +} diff --git a/src/main/scala/org/dbpedia/databus/SparqlClient.scala b/src/main/scala/org/dbpedia/databus/SparqlClient.scala index 08c7117..101646a 100644 --- a/src/main/scala/org/dbpedia/databus/SparqlClient.scala +++ b/src/main/scala/org/dbpedia/databus/SparqlClient.scala @@ -1,19 +1,19 @@ package org.dbpedia.databus +import com.apicatalog.jsonld.deseralization.JsonLdToRdf +import com.apicatalog.jsonld.document.JsonDocument +import com.apicatalog.jsonld.lang.Keywords +import com.apicatalog.jsonld.JsonLdOptions +import com.apicatalog.jsonld.loader.{DocumentLoader, DocumentLoaderOptions, HttpLoader, JsonLdInit} import java.io.{ByteArrayInputStream, ByteArrayOutputStream} -import java.net.URL -import com.github.jsonldjava.core -import com.github.jsonldjava.core.{JsonLdConsts, JsonLdOptions} -import com.github.jsonldjava.utils.JsonUtils +import java.net.{URI, URL} import com.mchange.v2.c3p0.ComboPooledDataSource -import org.apache.jena.atlas.json.JsonString import org.apache.jena.graph.{Graph, Node} import org.apache.jena.iri.ViolationCodes import org.apache.jena.rdf.model.{Model, ModelFactory} -import org.apache.jena.riot.lang.LangJSONLD10 +import org.apache.jena.riot.lang.LangJSONLD11 import org.apache.jena.riot.system.{ErrorHandler, ErrorHandlerFactory, StreamRDFLib} -import org.apache.jena.riot.writer.JsonLD10Writer import org.apache.jena.riot.{Lang, RDFDataMgr, RDFFormat, RDFLanguages, RDFParser, RDFParserBuilder, RDFWriter, RDFWriterBuilder, RIOT} import org.apache.jena.shacl.{ShaclValidator, Shapes, ValidationReport} import org.apache.jena.sparql.util @@ -23,6 +23,8 @@ import org.slf4j.LoggerFactory import sttp.client3.{DigestAuthenticationBackend, HttpURLConnectionBackend, basicRequest} import sttp.model.Uri +import java.util.concurrent.ConcurrentHashMap +import java.util.logging.{Handler, Level, LogRecord, Logger} import scala.util.{Failure, Success, Try} @@ -225,9 +227,9 @@ object RdfConversions { } case object JSONLD extends RDFContentFormat { - override def lang: Lang = RDFLanguages.JSONLD10 + override def lang: Lang = RDFLanguages.JSONLD11 - override def format: RDFFormat = RDFFormat.JSONLD10_COMPACT_PRETTY + override def format: RDFFormat = RDFFormat.JSONLD11_PRETTY override def extensions: Set[String] = Set("jsonld") } @@ -320,24 +322,27 @@ object RdfConversions { } // NOTE! Not thread safe! - class RDFGraphSerialiser protected(inputFormat: RDFContentFormat, extractor: GraphBytesExtractor, data: Array[Byte], base: Option[String]) { + class RDFGraphSerialiser protected(val inputFormat: RDFContentFormat, extractor: GraphBytesExtractor, data: Array[Byte], base: Option[String]) { private var parsed: Try[(Model, List[Warning])] = Failure(null) def graph: Try[(Model, List[Warning])] = parsed.orElse { extractor.extractGraphBytes(data).flatMap { bytes => + val eh = ErrorHandlerWithWarnings.getNew + ErrorHandlerWithWarnings.registerWithInterceptor(eh) val re = Try { val model = ModelFactory.createDefaultModel() val parser = RDFParser.create() .source(new ByteArrayInputStream(bytes)) .base(base.orNull) .lang(inputFormat.lang) - val eh = newErrorHandlerWithWarnings parser.errorHandler(eh) modifyParser(parser) parser.parse(StreamRDFLib.graph(model.getGraph)) (model, eh.warningsList) } + // hack to intercept warnings in logs and return errors when needed + ErrorHandlerWithWarnings.deregisterFromInterceptor(eh) parsed = re re } @@ -350,7 +355,9 @@ object RdfConversions { protected def modifyParser(parser: RDFParserBuilder): Try[Unit] = Success() - protected def modifyWriter(writer: RDFWriterBuilder): Try[Unit] = Success() + protected def modifyWriter(writer: RDFWriterBuilder): Try[Unit] = Try( + writer.format(inputFormat.format) + ) } @@ -359,99 +366,54 @@ object RdfConversions { import JsonLDSerialiser._ - // NOTE! Not thread safe! - private var remoteContext: Option[Try[(Option[URL], util.Context)]] = None - - override def graph: Try[(Model, List[Warning])] = { - parseContext(data, base) - super.graph - } + private val ctx = jsonLdContext(base) override def modifyParser(parser: RDFParserBuilder) = - remoteContext.get.map(cs => - parser.context(cs._2)) + super.modifyParser(parser) + .map(_ => parser.context(ctx)) override def modifyWriter(writer: RDFWriterBuilder) = - remoteContext.get.map(ctx => { - writer.context(ctx._2) - ctx._1.foreach(url => writer.set(JsonLD10Writer.JSONLD_CONTEXT_SUBSTITUTION, new JsonString(url.toString))) - }) - - - private def parseContext(body: Array[Byte], base: Option[String]): Option[Try[(Option[URL], util.Context)]] = - remoteContext match { - case None | Some(Failure(_)) => - remoteContext = Some(jsonLdContext(body, base)) - remoteContext - case other => other - } + super.modifyWriter(writer) + .map(_ => writer.context(ctx)) } object JsonLDSerialiser { + private val DocumentCache = new LRUDocumentCacheLoader(new CachingJsonldContexts(32), JsonLdInit.initLoader) - private lazy val CachingContext = new CachingJsonldContext(30, defaultJsonLdOpts(null)) - - def preloadContextFromAnotherUri(ctxUri: String, downloadUri: String): Try[core.Context] = Try { - val ctx = CachingContext.parse(downloadUri) - CachingContext.putInCache(ctxUri, ctx) - ctx + def preloadContextFromAnotherUri(ctxUri: String, downloadUri: String): Try[Unit] = Try { + val dl: DocumentLoader = HttpLoader.defaultInstance() + val d = dl.loadDocument(new URI(downloadUri), new DocumentLoaderOptions()) + DocumentCache.put(ctxUri, d) } - def preloadContextFromAnotherHost(ctxUri: String, downloadHost: String): Try[core.Context] = Try { - val ctxUrl = new URL(ctxUri) - ctxUri.replace(ctxUrl.getHost, downloadHost) - }.flatMap(preloadContextFromAnotherUri(ctxUri, _)) + private[databus] def jsonLdContext(base: Option[String]): util.Context = + jenaContext(jsonldOpts(base)) - private def defaultJsonLdOpts(base: String) = { - val opts = new JsonLdOptions(base) - opts.useNamespaces = true - opts + private def jsonldOpts(base: Option[String]): JsonLdOptions = { + val o = new JsonLdOptions() + o.setDocumentLoader(DocumentCache) + o.setUriValidation(true) + base.flatMap(b => Try(new URI(b)).toOption) + .foreach(o.setBase) + o } - /* - * Only for testing, do not use this in the app - * - * todo, remove in the future - * */ - private[databus] def jsonLdContextUrl(data: Array[Byte]): Option[URL] = - jsonLdContext(data, None) - .map(_._1) - .toOption - .flatten - - private def wrapWithBase(ctx: core.Context, baseUrl: Option[String]): core.Context = - baseUrl - .map(bu => { - val c = ctx.clone() - c.put("@base", bu) - c - }) - .getOrElse(ctx) - - private[databus] def jsonLdContext(data: Array[Byte], base: Option[String]): Try[(Option[URL], util.Context)] = - Try( - JsonUtils.fromInputStream(new ByteArrayInputStream(data)) - ) - .flatMap(j => - Try(j.asInstanceOf[java.util.Map[String, Object]])) - .flatMap(c => - Try(c.get(JsonLdConsts.CONTEXT)).flatMap { - case ctxs: String => Try(new URL(ctxs)) - .flatMap(uri => Try(CachingContext.parse(uri.toString)) - .map((Some(uri), _))) - case ctxo => Try(CachingContext.parse(ctxo)) - .map(c => (None, c)) - }).map(p => (p._1, jenaContext(wrapWithBase(p._2, base)))) - - private def jenaContext(jsonLdCtx: core.Context) = { + private def jenaContext(jsonLdOpts: JsonLdOptions): util.Context = { val context: util.Context = RIOT.getContext.copy() - jsonLdCtx.putAll(jsonLdCtx.getPrefixes(true)) - context.put(JsonLD10Writer.JSONLD_CONTEXT, jsonLdCtx) - context.put(LangJSONLD10.JSONLD_CONTEXT, jsonLdCtx) + context.put(LangJSONLD11.JSONLD_OPTIONS, jsonLdOpts) context } + def contextUrl(data: Array[Byte]): Try[URL] = Try { + val d = JsonDocument.of(new ByteArrayInputStream(data)) + new URL( + d.getJsonContent.get() + .getValue(s"/${Keywords.CONTEXT}") + .toString.drop(1).dropRight(1) + ) + } + } def validateWithShacl(model: Model, shacl: Graph): Try[ValidationReport] = @@ -585,7 +547,7 @@ object RdfConversions { case class Warning(message: String) - private class ErrorHandlerWithWarnings extends ErrorHandler { + class ErrorHandlerWithWarnings extends ErrorHandler { private val defaultEH = ErrorHandlerFactory.getDefaultErrorHandler private var warnings: List[Warning] = List.empty @@ -646,8 +608,45 @@ object RdfConversions { def warningsList: List[Warning] = warnings } - private def newErrorHandlerWithWarnings: ErrorHandlerWithWarnings = - new ErrorHandlerWithWarnings + object ErrorHandlerWithWarnings { + + /* + This code (until the line with deregisterFromInterceptor) is needed because Titanium library ignores wrong IRIs and silently drops triples with them, + to address this we have to intercept warnings. Sadly there is no config option to make it throw exception instead. + + See: com.apicatalog.jsonld.deseralization.JsonLdToRdf method build(), line 118 + + @todo P.S. Very ugly, maybe you could find a better way to do it. + */ + protected[databus] val JsonLdToRdfLOGGER = Logger.getLogger(classOf[JsonLdToRdf].getName) + protected[databus] val WarningsInterceptor = new Handler { + + val JenaErrorHandlers = ConcurrentHashMap.newKeySet[ErrorHandlerWithWarnings](1) + + override def publish(record: LogRecord): Unit = record.getLevel match { + case Level.WARNING if record.getMessage.contains("Non well-formed subject") => + val m = s"Wrong IRI '${record.getParameters.headOption.map(_.toString).orNull}'. ${record.getSourceClassName} ${record.getSourceMethodName}" + JenaErrorHandlers.forEach(d => d.error(m, 0, 0)) + } + + override def flush(): Unit = {} + + override def close(): Unit = JsonLdToRdfLOGGER.removeHandler(this) + } + JsonLdToRdfLOGGER.addHandler(WarningsInterceptor) + + def registerWithInterceptor(eh: ErrorHandlerWithWarnings) = + ErrorHandlerWithWarnings.WarningsInterceptor + .JenaErrorHandlers.add(eh) + + def deregisterFromInterceptor(eh: ErrorHandlerWithWarnings) = + ErrorHandlerWithWarnings.WarningsInterceptor + .JenaErrorHandlers.remove(eh) + + def getNew: ErrorHandlerWithWarnings = + new ErrorHandlerWithWarnings + + } } diff --git a/src/test/resources/nest.jsonld b/src/test/resources/nest.jsonld new file mode 100644 index 0000000..68ce24b --- /dev/null +++ b/src/test/resources/nest.jsonld @@ -0,0 +1,179 @@ +{ + "@context": { + "bfo": "http://purl.obolibrary.org/obo/bfo.owl#", + "csvw": "http://www.w3.org/ns/csvw#", + "dbo": "http://dbpedia.org/ontology/", + "dc": "http://purl.org/dc/elements/1.1/", + "geo": "http://www.w3.org/2003/01/geo/wgs84_pos#", + "gn": "http://www.geonames.org/ontology#", + "dct": "http://purl.org/dc/terms/", + "obda": "https://w3id.org/obda/vocabulary#", + "oeo": "http://openenergyplatform.org/ontology/oeo/", + "owl": "http://www.w3.org/2002/07/owl#", + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "saref": "https://saref.etsi.org/core/", + "time": "http://www.w3.org/2006/time#", + "xml": "http://www.w3.org/XML/1998/namespace", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "title": "dct:title", + "description": "dct:description", + "distributions": "dc:distribution", + "subject": { + "@id": "dc:subject", + "@type": "@id" + }, + "spatial": "@nest", + "lat": "geo:lat", + "long": "geo:long", + "schema": "@nest", + "resolutionInMeters": { + "@id" : "dcat:spatialResolutionInMeters", + "@type": "xsd:decimal" + }, + "extent": "dct:spatial", + "temporal": "@nest", + "timeseries": "dct:temporal", + "start": "dcat:startDate", + "end": "dcat:endDate", + "resolution": { + "@id": "dcat:temporalResolution", + "@type": "xsd:duration" + }, + "isAbout": "saref:isAbout", + "valueReference": "csvw:columnReference", + "fields": "csvw:column" + }, + "@id": "https://databus.dbpedia.org/kurzum/mastr/bnetza-mastr/01.04.00", + "title": "RLI - OEMetadata - Metadata example table", + "description": "Example table used to illustrate the metadata structure and meaning.", + "distributions": [ + { + "@id": "https://databus.dbpedia.org/kurzum/mastr/bnetza-mastr/01.04.00#table.csv", + "name": "oep_metadata_table_example_v160", + "title": "RLI - OEMetadata - Metadata example table", + "format": "csv", + "type": "csv", + "profile": "tabular-data-resource", + "encoding": "UTF-8", + "description": "Example table used to illustrate the metadata structure and meaning.", + "subject": [ { "@id" : "https://openenergy-platform.org/ontology/oeo/OEO_00000150" } ], + "language": [ + "en-GB", + "de-DE", + "fr-FR" + ], + "keywords": [ + "example", + "template", + "test" + ], + "publicationDate": "2019-02-06", + "context": { + "homepage": "https://openenergy-platform.org/", + "documentation": "https://openenergy-platform.org/about/", + "sourceCode": "https://github.com/OpenEnergyPlatform", + "contact": "contact@example.com", + "grantNo": "01AB2345", + "fundingAgency": "Bundesministerium für Wirtschaft und Klimaschutz", + "fundingAgencyLogo": "https://commons.wikimedia.org/wiki/File:BMWi_Logo_2021.svg#/media/File:BMWi_Logo_2021.svg", + "publisherLogo": "https://reiner-lemoine-institut.de//wp-content/uploads/2015/09/rlilogo.png" + }, + "spatial": { + "lat": "52.433509", + "long": "13.535855", + "extent": { + "gn:name": "Europe", + "@id": "http://sws.geonames.org/6252001/" + }, + "resolutionInMeters": "3" + }, + "temporal": { + "referenceDate": "2016-01-01", + "timeseries": [ + { + "start": "2019-02-06T10:12:04+00:00", + "end": "2019-02-07T10:12:04+00:00", + "alignment": "left", + "aggregationType": "sum" + } + ] + }, + "sources": [ + { + "title": "IPCC Fifth Assessment Report", + "description": "Scientific climate change report by the UN", + "@id": "https://www.ipcc.ch/site/assets/uploads/2018/02/ipcc_wg3_ar5_full.pdf", + "licenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "@id": "https://opendatacommons.org/licenses/odbl/1-0/index.html", + "instruction": "You are free to share and change, but you must attribute, and share derivations under the same license. See https://tldrlegal.com/license/odc-open-database-license-(odbl) for further information.", + "attribution": "© Intergovernmental Panel on Climate Change 2014" + } + ] + } + ], + "licenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "@id": "https://opendatacommons.org/licenses/odbl/1-0/index.html", + "instruction": "You are free to share and change, but you must attribute, and share derivations under the same license. See https://tldrlegal.com/license/odc-open-database-license-(odbl) for further information.", + "attribution": "© Reiner Lemoine Institut" + } + ], + "contributors": [ + { + "title": "John Doe", + "email": "contact@example.com", + "date": "2016-06-16", + "object": "data and metadata", + "comment": "Fix typo in the title." + } + ], + "schema": { + "fields": [ + { + "name": "year", + "description": "Reference year for which the data were collected.", + "type": "geometry(Point, 4326)", + "isAbout": [ + { + "name": "wind energy converting unit", + "@id": "https://openenergy-platform.org/ontology/oeo/OEO_00000044" + } + ], + "valueReference": [ + { + "value": "onshore", + "name": "onshore wind farm", + "@id": "https://openenergy-platform.org/ontology/oeo/OEO_00000311" + } + ], + "unit": "MW" + } + ], + "primaryKey": ["id"], + "foreignKeys": [ + { + "fields": ["version"], + "reference": { + "resource": "schema.table", + "fields": ["version"] + } + } + ] + }, + "dialect": { + "delimiter": ",", + "decimalSeparator": "." + }, + "review": { + "@id": "https://www.example.com", + "badge": "Platinum" + } + } + ] +} \ No newline at end of file diff --git a/src/test/resources/space_in_iri.jsonld b/src/test/resources/space_in_iri.jsonld index a0c2f78..1c6cee9 100644 --- a/src/test/resources/space_in_iri.jsonld +++ b/src/test/resources/space_in_iri.jsonld @@ -24,7 +24,7 @@ "abstract": "Countries\n2023-09-18T12:22:14Z", "description": "Countries\n2023-09-18T12:22:14Z", "license": "https://dalicc.net/licenselibrary/Cc010Universal", - "wasDerivedFrom": "https://metadata.coypu.org/dataset/wikidata-distribution\nWikidata Query Service\nhttps://query.wikidata.org/", + "wasDerivedFrom": "https://metadata.coypu.org/dataset/wikidata-distributionWikidata Query Service", "distribution": [ { "@type": "Part", diff --git a/src/test/resources/test-relative.jsonld b/src/test/resources/test-relative.jsonld index 66b21d4..21adbb3 100644 --- a/src/test/resources/test-relative.jsonld +++ b/src/test/resources/test-relative.jsonld @@ -1,5 +1,5 @@ { - "@context" : "https://raw.githubusercontent.com/dbpedia/databus-moss/dev/devenv/context2.jsonld", + "@context" : "https://raw.githubusercontent.com/dbpedia/databus-moss-lodgeoss/a40b6d11271b4a81a8d6aa9ba18b5b0a1903dabb/devenv/context2.jsonld", "@id" : "#mod", "wasGeneratedBy" : { "@id" : "#layer", diff --git a/src/test/scala/org/dbpedia/databus/CacheTests.scala b/src/test/scala/org/dbpedia/databus/CacheTests.scala index 432e9c5..4b9a0f5 100644 --- a/src/test/scala/org/dbpedia/databus/CacheTests.scala +++ b/src/test/scala/org/dbpedia/databus/CacheTests.scala @@ -1,29 +1,20 @@ package org.dbpedia.databus -import org.apache.jena.sys.JenaSystem - import java.util.UUID -import org.dbpedia.databus.CachingJsonldContext.ApproxSizeStringKeyCache +import org.dbpedia.databus.CachingJsonldContexts.ApproxSizeStringKeyCache import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} class CacheTests extends FlatSpec with Matchers with BeforeAndAfter { - before { - JenaSystem.init() - } - after { - JenaSystem.shutdown() - } - "CacheKey" should "be sorted by time of creation" in { val caches = Seq( - new CachingJsonldContext.StringCacheKey("scsc", 0), - new CachingJsonldContext.StringCacheKey("scsc", -10), - new CachingJsonldContext.StringCacheKey("scsc", 100), - new CachingJsonldContext.StringCacheKey("zzzz", 0), - new CachingJsonldContext.StringCacheKey("aaaa", 0) + new CachingJsonldContexts.StringCacheKey("scsc", 0), + new CachingJsonldContexts.StringCacheKey("scsc", -10), + new CachingJsonldContexts.StringCacheKey("scsc", 100), + new CachingJsonldContexts.StringCacheKey("zzzz", 0), + new CachingJsonldContexts.StringCacheKey("aaaa", 0) ) caches.sorted.map(k => k.order) should contain theSameElementsInOrderAs (Seq(-10, 0, 0, 0, 100)) @@ -31,10 +22,10 @@ class CacheTests extends FlatSpec with Matchers with BeforeAndAfter { } "CacheKey" should "be equal with same string" in { - val re = new CachingJsonldContext.StringCacheKey("scsc", 0) == new CachingJsonldContext.StringCacheKey("scsc", -10) + val re = new CachingJsonldContexts.StringCacheKey("scsc", 0) == new CachingJsonldContexts.StringCacheKey("scsc", -10) re should be(true) - val re2 = new CachingJsonldContext.StringCacheKey("scsc", 0) == new CachingJsonldContext.StringCacheKey("aaaa", 0) + val re2 = new CachingJsonldContexts.StringCacheKey("scsc", 0) == new CachingJsonldContexts.StringCacheKey("aaaa", 0) re2 should be(false) } diff --git a/src/test/scala/org/dbpedia/databus/DatabusScalatraTest.scala b/src/test/scala/org/dbpedia/databus/DatabusScalatraTest.scala index ee07453..29d5dc8 100644 --- a/src/test/scala/org/dbpedia/databus/DatabusScalatraTest.scala +++ b/src/test/scala/org/dbpedia/databus/DatabusScalatraTest.scala @@ -1,15 +1,11 @@ package org.dbpedia.databus - -import com.github.jsonldjava.core.JsonLdConsts -import com.github.jsonldjava.utils.JsonUtils -import org.apache.jena.iri.ViolationCodes - import java.io.ByteArrayInputStream import java.nio.file.{Files, Paths} import org.apache.jena.rdf.model.ModelFactory import org.apache.jena.riot.{Lang, RDFDataMgr} import org.dbpedia.databus.ApiImpl.Config +import org.dbpedia.databus.RdfConversions.{ErrorHandlerWithWarnings, JsonLDSerialiser} import org.dbpedia.databus.swagger.DatabusSwagger import org.dbpedia.databus.swagger.api.DefaultApi import org.scalatest.BeforeAndAfter @@ -85,6 +81,8 @@ class DatabusScalatraTest extends ScalatraFlatSpec with BeforeAndAfter { status should equal(200) } + (ErrorHandlerWithWarnings.WarningsInterceptor.JenaErrorHandlers.size() < 3) should equal(true) + get("/databus/document/history?repo=kuckuck&limit=2") { status should equal(200) body should include("author_name") @@ -95,9 +93,7 @@ class DatabusScalatraTest extends ScalatraFlatSpec with BeforeAndAfter { get("/databus/graph/read?repo=kuckuck&path=pa/fl.jsonld") { status should equal(200) - val respCtx = RdfConversions.JsonLDSerialiser.jsonLdContextUrl(bodyBytes) - respCtx should equal(RdfConversions.JsonLDSerialiser.jsonLdContextUrl(bytes)) - respCtx.get.toString.nonEmpty should equal(true) + body.contains("This a short abstract for the dataset. Since this") should be(true) } get("/databus/document/read?repo=kuckuck&path=pa/fl.jsonld") { @@ -111,12 +107,10 @@ class DatabusScalatraTest extends ScalatraFlatSpec with BeforeAndAfter { val file = "group_with_localhostcontext.jsonld" val bytes = Files.readAllBytes(Paths.get(getClass.getClassLoader.getResource(file).getFile)) - val localhostContext = JsonUtils.fromString(new String(bytes)) - .asInstanceOf[java.util.Map[String, Object]] - .get(JsonLdConsts.CONTEXT).toString + val localhostContext = JsonLDSerialiser.contextUrl(bytes).get.toString val vfile = "group.jsonld" val vbytes = Files.readAllBytes(Paths.get(getClass.getClassLoader.getResource(vfile).getFile)) - val validContext = RdfConversions.JsonLDSerialiser.jsonLdContextUrl(vbytes).get.toString + val validContext = JsonLDSerialiser.contextUrl(vbytes).get.toString RdfConversions.JsonLDSerialiser.preloadContextFromAnotherUri(localhostContext, validContext) @@ -124,6 +118,11 @@ class DatabusScalatraTest extends ScalatraFlatSpec with BeforeAndAfter { status should equal(200) } + get(s"/databus/graph/read?repo=kuckuck&path=pa/$file") { + status should equal(200) + body.contains("This a short abstract for the dataset. Since this") should be(true) + } + } "File save" should "save and return original jsonld with json fields" in { @@ -139,7 +138,7 @@ class DatabusScalatraTest extends ScalatraFlatSpec with BeforeAndAfter { } get(s"/databus/graph/read?repo=kuckuck&path=pa/$file") { status should equal(200) - bodyBytes should not equal(bytes) + bodyBytes should not equal (bytes) } } @@ -174,23 +173,39 @@ class DatabusScalatraTest extends ScalatraFlatSpec with BeforeAndAfter { get("/databus/graph/read?repo=kuckuck&path=pa/rel_test.jsonld") { status should equal(200) - val respCtx = RdfConversions.JsonLDSerialiser.jsonLdContextUrl(bodyBytes) - respCtx should equal(RdfConversions.JsonLDSerialiser.jsonLdContextUrl(bytes)) - respCtx.get.toString.nonEmpty should equal(true) - body.contains(" \"generated\" : \"#mod\",") should equal(true) + body.contains("#generated\"") should equal(true) } } "File save" should "report problems in input" in { - val file = "space_in_iri.jsonld" val bytes = Files.readAllBytes(Paths.get(getClass.getClassLoader.getResource(file).getFile)) post("/databus/document/save?repo=kuckuck&path=pa/syntax_err.jsonld", bytes) { - (status >= 400) should equal(true) - response.body.contains("Spaces are not legal in URIs/IRIs") should equal(true) + status should equal(400) + body should include("Wrong IRI") + body should include("https://metadata.coypu.org/dataset/wikidata-distributionWikidata Query Service") + } + + (ErrorHandlerWithWarnings.WarningsInterceptor.JenaErrorHandlers.size() < 3) should equal(true) + + } + + "File save" should "work with @nest keyword" in { + + val file = "nest.jsonld" + val bytes = Files.readAllBytes(Paths.get(getClass.getClassLoader.getResource(file).getFile)) + + post("/databus/document/save?repo=kuckuck&path=pa/nest.jsonld", bytes) { + status should equal(200) + } + + get("/databus/graph/read?repo=kuckuck&path=pa/nest.jsonld") { + status should equal(200) + body.contains("\"dct:description\": \"Example table used to illustrate") should equal(true) + body.contains("\"dct:spatial\": {\n \"@id\": \"http://sws.geonames.org/6252001/\"") should equal(true) } } @@ -204,8 +219,8 @@ class DatabusScalatraTest extends ScalatraFlatSpec with BeforeAndAfter { post("/databus/shacl/validate", Map.empty, Map("shacl" -> shacl, "graph" -> bytes)) { status should equal(400) - body should include("Bad IRI") - body should include(s"Spaces are not legal") + body should include("Wrong IRI") + body should include("https://metadata.coypu.org/dataset/wikidata-distributionWikidata Query Service") } } @@ -246,9 +261,9 @@ class DatabusScalatraTest extends ScalatraFlatSpec with BeforeAndAfter { val shacl = Paths.get(getClass.getClassLoader.getResource(sha).getFile).toFile post("/databus/shacl/validate", Map.empty, Map("shacl" -> shacl, "graph" -> bytes)) { - status should equal(400) - body should include("Bad IRI") - body should include(s"Code: ${ViolationCodes.CONTROL_CHARACTER}") + (status == 400) should equal(true) + body should include("Wrong IRI") + body should include("hTtps://metadata.coypu.org/dataset/wikidata-distribution\\nWikidataQueryService\\n") } } @@ -271,12 +286,12 @@ class DatabusScalatraTest extends ScalatraFlatSpec with BeforeAndAfter { post("/databus/shacl/validate", Map.empty, Map("shacl" -> shacl, "graph" -> bytes)) { status should equal(200) - body should include("\"sh:conforms\" : true") + body should include("\"sh:conforms\": {\n \"@value\": \"true\",") } post("/databus/shacl/validate", Map.empty, Map("shacl" -> shacl, "graph" -> err)) { status should equal(200) - body should include("\"sh:conforms\" : false") + body should include("\"sh:conforms\": {\n \"@value\": \"false\",") } } @@ -297,11 +312,14 @@ class DatabusScalatraTest extends ScalatraFlatSpec with BeforeAndAfter { val model = ModelFactory.createDefaultModel() val dataStream = new ByteArrayInputStream(version) - RDFDataMgr.read(model, dataStream, Lang.JSONLD10) + RDFDataMgr.read(model, dataStream, Lang.JSONLD11) val tr = Tractate.extract(model.getGraph, TractateV1.Version) body should equal(tr.get.stringForSigning) } } + "Number of handlers" should "be close to 0" in { + (ErrorHandlerWithWarnings.WarningsInterceptor.JenaErrorHandlers.size() < 3) should equal(true) + } } diff --git a/src/test/scala/org/dbpedia/databus/TractateTest.scala b/src/test/scala/org/dbpedia/databus/TractateTest.scala index 25844f6..99c6a22 100644 --- a/src/test/scala/org/dbpedia/databus/TractateTest.scala +++ b/src/test/scala/org/dbpedia/databus/TractateTest.scala @@ -1,5 +1,7 @@ package org.dbpedia.databus +import com.apicatalog.jsonld.loader.JsonLdInit + import java.io.ByteArrayInputStream import java.nio.file.{Files, Paths} import org.apache.jena.rdf.model.ModelFactory @@ -10,6 +12,7 @@ import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} class TractateTest extends FlatSpec with Matchers with BeforeAndAfter { before { + JsonLdInit.initLoader JenaSystem.init() } @@ -23,7 +26,7 @@ class TractateTest extends FlatSpec with Matchers with BeforeAndAfter { val bytes = Files.readAllBytes(Paths.get(getClass.getClassLoader.getResource(file).getFile)) val model = ModelFactory.createDefaultModel() val dataStream = new ByteArrayInputStream(bytes) - RDFDataMgr.read(model, dataStream, Lang.JSONLD10) + RDFDataMgr.read(model, dataStream, Lang.JSONLD11) val t = Tractate.extract(model.getGraph, TractateV1.Version) val expected = """Databus Tractate V1 diff --git a/src/test/scala/org/dbpedia/databus/VirtuosoQueriesTest.scala b/src/test/scala/org/dbpedia/databus/VirtuosoQueriesTest.scala index e6358bd..60a2bfa 100644 --- a/src/test/scala/org/dbpedia/databus/VirtuosoQueriesTest.scala +++ b/src/test/scala/org/dbpedia/databus/VirtuosoQueriesTest.scala @@ -1,5 +1,7 @@ package org.dbpedia.databus +import com.apicatalog.jsonld.loader.JsonLdInit + import java.io.ByteArrayInputStream import java.nio.file.{Files, Paths} import org.apache.jena.rdf.model.ModelFactory @@ -11,6 +13,7 @@ class VirtuosoQueriesTest extends FlatSpec with Matchers with BeforeAndAfter { import collection.JavaConverters._ before { + JsonLdInit.initLoader JenaSystem.init() } after { @@ -22,7 +25,7 @@ class VirtuosoQueriesTest extends FlatSpec with Matchers with BeforeAndAfter { val bytes = Files.readAllBytes(Paths.get(getClass.getClassLoader.getResource(file).getFile)) val model = ModelFactory.createDefaultModel() val dataStream = new ByteArrayInputStream(bytes) - RDFDataMgr.read(model, dataStream, Lang.JSONLD10) + RDFDataMgr.read(model, dataStream, Lang.JSONLD11) val bld = RdfConversions.makeInsertSparqlQuery(model.getGraph.find().asScala.toSeq, "http://randomGraphId") @@ -35,7 +38,7 @@ class VirtuosoQueriesTest extends FlatSpec with Matchers with BeforeAndAfter { val bytes = Files.readAllBytes(Paths.get(getClass.getClassLoader.getResource(file).getFile)) val model = ModelFactory.createDefaultModel() val dataStream = new ByteArrayInputStream(bytes) - RDFDataMgr.read(model, dataStream, Lang.JSONLD10) + RDFDataMgr.read(model, dataStream, Lang.JSONLD11) val bld = RdfConversions.makeInsertSparqlQuery(model.getGraph.find().asScala.toSeq, "http://randomGraphId")