From fcfca4f89a844b7ef2b7a113eca5596762c3f170 Mon Sep 17 00:00:00 2001 From: Alexander Bigerl Date: Mon, 10 Jul 2023 12:35:58 +0200 Subject: [PATCH] starts --- example-suite.yml | 13 ++ .../org/aksw/iguana/cc/config/CONSTANTS.java | 34 --- .../cc/config/elements/MetricConfig.java | 42 ---- .../iguana/cc/controller/MainController.java | 17 +- .../cc/lang/AbstractLanguageProcessor.java | 63 ----- .../iguana/cc/lang/LanguageProcessor.java | 90 ++++---- .../org/aksw/iguana/cc/lang/QueryWrapper.java | 31 --- .../cc/lang/impl/RDFLanguageProcessor.java | 110 --------- .../cc/lang/impl/SPARQLLanguageProcessor.java | 189 --------------- .../SaxSparqlJsonResultCountingParser.java | 216 +++++++++++------- .../lang/impl/ThrowawayLanguageProcessor.java | 35 --- .../iguana/cc/lang2/LanguageProcessor.java | 57 ----- .../SaxSparqlJsonResultCountingParser.java | 174 -------------- .../iguana/cc/model/QueryExecutionStats.java | 62 ----- .../iguana/cc/query/handler/QueryHandler.java | 10 +- .../iguana/cc/suite/IguanaSuiteParser.java | 136 ++++++----- .../java/org/aksw/iguana/cc/suite/Suite.java | 13 +- .../cc/worker/ResponseBodyProcessor.java | 4 +- .../cc/worker/impl/SPARQLProtocolWorker.java | 50 ++-- .../cc/lang/MockCloseableHttpResponse.java | 49 ---- .../cc/lang/RDFLanguageProcessorTest.java | 65 ------ .../cc/lang/SPARQLLanguageProcessorTest.java | 144 ------------ .../org/aksw/iguana/cc/utils/ServerMock.java | 48 ---- 23 files changed, 324 insertions(+), 1328 deletions(-) delete mode 100644 src/main/java/org/aksw/iguana/cc/config/CONSTANTS.java delete mode 100644 src/main/java/org/aksw/iguana/cc/config/elements/MetricConfig.java delete mode 100644 src/main/java/org/aksw/iguana/cc/lang/AbstractLanguageProcessor.java delete mode 100644 src/main/java/org/aksw/iguana/cc/lang/QueryWrapper.java delete mode 100644 src/main/java/org/aksw/iguana/cc/lang/impl/RDFLanguageProcessor.java delete mode 100644 src/main/java/org/aksw/iguana/cc/lang/impl/SPARQLLanguageProcessor.java delete mode 100644 src/main/java/org/aksw/iguana/cc/lang/impl/ThrowawayLanguageProcessor.java delete mode 100644 src/main/java/org/aksw/iguana/cc/lang2/LanguageProcessor.java delete mode 100644 src/main/java/org/aksw/iguana/cc/lang2/impl/SaxSparqlJsonResultCountingParser.java delete mode 100644 src/main/java/org/aksw/iguana/cc/model/QueryExecutionStats.java delete mode 100644 src/test/java/org/aksw/iguana/cc/lang/MockCloseableHttpResponse.java delete mode 100644 src/test/java/org/aksw/iguana/cc/lang/RDFLanguageProcessorTest.java delete mode 100644 src/test/java/org/aksw/iguana/cc/lang/SPARQLLanguageProcessorTest.java delete mode 100644 src/test/java/org/aksw/iguana/cc/utils/ServerMock.java diff --git a/example-suite.yml b/example-suite.yml index ef925a0c..d9515337 100644 --- a/example-suite.yml +++ b/example-suite.yml @@ -7,6 +7,7 @@ connections: user: "dba" password: "dba" endpoint: "http://localhost:8890/sparql" + dataset: DatasetName - name: "Virtuoso6" user: "dba" password: "dba" @@ -28,6 +29,7 @@ tasks: queries: path: "/home/bigerl/IdeaProjects/IGUANA/LICENSE" timeout: 0.02s + connection: Virtuoso7 completionTarget: duration: 1000s workers: @@ -36,10 +38,15 @@ tasks: queries: path: "/home/bigerl/IdeaProjects/IGUANA/LICENSE" timeout: 180s + connection: Virtuoso7 completionTarget: duration: 1000s + requestType: get query - number: 4 type: "SPARQLProtocolWorker" + connection: Virtuoso7 + completionTarget: + duration: 1000s queries: path: "/home/bigerl/IdeaProjects/IGUANA/LICENSE" timeout: 100s @@ -47,13 +54,19 @@ tasks: - type: stresstest workers: - type: "SPARQLProtocolWorker" + connection: Virtuoso7 number: 16 + requestType: get query queries: path: "/home/bigerl/IdeaProjects/IGUANA/LICENSE" timeout: 180s completionTarget: duration: 1000s - number: 4 + requestType: get query + connection: Virtuoso7 + completionTarget: + duration: 1000s type: "SPARQLProtocolWorker" queries: path: "/home/bigerl/IdeaProjects/IGUANA/LICENSE" diff --git a/src/main/java/org/aksw/iguana/cc/config/CONSTANTS.java b/src/main/java/org/aksw/iguana/cc/config/CONSTANTS.java deleted file mode 100644 index 4b3c7ebc..00000000 --- a/src/main/java/org/aksw/iguana/cc/config/CONSTANTS.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.aksw.iguana.cc.config; - -/** - * Constants used only by the Core controller - * - * @author f.conrads - * - */ -public class CONSTANTS { - - /** - * The key to set the workerID in the Extra Meta properties - * and the properties name in the final results to get the workerID - */ - public static final String WORKER_ID_KEY = "workerID"; - - /** - * The key to set the workerType in the Extra Meta properties - * and the properties name in the final results to get the workerType - */ - public static final String WORKER_TYPE_KEY = "workerType"; - - /** - * The key to get the timeLimit parameter. - * be aware that timeLimit can be null. - */ - public static final String TIME_LIMIT = "timeLimit"; - - - public static final String NO_OF_QUERY_MIXES = "numberOfQueryMixes"; - - - public static final String WORKER_TIMEOUT_MS = "timeOutMS"; -} diff --git a/src/main/java/org/aksw/iguana/cc/config/elements/MetricConfig.java b/src/main/java/org/aksw/iguana/cc/config/elements/MetricConfig.java deleted file mode 100644 index ac504ca1..00000000 --- a/src/main/java/org/aksw/iguana/cc/config/elements/MetricConfig.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.aksw.iguana.cc.config.elements; - -import com.fasterxml.jackson.annotation.JsonProperty; -import org.aksw.iguana.commons.factory.TypedFactory; -import org.aksw.iguana.rp.metrics.Metric; - -import java.util.HashMap; -import java.util.Map; - -/** - * Metric Config class - */ -public class MetricConfig { - - @JsonProperty(required = true) - private String className; - - @JsonProperty() - private Map configuration = new HashMap<>(); - - - public String getClassName() { - return className; - } - - public void setClassName(String className) { - this.className = className; - } - - public Map getConfiguration() { - return configuration; - } - - public void setConfiguration(Map configuration) { - this.configuration = configuration; - } - - public Metric createMetric() { - TypedFactory factory = new TypedFactory<>(); - return factory.create(className, configuration); - } -} diff --git a/src/main/java/org/aksw/iguana/cc/controller/MainController.java b/src/main/java/org/aksw/iguana/cc/controller/MainController.java index ef2f2c7f..3a955ebc 100644 --- a/src/main/java/org/aksw/iguana/cc/controller/MainController.java +++ b/src/main/java/org/aksw/iguana/cc/controller/MainController.java @@ -1,8 +1,6 @@ package org.aksw.iguana.cc.controller; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; +import com.beust.jcommander.*; import org.aksw.iguana.cc.suite.IguanaSuiteParser; import org.aksw.iguana.cc.suite.Suite; import org.slf4j.Logger; @@ -17,15 +15,24 @@ */ public class MainController { + public static class Args { - @Parameter(description = "Suite file describing the configuration of the experiment.") - private Path suitePath; + public class PathConverter implements IStringConverter { + @Override + public Path convert(String value) { + return Path.of(value); + } + } + @Parameter(names = {"--ignore-schema", "-is"}, description = "Do not check the schema before parsing the suite file.") private boolean ignoreShema = false; @Parameter(names = "--help", help = true) private boolean help; + + @Parameter(description = "suite file {yml,yaml,json}", arity = 1, required = true, converter = PathConverter.class) + private Path suitePath; } diff --git a/src/main/java/org/aksw/iguana/cc/lang/AbstractLanguageProcessor.java b/src/main/java/org/aksw/iguana/cc/lang/AbstractLanguageProcessor.java deleted file mode 100644 index b31da62b..00000000 --- a/src/main/java/org/aksw/iguana/cc/lang/AbstractLanguageProcessor.java +++ /dev/null @@ -1,63 +0,0 @@ -package org.aksw.iguana.cc.lang; - -import org.aksw.iguana.commons.constants.COMMON; -import org.aksw.iguana.commons.streams.Streams; -import org.aksw.iguana.rp.vocab.Vocab; -import org.apache.http.Header; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.jena.rdf.model.Model; -import org.apache.jena.rdf.model.ModelFactory; -import org.apache.jena.rdf.model.Resource; -import org.apache.jena.rdf.model.ResourceFactory; -import org.apache.jena.vocabulary.RDF; -import org.apache.jena.vocabulary.RDFS; -import org.json.simple.parser.ParseException; -import org.xml.sax.SAXException; - -import javax.xml.parsers.ParserConfigurationException; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.time.Instant; -import java.util.List; -import java.util.concurrent.TimeoutException; - -public abstract class AbstractLanguageProcessor implements LanguageProcessor { - - @Override - public String getQueryPrefix() { - return "query"; - } - - @Override - public Model generateTripleStats(List queries, String resourcePrefix, String taskID) { - Model model = ModelFactory.createDefaultModel(); - for(QueryWrapper wrappedQuery : queries) { - Resource subject = ResourceFactory.createResource(COMMON.RES_BASE_URI + resourcePrefix + "/" + wrappedQuery.getId()); - model.add(subject, RDF.type, Vocab.queryClass); - model.add(subject, Vocab.rdfsID, wrappedQuery.getId()); - model.add(subject, RDFS.label, wrappedQuery.getQuery().toString()); - } - return model; - } - - @Override - public Long getResultSize(CloseableHttpResponse response) throws ParserConfigurationException, SAXException, ParseException, IOException { - return response.getEntity().getContentLength(); - } - - @Override - public Long getResultSize(Header contentTypeHeader, ByteArrayOutputStream content, long contentLength) throws ParserConfigurationException, SAXException, ParseException, IOException { - return Long.valueOf(content.size()); - } - - @Override - public long readResponse(InputStream inputStream, ByteArrayOutputStream responseBody) throws IOException, TimeoutException { - return Streams.inputStream2ByteArrayOutputStream(inputStream, responseBody); - } - - //@Override - public long readResponse(InputStream inputStream, Instant startTime, Double timeOut, ByteArrayOutputStream responseBody) throws IOException, TimeoutException { - return Streams.inputStream2ByteArrayOutputStream(inputStream, startTime, timeOut, responseBody); - } -} diff --git a/src/main/java/org/aksw/iguana/cc/lang/LanguageProcessor.java b/src/main/java/org/aksw/iguana/cc/lang/LanguageProcessor.java index 211d50aa..bb07f680 100644 --- a/src/main/java/org/aksw/iguana/cc/lang/LanguageProcessor.java +++ b/src/main/java/org/aksw/iguana/cc/lang/LanguageProcessor.java @@ -1,55 +1,57 @@ package org.aksw.iguana.cc.lang; -import org.apache.http.Header; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.jena.rdf.model.Model; -import org.json.simple.parser.ParseException; -import org.xml.sax.SAXException; - -import javax.xml.parsers.ParserConfigurationException; -import java.io.ByteArrayOutputStream; -import java.io.IOException; import java.io.InputStream; -import java.time.Instant; -import java.util.List; -import java.util.concurrent.TimeoutException; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.lang.reflect.InvocationTargetException; +import java.util.HashMap; +import java.util.Map; +import java.util.ServiceLoader; /** - * Language Processor tells how to handle Http responses as well as how to analyze queries and generate stats. + * Interface for abstract language processors that work on InputStreams. */ -public interface LanguageProcessor { - - /** - * Returns the prefix used for the queries (e.g. sparql, query or document) - * @return - */ - String getQueryPrefix(); - - /** - * Method to generate Triple Statistics for provided queries - * - * - * @param taskID - * @return Model with the triples to add to the results - */ - Model generateTripleStats(List queries, String resourcePrefix, String taskID); - - +public abstract class LanguageProcessor { /** - * Gets the result size of a given HTTP response - * - * @param response - * @return - * @throws ParserConfigurationException - * @throws SAXException - * @throws ParseException - * @throws IOException + * Provides the content type that a LanguageProcessor consumes. */ - Long getResultSize(CloseableHttpResponse response) throws ParserConfigurationException, SAXException, ParseException, IOException; - - Long getResultSize(Header contentTypeHeader, ByteArrayOutputStream content, long contentLength) throws ParserConfigurationException, SAXException, ParseException, IOException; - + @Retention(RetentionPolicy.RUNTIME) + @Target(ElementType.TYPE) + public @interface ContentType { + String value(); + } + + public interface LanguageProcessingData { + Class processor(); + } + + public abstract LanguageProcessingData process(InputStream inputStream); + + final private static Map> processors = new HashMap<>(); + + static { + for (LanguageProcessor processor : ServiceLoader.load(LanguageProcessor.class)) { + LanguageProcessor.ContentType contentType = processor.getClass().getAnnotation(LanguageProcessor.ContentType.class); + if (contentType != null) { + processors.put(contentType.value(), processor.getClass()); + } + } + } + + public static LanguageProcessor getInstance(String contentType) { + Class processorClass = processors.get(contentType); + if (processorClass != null) { + try { + return processorClass.getDeclaredConstructor().newInstance(); + } catch (InstantiationException | IllegalAccessException | InvocationTargetException | + NoSuchMethodException e) { + throw new RuntimeException(e); + } + } + throw new IllegalArgumentException("No LanguageProcessor for ContentType " + contentType); + } - long readResponse(InputStream inputStream, ByteArrayOutputStream responseBody) throws IOException, TimeoutException; } diff --git a/src/main/java/org/aksw/iguana/cc/lang/QueryWrapper.java b/src/main/java/org/aksw/iguana/cc/lang/QueryWrapper.java deleted file mode 100644 index 080a8791..00000000 --- a/src/main/java/org/aksw/iguana/cc/lang/QueryWrapper.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.aksw.iguana.cc.lang; - -/** - * Util class to wrap a Query of what ever class it may be and it's id - */ -public class QueryWrapper { - - private Object query; - private String id; - - public QueryWrapper(Object query, String id){ - this.query=query; - this.id=id; - } - - public Object getQuery() { - return query; - } - - public void setQuery(Object query) { - this.query = query; - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } -} diff --git a/src/main/java/org/aksw/iguana/cc/lang/impl/RDFLanguageProcessor.java b/src/main/java/org/aksw/iguana/cc/lang/impl/RDFLanguageProcessor.java deleted file mode 100644 index a69e5e67..00000000 --- a/src/main/java/org/aksw/iguana/cc/lang/impl/RDFLanguageProcessor.java +++ /dev/null @@ -1,110 +0,0 @@ -package org.aksw.iguana.cc.lang.impl; - -import org.aksw.iguana.cc.lang.AbstractLanguageProcessor; -import org.aksw.iguana.cc.lang.LanguageProcessor; -import org.aksw.iguana.cc.lang.QueryWrapper; -import org.aksw.iguana.commons.annotation.Shorthand; -import org.aksw.iguana.commons.constants.COMMON; -import org.aksw.iguana.rp.vocab.Vocab; -import org.apache.http.Header; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.jena.rdf.model.Model; -import org.apache.jena.rdf.model.ModelFactory; -import org.apache.jena.rdf.model.Resource; -import org.apache.jena.rdf.model.ResourceFactory; -import org.apache.jena.riot.Lang; -import org.apache.jena.vocabulary.RDF; -import org.apache.jena.vocabulary.RDFS; -import org.json.simple.parser.ParseException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.xml.sax.SAXException; - -import javax.xml.parsers.ParserConfigurationException; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.lang.reflect.Field; -import java.util.List; - -/** - * Language for everything which returns RDF in any rdf format. - * - * Counts triples returned as ResultSize - */ -@Shorthand("lang.RDF") -public class RDFLanguageProcessor extends AbstractLanguageProcessor implements LanguageProcessor { - - private static Logger LOGGER = LoggerFactory.getLogger(RDFLanguageProcessor.class); - protected String queryPrefix="document"; - - @Override - public String getQueryPrefix() { - return this.queryPrefix; - } - - @Override - public Model generateTripleStats(List queries, String resourcePrefix, String taskID) { - Model model = ModelFactory.createDefaultModel(); - for(QueryWrapper wrappedQuery : queries) { - Resource subject = ResourceFactory.createResource(COMMON.RES_BASE_URI + resourcePrefix + "/" + wrappedQuery.getId()); - model.add(subject, RDF.type, Vocab.queryClass); - model.add(subject, Vocab.rdfsID, wrappedQuery.getId().replace(queryPrefix, "").replace("sparql", "")); - model.add(subject, RDFS.label, wrappedQuery.getQuery().toString()); - } - return model; - } - - @Override - public Long getResultSize(CloseableHttpResponse response) throws ParserConfigurationException, SAXException, ParseException, IOException { - Model m; - try { - Header contentTypeHeader = response.getEntity().getContentType(); - InputStream inputStream = response.getEntity().getContent(); - m = getModel(contentTypeHeader, inputStream); - } catch (IllegalAccessException e) { - LOGGER.error("Could not read response as model", e); - return -1L; - } - return countSize(m); - } - - @Override - public Long getResultSize(Header contentTypeHeader, ByteArrayOutputStream content, long contentLength) throws IOException { - Model m; - try { - //TODO BBAIS - InputStream inputStream = new ByteArrayInputStream(content.toByteArray()); - m = getModel(contentTypeHeader, inputStream); - } catch (IllegalAccessException e) { - LOGGER.error("Could not read response as model", e); - return -1L; - } - return countSize(m); - } - - protected Long countSize(Model m) { - return m.size(); - } - - private Model getModel(Header contentTypeHeader, InputStream contentInputStream) throws IOException, IllegalAccessException { - Model m = ModelFactory.createDefaultModel(); - Lang lang = null; - // get actual content type - String contentType = contentTypeHeader.getValue(); - // use reflect to iterate over all static Lang fields of the Lang.class - for (Field langField : Lang.class.getFields()) { - //create the Language of the field - Lang susLang = (Lang) langField.get(Lang.class); - //if they are the same we have our language - if (contentType.equals(susLang.getContentType().getContentTypeStr())) { - lang = susLang; - break; - } - } - if (lang != null) - m.read(contentInputStream, null, lang.getName()); - return m; - } -} diff --git a/src/main/java/org/aksw/iguana/cc/lang/impl/SPARQLLanguageProcessor.java b/src/main/java/org/aksw/iguana/cc/lang/impl/SPARQLLanguageProcessor.java deleted file mode 100644 index d1710485..00000000 --- a/src/main/java/org/aksw/iguana/cc/lang/impl/SPARQLLanguageProcessor.java +++ /dev/null @@ -1,189 +0,0 @@ -package org.aksw.iguana.cc.lang.impl; - -import org.aksw.iguana.cc.lang.AbstractLanguageProcessor; -import org.aksw.iguana.cc.lang.LanguageProcessor; -import org.aksw.iguana.cc.lang.QueryWrapper; -import org.aksw.iguana.cc.utils.SPARQLQueryStatistics; -import org.aksw.iguana.commons.annotation.Shorthand; -import org.aksw.iguana.commons.constants.COMMON; -import org.aksw.iguana.rp.vocab.Vocab; -import org.apache.commons.lang.StringUtils; -import org.apache.http.Header; -import org.apache.http.HeaderElement; -import org.apache.http.HttpEntity; -import org.apache.http.NameValuePair; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.jena.ext.com.google.common.hash.HashCode; -import org.apache.jena.ext.com.google.common.hash.Hashing; -import org.apache.jena.ext.com.google.common.io.BaseEncoding; -import org.apache.jena.query.Query; -import org.apache.jena.query.QueryFactory; -import org.apache.jena.rdf.model.Model; -import org.apache.jena.rdf.model.ModelFactory; -import org.apache.jena.rdf.model.Resource; -import org.apache.jena.rdf.model.ResourceFactory; -import org.apache.jena.vocabulary.OWL; -import org.apache.jena.vocabulary.RDF; -import org.apache.jena.vocabulary.RDFS; -import org.json.simple.parser.JSONParser; -import org.json.simple.parser.ParseException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.w3c.dom.Document; -import org.w3c.dom.NodeList; -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import java.io.*; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.util.List; - -import static org.aksw.iguana.commons.streams.Streams.inputStream2String; - -/** - * SPARQL Language Processor. - * Tries to analyze Queries as SPARQL queries and checks http response for either application/sparql-results+json - * or application/sparql-results+xml to count the result size correctly. Otherwise assumes it record per line and counts the returning lines. - */ -@Shorthand("lang.SPARQL") -public class SPARQLLanguageProcessor extends AbstractLanguageProcessor implements LanguageProcessor { - - private static Logger LOGGER = LoggerFactory.getLogger(SPARQLLanguageProcessor.class); - - public static final String XML_RESULT_ELEMENT_NAME = "result"; - public static final String XML_RESULT_ROOT_ELEMENT_NAME = "results"; - public static final String QUERY_RESULT_TYPE_JSON = "application/sparql-results+json"; - public static final String QUERY_RESULT_TYPE_XML = "application/sparql-results+xml"; - private static final String LSQ_RES = "http://lsq.aksw.org/res/q-"; - - @Override - public String getQueryPrefix() { - return "sparql"; - } - - @Override - public Model generateTripleStats(List queries, String resourcePrefix, String taskID) { - Model model = ModelFactory.createDefaultModel(); - for(QueryWrapper wrappedQuery : queries) { - Resource subject = ResourceFactory.createResource(COMMON.RES_BASE_URI + resourcePrefix + "/" + wrappedQuery.getId()); - model.add(subject, RDF.type, Vocab.queryClass); - model.add(subject, Vocab.rdfsID, wrappedQuery.getId().replace("sparql", "")); - model.add(subject, RDFS.label, wrappedQuery.getQuery().toString()); - try { - Query q = QueryFactory.create(wrappedQuery.getQuery().toString()); - SPARQLQueryStatistics qs2 = new SPARQLQueryStatistics(); - qs2.getStatistics(q); - - model.add(subject, Vocab.aggrProperty, model.createTypedLiteral(qs2.aggr==1)); - model.add(subject, Vocab.filterProperty, model.createTypedLiteral(qs2.filter==1)); - model.add(subject, Vocab.groupByProperty, model.createTypedLiteral(qs2.groupBy==1)); - model.add(subject, Vocab.havingProperty, model.createTypedLiteral(qs2.having==1)); - model.add(subject, Vocab.triplesProperty, model.createTypedLiteral(qs2.triples)); - model.add(subject, Vocab.offsetProperty, model.createTypedLiteral(qs2.offset==1)); - model.add(subject, Vocab.optionalProperty, model.createTypedLiteral(qs2.optional==1)); - model.add(subject, Vocab.orderByProperty, model.createTypedLiteral(qs2.orderBy==1)); - model.add(subject, Vocab.unionProperty, model.createTypedLiteral(qs2.union==1)); - model.add(subject, OWL.sameAs, getLSQHash(q)); - }catch(Exception e){ - LOGGER.warn("Query statistics could not be created. Not using SPARQL?"); - } - } - return model; - } - - private Resource getLSQHash(Query query){ - HashCode hashCode = Hashing.sha256().hashString(query.toString(), StandardCharsets.UTF_8); - String result = BaseEncoding.base64Url().omitPadding().encode(hashCode.asBytes()); - return ResourceFactory.createResource(LSQ_RES+result); - } - - - public static String getContentTypeVal(Header header) { - for (HeaderElement el : header.getElements()) { - NameValuePair cTypePair = el.getParameterByName("Content-Type"); - - if (cTypePair != null && !cTypePair.getValue().isEmpty()) { - return cTypePair.getValue(); - } - } - int index = header.toString().indexOf("Content-Type"); - if (index >= 0) { - String ret = header.toString().substring(index + "Content-Type".length() + 1); - if (ret.contains(";")) { - return ret.substring(0, ret.indexOf(";")).trim(); - } - return ret.trim(); - } - return "application/sparql-results+json"; - } - - public static long getJsonResultSize(ByteArrayOutputStream res) throws ParseException, UnsupportedEncodingException { - JSONParser parser = new JSONParser(); - SaxSparqlJsonResultCountingParser handler = new SaxSparqlJsonResultCountingParser(); - parser.parse(res.toString(StandardCharsets.UTF_8), handler, true); - return handler.getNoBindings(); - } - - public static long getXmlResultSize(ByteArrayOutputStream res) throws ParserConfigurationException, IOException, SAXException { - DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); - - ByteArrayInputStream bbais = new ByteArrayInputStream(res.toByteArray()); - Document doc = dBuilder.parse(bbais); - NodeList childNodes = doc.getDocumentElement().getElementsByTagName(XML_RESULT_ROOT_ELEMENT_NAME).item(0).getChildNodes(); - - long size = 0; - for (int i = 0; i < childNodes.getLength(); i++) { - if (XML_RESULT_ELEMENT_NAME.equalsIgnoreCase(childNodes.item(i).getNodeName())) { - size++; - } - } - return size; - - } - - @Override - public Long getResultSize(CloseableHttpResponse response) throws ParserConfigurationException, SAXException, ParseException, IOException { - HttpEntity httpResponse = response.getEntity(); - Header contentTypeHeader = response.getEntity().getContentType(); - - ByteArrayOutputStream entity; - try (InputStream inputStream = httpResponse.getContent()) { - - entity = inputStream2String(inputStream); - } catch (IOException e) { - LOGGER.error("Query result could not be read.", e); - throw e; - } - return getResultSize(contentTypeHeader, entity, entity.size()); - } - - @Override - public Long getResultSize(Header contentTypeHeader, ByteArrayOutputStream content, long contentLength) throws ParserConfigurationException, SAXException, ParseException, IOException { - try { - switch (getContentTypeVal(contentTypeHeader)) { - case QUERY_RESULT_TYPE_JSON: - return getJsonResultSize(content); - - case QUERY_RESULT_TYPE_XML: - return getXmlResultSize(content); - default: - //return content.countMatches('\n')+1; - long matches=0; - for(byte b: content.toByteArray()){ - if(b=='\n'){ - matches++; - } - } - return matches+1; - } - } catch (ParseException | ParserConfigurationException | IOException | SAXException e) { - LOGGER.error("Query results could not be parsed: ", e); - throw e; - } - } -} diff --git a/src/main/java/org/aksw/iguana/cc/lang/impl/SaxSparqlJsonResultCountingParser.java b/src/main/java/org/aksw/iguana/cc/lang/impl/SaxSparqlJsonResultCountingParser.java index d4c1f3e2..09fc3838 100644 --- a/src/main/java/org/aksw/iguana/cc/lang/impl/SaxSparqlJsonResultCountingParser.java +++ b/src/main/java/org/aksw/iguana/cc/lang/impl/SaxSparqlJsonResultCountingParser.java @@ -1,114 +1,174 @@ package org.aksw.iguana.cc.lang.impl; +import org.aksw.iguana.cc.lang.LanguageProcessor; import org.json.simple.parser.ContentHandler; +import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; import static org.json.simple.parser.ParseException.ERROR_UNEXPECTED_EXCEPTION; /** * SAX Parser for SPARQL JSON Results. - * For correct SPARQL JSON Results it returns the correct size. + * For correct SPARQL JSON Results it returns the number of solutions, bound values and the names of the variables. * For malformed results it may or may not fail. For malformed JSON it fails if the underlying json.simple.parser fails. */ -class SaxSparqlJsonResultCountingParser implements ContentHandler { +@LanguageProcessor.ContentType("application/sparql-results+json") +class SaxSparqlJsonResultCountingParser extends LanguageProcessor { - private boolean headFound = false; - - private int objectDepth = 0; - private boolean inResults = false; - private boolean inBindings = false; - private boolean inBindingsArray = false; + @Override + public LanguageProcessingData process(InputStream inputStream) { + var parser = new JSONParser(); + var handler = new SaxSparqlJsonResultContentHandler(); + try { + parser.parse(new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)), handler); + return new SaxSparqlJsonResultData(handler.solutions(), handler.boundValues(), handler.variables(), null); + } catch (IOException e) { + throw new RuntimeException(e); + } catch (ParseException e) { + return new SaxSparqlJsonResultData(-1, -1, null, e); + } + } - private long noBindings = 0; + record SaxSparqlJsonResultData(long results, long bindings, + List variables, Exception exception) implements LanguageProcessingData { - public long getNoBindings() { - return noBindings; + @Override + public Class processor() { + return SaxSparqlJsonResultCountingParser.class; + } } - @Override - public void startJSON() { - } + private static class SaxSparqlJsonResultContentHandler implements ContentHandler { + // TODO: add support for ask queries and link + // TODO: code is unnecessary complicated - @Override - public void endJSON() throws ParseException { - if (inResults || inBindings || inBindingsArray || !headFound || objectDepth != 0) - throw new ParseException(ERROR_UNEXPECTED_EXCEPTION, "SPARQL Json Response was malformed."); - } + private boolean headFound = false; - @Override - public boolean startObject() { - objectDepth += 1; - if (objectDepth == 3 && inBindingsArray) { - noBindings += 1; + private int objectDepth = 0; + private boolean inResults = false; + private boolean inBindings = false; + private boolean inBindingsArray = false; + private boolean inVars = false; + + private long boundValues = 0; + + private long solutions = 0; + + private final List variables = new ArrayList<>(); + + + @Override + public void startJSON() { } - return true; - } - @Override - public boolean endObject() { - switch (objectDepth) { - case 1: - if (inResults) - inResults = false; - break; - case 2: - if (inBindings) { - inBindings = false; + @Override + public void endJSON() throws ParseException { + if (inResults || inBindings || inBindingsArray || !headFound || objectDepth != 0) + throw new ParseException(ERROR_UNEXPECTED_EXCEPTION, "SPARQL Json Response was malformed."); + } + + @Override + public boolean startObject() { + objectDepth += 1; + if (inBindingsArray) { + switch (objectDepth) { + case 3 -> solutions += 1; + case 4 -> boundValues += 1; } - break; + } + return true; } - objectDepth -= 1; - return true; - } - @Override - public boolean startArray() { - if (objectDepth == 2 && inResults && inBindings && !inBindingsArray) { - inBindingsArray = true; + @Override + public boolean endObject() { + switch (objectDepth) { + case 1: + if (inResults) + inResults = false; + break; + case 2: + if (inBindings) { + inBindings = false; + } + break; + } + objectDepth -= 1; + return true; } - return true; - } - @Override - public boolean endArray() { - if (objectDepth == 2 && inResults && inBindings && inBindingsArray) { - inBindingsArray = false; + @Override + public boolean startArray() { + if (objectDepth == 2 && inResults && inBindings && !inBindingsArray) { + inBindingsArray = true; + } + return true; } - return true; - } - @Override - public boolean startObjectEntry(String key) { - switch (objectDepth) { - case 1: - switch (key) { - case "head": - headFound = true; - break; - case "results": - if (headFound) - inResults = true; - break; + @Override + public boolean endArray() { + if (inVars) + inVars = false; + if (objectDepth == 2 && inResults && inBindings && inBindingsArray) { + inBindingsArray = false; + } + return true; + } + + + @Override + public boolean startObjectEntry(String key) { + switch (objectDepth) { + case 1 -> { + switch (key) { + case "head" -> headFound = true; + case "results" -> { + if (headFound) + inResults = true; + } + } } - break; - case 2: - if ("bindings".compareTo(key) == 0) { - inBindings = true; + case 2 -> { + if ("bindings".compareTo(key) == 0) { + inBindings = true; + } + if ("vars".compareTo(key) == 0) { + inVars = true; + } } - break; + } + return true; } - return true; - } - @Override - public boolean endObjectEntry() { - return true; - } + @Override + public boolean endObjectEntry() { + return true; + } - public boolean primitive(Object value) { - return true; - } + public boolean primitive(Object value) { + if (inVars) + variables.add(value.toString()); + + return true; + } + public long boundValues() { + return boundValues; + } + + public long solutions() { + return solutions; + } + public List variables() { + return variables; + } + } } \ No newline at end of file diff --git a/src/main/java/org/aksw/iguana/cc/lang/impl/ThrowawayLanguageProcessor.java b/src/main/java/org/aksw/iguana/cc/lang/impl/ThrowawayLanguageProcessor.java deleted file mode 100644 index 5f226793..00000000 --- a/src/main/java/org/aksw/iguana/cc/lang/impl/ThrowawayLanguageProcessor.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.aksw.iguana.cc.lang.impl; - -import org.aksw.iguana.cc.lang.AbstractLanguageProcessor; -import org.aksw.iguana.commons.annotation.Shorthand; -import org.aksw.iguana.commons.streams.Streams; -import org.apache.http.Header; -import org.json.simple.parser.ParseException; -import org.xml.sax.SAXException; - -import javax.xml.parsers.ParserConfigurationException; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.time.Instant; -import java.util.concurrent.TimeoutException; - -@Shorthand("lang.SIMPLE") -public class ThrowawayLanguageProcessor extends AbstractLanguageProcessor { - - @Override - public long readResponse(InputStream inputStream, ByteArrayOutputStream responseBody) throws IOException, TimeoutException { - return Streams.inputStream2Length(inputStream, Instant.now(), 0); - } - - @Override - public long readResponse(InputStream inputStream, Instant startTime, Double timeOut, ByteArrayOutputStream responseBody) throws IOException, TimeoutException { - return Streams.inputStream2Length(inputStream, startTime, timeOut); - } - - @Override - public Long getResultSize(Header contentTypeHeader, ByteArrayOutputStream content, long contentLength) throws ParserConfigurationException, SAXException, ParseException, IOException { - return contentLength; - } - -} diff --git a/src/main/java/org/aksw/iguana/cc/lang2/LanguageProcessor.java b/src/main/java/org/aksw/iguana/cc/lang2/LanguageProcessor.java deleted file mode 100644 index 31429482..00000000 --- a/src/main/java/org/aksw/iguana/cc/lang2/LanguageProcessor.java +++ /dev/null @@ -1,57 +0,0 @@ -package org.aksw.iguana.cc.lang2; - -import java.io.InputStream; -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; -import java.lang.reflect.InvocationTargetException; -import java.util.HashMap; -import java.util.Map; -import java.util.ServiceLoader; - -/** - * Interface for abstract language processors that work on InputStreams. - */ -public abstract class LanguageProcessor { - /** - * Provides the content type that a LanguageProcessor consumes. - */ - @Retention(RetentionPolicy.RUNTIME) - @Target(ElementType.TYPE) - public @interface ContentType { - String value(); - } - - public interface LanguageProcessingData { - Class processor(); - } - - public abstract LanguageProcessingData process(InputStream inputStream); - - final private static Map> processors = new HashMap<>(); - - static { - for (LanguageProcessor processor : ServiceLoader.load(LanguageProcessor.class)) { - LanguageProcessor.ContentType contentType = processor.getClass().getAnnotation(LanguageProcessor.ContentType.class); - if (contentType != null) { - processors.put(contentType.value(), processor.getClass()); - } - } - } - - public static LanguageProcessor getInstance(String contentType) { - Class processorClass = processors.get(contentType); - if (processorClass != null) { - try { - return processorClass.getDeclaredConstructor().newInstance(); - } catch (InstantiationException | IllegalAccessException | InvocationTargetException | - NoSuchMethodException e) { - throw new RuntimeException(e); - } - } - throw new IllegalArgumentException("No LanguageProcessor for ContentType " + contentType); - } - - -} diff --git a/src/main/java/org/aksw/iguana/cc/lang2/impl/SaxSparqlJsonResultCountingParser.java b/src/main/java/org/aksw/iguana/cc/lang2/impl/SaxSparqlJsonResultCountingParser.java deleted file mode 100644 index e862fc3e..00000000 --- a/src/main/java/org/aksw/iguana/cc/lang2/impl/SaxSparqlJsonResultCountingParser.java +++ /dev/null @@ -1,174 +0,0 @@ -package org.aksw.iguana.cc.lang2.impl; - -import org.aksw.iguana.cc.lang2.LanguageProcessor; -import org.json.simple.parser.ContentHandler; -import org.json.simple.parser.JSONParser; -import org.json.simple.parser.ParseException; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; - -import static org.json.simple.parser.ParseException.ERROR_UNEXPECTED_EXCEPTION; - -/** - * SAX Parser for SPARQL JSON Results. - * For correct SPARQL JSON Results it returns the number of solutions, bound values and the names of the variables. - * For malformed results it may or may not fail. For malformed JSON it fails if the underlying json.simple.parser fails. - */ -@LanguageProcessor.ContentType("application/sparql-results+json") -class SaxSparqlJsonResultCountingParser extends LanguageProcessor { - - @Override - public LanguageProcessingData process(InputStream inputStream) { - var parser = new JSONParser(); - var handler = new SaxSparqlJsonResultContentHandler(); - try { - parser.parse(new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)), handler); - return new SaxSparqlJsonResultData(handler.solutions(), handler.boundValues(), handler.variables(), null); - } catch (IOException e) { - throw new RuntimeException(e); - } catch (ParseException e) { - return new SaxSparqlJsonResultData(-1, -1, null, e); - } - } - - record SaxSparqlJsonResultData(long results, long bindings, - List variables, Exception exception) implements LanguageProcessingData { - - @Override - public Class processor() { - return SaxSparqlJsonResultCountingParser.class; - } - } - - private static class SaxSparqlJsonResultContentHandler implements ContentHandler { - // TODO: add support for ask queries and link - // TODO: code is unnecessary complicated - - private boolean headFound = false; - - private int objectDepth = 0; - private boolean inResults = false; - private boolean inBindings = false; - private boolean inBindingsArray = false; - private boolean inVars = false; - - private long boundValues = 0; - - private long solutions = 0; - - private final List variables = new ArrayList<>(); - - - @Override - public void startJSON() { - } - - @Override - public void endJSON() throws ParseException { - if (inResults || inBindings || inBindingsArray || !headFound || objectDepth != 0) - throw new ParseException(ERROR_UNEXPECTED_EXCEPTION, "SPARQL Json Response was malformed."); - } - - @Override - public boolean startObject() { - objectDepth += 1; - if (inBindingsArray) { - switch (objectDepth) { - case 3 -> solutions += 1; - case 4 -> boundValues += 1; - } - } - return true; - } - - @Override - public boolean endObject() { - switch (objectDepth) { - case 1: - if (inResults) - inResults = false; - break; - case 2: - if (inBindings) { - inBindings = false; - } - break; - } - objectDepth -= 1; - return true; - } - - @Override - public boolean startArray() { - if (objectDepth == 2 && inResults && inBindings && !inBindingsArray) { - inBindingsArray = true; - } - return true; - } - - @Override - public boolean endArray() { - if (inVars) - inVars = false; - if (objectDepth == 2 && inResults && inBindings && inBindingsArray) { - inBindingsArray = false; - } - return true; - } - - - @Override - public boolean startObjectEntry(String key) { - switch (objectDepth) { - case 1 -> { - switch (key) { - case "head" -> headFound = true; - case "results" -> { - if (headFound) - inResults = true; - } - } - } - case 2 -> { - if ("bindings".compareTo(key) == 0) { - inBindings = true; - } - if ("vars".compareTo(key) == 0) { - inVars = true; - } - } - } - return true; - } - - @Override - public boolean endObjectEntry() { - return true; - } - - public boolean primitive(Object value) { - if (inVars) - variables.add(value.toString()); - - return true; - } - - public long boundValues() { - return boundValues; - } - - public long solutions() { - return solutions; - } - - public List variables() { - return variables; - } - } -} \ No newline at end of file diff --git a/src/main/java/org/aksw/iguana/cc/model/QueryExecutionStats.java b/src/main/java/org/aksw/iguana/cc/model/QueryExecutionStats.java deleted file mode 100644 index 3103e277..00000000 --- a/src/main/java/org/aksw/iguana/cc/model/QueryExecutionStats.java +++ /dev/null @@ -1,62 +0,0 @@ -package org.aksw.iguana.cc.model; - -/** - * Wrapper for a query execution. - */ -public class QueryExecutionStats { - private String queryID; - private long responseCode; - private double executionTime; - private long resultSize; - - public QueryExecutionStats(String queryID, long responseCode, double executionTime) - { - this.queryID = queryID; - this.responseCode = responseCode; - this.executionTime = executionTime; - } - - - public QueryExecutionStats(String queryID, long responseCode, double executionTime, long resultSize) - { - this.queryID = queryID; - this.responseCode = responseCode; - this.executionTime = executionTime; - this.resultSize = resultSize; - } - - public QueryExecutionStats() { - } - - public String getQueryID() { - return queryID; - } - - public void setQueryID(String queryID) { - this.queryID = queryID; - } - - public long getResponseCode() { - return responseCode; - } - - public void setResponseCode(long responseCode) { - this.responseCode = responseCode; - } - - public double getExecutionTime() { - return executionTime; - } - - public void setExecutionTime(double executionTime) { - this.executionTime = executionTime; - } - - public long getResultSize() { - return resultSize; - } - - public void setResultSize(long resultSize) { - this.resultSize = resultSize; - } -} diff --git a/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java b/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java index 70d71cb9..6b6f32f3 100644 --- a/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java +++ b/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java @@ -16,6 +16,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; +import java.util.Objects; /** * The QueryHandler is used by every worker that extends the AbstractWorker. @@ -27,7 +28,7 @@ */ public class QueryHandler { - public record Config(@JsonProperty(required = true) String path, + public record Config(String path, Format format, Boolean caching, Order order, @@ -35,7 +36,7 @@ public record Config(@JsonProperty(required = true) String path, Language lang ) { - public Config(String path, Format format, Boolean caching, Order order, Long seed, Language lang) { + public Config(@JsonProperty(required = true) String path, Format format, Boolean caching, Order order, Long seed, Language lang) { this.path = path; this.format = format == null ? Format.ONE_PER_LINE : format; this.caching = caching == null ? true : caching; @@ -53,10 +54,7 @@ public enum Format { @JsonCreator Format(String value) { - if (value == null) - this.value = "one-per-line"; - else - this.value = value; + this.value = Objects.requireNonNullElse(value, "one-per-line"); } @JsonValue diff --git a/src/main/java/org/aksw/iguana/cc/suite/IguanaSuiteParser.java b/src/main/java/org/aksw/iguana/cc/suite/IguanaSuiteParser.java index a29df113..22883d6d 100644 --- a/src/main/java/org/aksw/iguana/cc/suite/IguanaSuiteParser.java +++ b/src/main/java/org/aksw/iguana/cc/suite/IguanaSuiteParser.java @@ -10,6 +10,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.deser.std.StdDeserializer; import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.networknt.schema.JsonSchema; @@ -95,13 +96,70 @@ record SuiteConfigWithID(long id, Suite.Config config) { * @throws IOException */ private static SuiteConfigWithID parse(InputStream inputStream, JsonFactory factory, Boolean validate) throws IOException { - final ObjectMapper mapper = new ObjectMapper(factory); + ObjectMapper mapper = new ObjectMapper(factory); - String input = new String(inputStream.readAllBytes(), StandardCharsets.UTF_8); + final var input = new String(inputStream.readAllBytes(), StandardCharsets.UTF_8); final var datasets = preparseDataset(mapper, input); + class DatasetDeserializer extends StdDeserializer { + public DatasetDeserializer() { + this(null); + } + + protected DatasetDeserializer(Class vc) { + super(vc); + } + + @Override + public DatasetConfig deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException { + JsonNode node = jp.getCodec().readTree(jp); + if (node.isTextual()) { + final var datasetName = node.asText(); + if (!datasets.containsKey(datasetName)) + throw new IllegalStateException(MessageFormat.format("Unknown dataset name: {0}", datasetName)); + return datasets.get(datasetName); + } else { + DatasetConfig datasetConfig = ctxt.readValue(jp, DatasetConfig.class); + if (datasets.containsKey(datasetConfig.name())) + assert datasets.get(datasetConfig.name()) == datasetConfig; + else datasets.put(datasetConfig.name(), datasetConfig); + return datasetConfig; // TODO: double check if this really works + } + } + } + mapper = new ObjectMapper(factory).registerModule(new SimpleModule() + .addDeserializer(DatasetConfig.class, new DatasetDeserializer())); + final var connections = preparseConnections(mapper, input); + class ConnectionDeserializer extends StdDeserializer { + + public ConnectionDeserializer() { + this(null); + } + + protected ConnectionDeserializer(Class vc) { + super(vc); + } + + @Override + public ConnectionConfig deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException, JsonProcessingException { + JsonNode node = jp.getCodec().readTree(jp); + if (node.isTextual()) { + final var connectionName = node.asText(); + if (!connections.containsKey(connectionName)) + throw new IllegalStateException(MessageFormat.format("Unknown connection name: {0}", connectionName)); + return connections.get(connectionName); + } else { + ConnectionConfig connectionConfig = ctxt.readValue(jp, ConnectionConfig.class); + if (connections.containsKey(connectionConfig.name())) + assert connections.get(connectionConfig.name()) == connectionConfig; + else connections.put(connectionConfig.name(), connectionConfig); + return connectionConfig; // TODO: double check if this really works + } + } + } + final var queryHandlers = new HashMap(); class QueryHandlerDeserializer extends StdDeserializer { @@ -159,12 +217,13 @@ public Duration deserialize(JsonParser jp, DeserializationContext ctxt) throws I } - mapper.registerModule(new JavaTimeModule()) + mapper = new ObjectMapper(factory).registerModule(new JavaTimeModule()) .registerModule(new SimpleModule() + .addDeserializer(DatasetConfig.class, new DatasetDeserializer()) + .addDeserializer(ConnectionConfig.class, new ConnectionDeserializer()) .addDeserializer(QueryHandler.class, new QueryHandlerDeserializer()) - .addDeserializer(Duration.class, new HumanReadableDurationDeserializer())) - ; - // TODO: update validator + .addDeserializer(Duration.class, new HumanReadableDurationDeserializer())); + // TODO: update validator and reactivate // if(validate && !validateConfig(config, schemaFile, mapper)){ // return null; // } @@ -185,37 +244,7 @@ record PreparsingDatasets(@JsonProperty(required = true) List dat } final var preparsingDatasets = mapper.readValue(input, PreparsingDatasets.class); - final var datasets = preparsingDatasets.datasets().stream().collect(Collectors.toMap(DatasetConfig::name, Function.identity())); - - class DatasetDeserializer extends StdDeserializer { - public DatasetDeserializer() { - this(null); - } - - protected DatasetDeserializer(Class vc) { - super(vc); - } - - @Override - public DatasetConfig deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException, JsonProcessingException { - JsonNode node = jp.getCodec().readTree(jp); - if (node.isTextual()) { - final var datasetName = node.asText(); - if (!datasets.containsKey(datasetName)) - throw new IllegalStateException(MessageFormat.format("Unknown dataset name: {0}", datasetName)); - return datasets.get(datasetName); - } else { - DatasetConfig datasetConfig = ctxt.readValue(jp, DatasetConfig.class); - if (datasets.containsKey(datasetConfig.name())) - assert datasets.get(datasetConfig.name()) == datasetConfig; - else datasets.put(datasetConfig.name(), datasetConfig); - return datasetConfig; // TODO: double check if this really works - } - } - } - mapper.registerModule(new SimpleModule() - .addDeserializer(DatasetConfig.class, new DatasetDeserializer())); - return datasets; + return preparsingDatasets.datasets().stream().collect(Collectors.toMap(DatasetConfig::name, Function.identity())); } private static Map preparseConnections(ObjectMapper mapper, String input) throws JsonProcessingException { @@ -224,38 +253,7 @@ record PreparsingConnections(@JsonProperty(required = true) List { - - public ConnectionDeserializer() { - this(null); - } - - protected ConnectionDeserializer(Class vc) { - super(vc); - } - - @Override - public ConnectionConfig deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException, JsonProcessingException { - JsonNode node = jp.getCodec().readTree(jp); - if (node.isTextual()) { - final var connectionName = node.asText(); - if (!connections.containsKey(connectionName)) - throw new IllegalStateException(MessageFormat.format("Unknown connection name: {0}", connectionName)); - return connections.get(connectionName); - } else { - ConnectionConfig connectionConfig = ctxt.readValue(jp, ConnectionConfig.class); - if (connections.containsKey(connectionConfig.name())) - assert connections.get(connectionConfig.name()) == connectionConfig; - else connections.put(connectionConfig.name(), connectionConfig); - return connectionConfig; // TODO: double check if this really works - } - } - } - mapper.registerModule(new SimpleModule() - .addDeserializer(ConnectionConfig.class, new ConnectionDeserializer())); - return connections; + return preparsingConnections.connections().stream().collect(Collectors.toMap(ConnectionConfig::name, Function.identity())); } private static boolean validateConfig(Path config, String schemaFile, ObjectMapper mapper) throws IOException { diff --git a/src/main/java/org/aksw/iguana/cc/suite/Suite.java b/src/main/java/org/aksw/iguana/cc/suite/Suite.java index 93533bdf..af1a9698 100644 --- a/src/main/java/org/aksw/iguana/cc/suite/Suite.java +++ b/src/main/java/org/aksw/iguana/cc/suite/Suite.java @@ -1,5 +1,6 @@ package org.aksw.iguana.cc.suite; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import org.aksw.iguana.cc.config.elements.ConnectionConfig; import org.aksw.iguana.cc.config.elements.DatasetConfig; @@ -14,15 +15,15 @@ public class Suite { public record Config( - @JsonProperty(required = true) - List datasets, - @JsonProperty(required = true) - List connections, + @JsonIgnore + List datasets, /* Will already be consumed and ignored herein */ + @JsonIgnore + List connections, /* Will already be consumed and ignored herein */ @JsonProperty(required = true) List tasks, - @JsonProperty List storages) { } + public record Result(List stresstest) { } @@ -33,7 +34,7 @@ public record Result(List stresstest) { private final List stresstests = new ArrayList<>(); - Suite(long suiteId, Config config){ + Suite(long suiteId, Config config) { this.suiteId = suiteId; this.config = config; diff --git a/src/main/java/org/aksw/iguana/cc/worker/ResponseBodyProcessor.java b/src/main/java/org/aksw/iguana/cc/worker/ResponseBodyProcessor.java index 8c2c188d..d127f7e0 100644 --- a/src/main/java/org/aksw/iguana/cc/worker/ResponseBodyProcessor.java +++ b/src/main/java/org/aksw/iguana/cc/worker/ResponseBodyProcessor.java @@ -1,6 +1,6 @@ package org.aksw.iguana.cc.worker; -import org.aksw.iguana.cc.lang2.LanguageProcessor; +import org.aksw.iguana.cc.lang.LanguageProcessor; import org.aksw.iguana.commons.io.BigByteArrayInputStream; import org.aksw.iguana.commons.io.BigByteArrayOutputStream; import org.slf4j.Logger; @@ -9,8 +9,6 @@ import java.text.MessageFormat; import java.time.Duration; import java.time.temporal.ChronoUnit; -import java.util.HashMap; -import java.util.Map; import java.util.concurrent.*; public class ResponseBodyProcessor { diff --git a/src/main/java/org/aksw/iguana/cc/worker/impl/SPARQLProtocolWorker.java b/src/main/java/org/aksw/iguana/cc/worker/impl/SPARQLProtocolWorker.java index 0f289b00..4c105c2f 100644 --- a/src/main/java/org/aksw/iguana/cc/worker/impl/SPARQLProtocolWorker.java +++ b/src/main/java/org/aksw/iguana/cc/worker/impl/SPARQLProtocolWorker.java @@ -1,6 +1,8 @@ package org.aksw.iguana.cc.worker.impl; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonValue; import net.jpountz.xxhash.XXHashFactory; import org.aksw.iguana.cc.config.elements.ConnectionConfig; import org.aksw.iguana.cc.query.handler.QueryHandler; @@ -30,11 +32,23 @@ public class SPARQLProtocolWorker extends HttpWorker { public final static class SPARQLProtocolRequestFactory { public enum RequestType { - GET_QUERY, - POST_URL_ENC_QUERY, - POST_QUERY, - POST_URL_ENC_UPDATE, - POST_UPDATE + GET_QUERY("get query"), + POST_URL_ENC_QUERY("post url-enc query"), + POST_QUERY("post query"), + POST_URL_ENC_UPDATE("post url-enc update"), + POST_UPDATE("post update"); + + private final String value; + + @JsonCreator + RequestType(String value) { + this.value = Objects.requireNonNullElse(value, "one-per-line"); + } + + @JsonValue + public String value() { + return value; + } } private final RequestType requestType; @@ -55,8 +69,9 @@ public HttpRequest buildHttpRequest(InputStream queryStream, URI endpoint, String requestHeader) throws URISyntaxException, IOException { HttpRequest.Builder request = HttpRequest.newBuilder() - .timeout(timeout) - .header("Accept", requestHeader); + .timeout(timeout); + if (requestHeader != null) + request.header("Accept", requestHeader); switch (this.requestType) { case GET_QUERY -> { request.uri(new URIBuilder(endpoint) @@ -111,22 +126,29 @@ public HttpRequest buildHttpRequest(InputStream queryStream, // @JsonTypeName("SPARQLProtocolWorker") public record Config(Integer number, - @JsonProperty(required = true) QueryHandler queries, - @JsonProperty(required = true) CompletionTarget completionTarget, - @JsonProperty(required = true) ConnectionConfig connection, - @JsonProperty(required = true) Duration timeout, + QueryHandler queries, + CompletionTarget completionTarget, + ConnectionConfig connection, + Duration timeout, String acceptHeader /* e.g. application/sparql-results+json */, - @JsonProperty(required = true) SPARQLProtocolRequestFactory.RequestType requestType, + SPARQLProtocolRequestFactory.RequestType requestType, boolean parseResults ) implements HttpWorker.Config { - public Config(Integer number, QueryHandler queries, CompletionTarget completionTarget, ConnectionConfig connection, Duration timeout, String acceptHeader, SPARQLProtocolRequestFactory.RequestType requestType, boolean parseResults) { + public Config(Integer number, + @JsonProperty(required = true) QueryHandler queries, + @JsonProperty(required = true) CompletionTarget completionTarget, + @JsonProperty(required = true) ConnectionConfig connection, + @JsonProperty(required = true) Duration timeout, + String acceptHeader, + SPARQLProtocolRequestFactory.RequestType requestType, + boolean parseResults) { this.number = number == null ? 1 : number; this.queries = queries; this.completionTarget = completionTarget; this.connection = connection; this.timeout = timeout; this.acceptHeader = acceptHeader; - this.requestType = requestType; + this.requestType = requestType == null ? SPARQLProtocolRequestFactory.RequestType.GET_QUERY : requestType; this.parseResults = parseResults; } } diff --git a/src/test/java/org/aksw/iguana/cc/lang/MockCloseableHttpResponse.java b/src/test/java/org/aksw/iguana/cc/lang/MockCloseableHttpResponse.java deleted file mode 100644 index 5d7fc06e..00000000 --- a/src/test/java/org/aksw/iguana/cc/lang/MockCloseableHttpResponse.java +++ /dev/null @@ -1,49 +0,0 @@ -package org.aksw.iguana.cc.lang; - -import org.apache.http.HttpStatus; -import org.apache.http.ProtocolVersion; -import org.apache.http.ReasonPhraseCatalog; -import org.apache.http.StatusLine; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.entity.BasicHttpEntity; -import org.apache.http.message.BasicHttpResponse; -import org.apache.http.message.BasicStatusLine; - -import java.io.*; -import java.net.URL; -import java.util.Locale; - -public class MockCloseableHttpResponse extends BasicHttpResponse implements CloseableHttpResponse { - - public MockCloseableHttpResponse(StatusLine statusline, ReasonPhraseCatalog catalog, Locale locale) { - super(statusline, catalog, locale); - } - - public MockCloseableHttpResponse(StatusLine statusline) { - super(statusline); - } - - public MockCloseableHttpResponse(ProtocolVersion ver, int code, String reason) { - super(ver, code, reason); - } - - @Override - public void close() throws IOException { - - } - - public static CloseableHttpResponse buildMockResponse(String data, String contentType) throws FileNotFoundException, UnsupportedEncodingException { - ProtocolVersion protocolVersion = new ProtocolVersion("HTTP", 1, 1); - String reasonPhrase = "OK"; - StatusLine statusline = new BasicStatusLine(protocolVersion, HttpStatus.SC_OK, reasonPhrase); - MockCloseableHttpResponse mockResponse = new MockCloseableHttpResponse(statusline); - BasicHttpEntity entity = new BasicHttpEntity(); - entity.setContentType(contentType); - //entity.setContentType(contentType); - URL url = Thread.currentThread().getContextClassLoader().getResource("response.txt"); - InputStream instream = new ByteArrayInputStream(data.getBytes()); - entity.setContent(instream); - mockResponse.setEntity(entity); - return mockResponse; - } -} diff --git a/src/test/java/org/aksw/iguana/cc/lang/RDFLanguageProcessorTest.java b/src/test/java/org/aksw/iguana/cc/lang/RDFLanguageProcessorTest.java deleted file mode 100644 index 37bb4176..00000000 --- a/src/test/java/org/aksw/iguana/cc/lang/RDFLanguageProcessorTest.java +++ /dev/null @@ -1,65 +0,0 @@ -package org.aksw.iguana.cc.lang; - -import org.aksw.iguana.cc.lang.impl.RDFLanguageProcessor; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.jena.rdf.model.Model; -import org.apache.jena.rdf.model.ModelFactory; -import org.apache.jena.rdf.model.ResourceFactory; -import org.apache.jena.riot.Lang; -import org.json.simple.parser.ParseException; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.xml.sax.SAXException; - -import javax.xml.parsers.ParserConfigurationException; -import java.io.IOException; -import java.io.StringWriter; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.Collection; - -import static org.junit.Assert.assertEquals; - -@RunWith(Parameterized.class) -public class RDFLanguageProcessorTest { - - private static Logger LOGGER = LoggerFactory.getLogger(RDFLanguageProcessorTest.class); - private final Lang lang; - private final Model m; - - @Parameterized.Parameters - public static Collection data() throws IllegalAccessException { - Collection testData = new ArrayList(); - for(Field langField : Lang.class.getFields()) { - Lang susLang = (Lang)langField.get(Lang.class); - if(susLang.equals(Lang.RDFTHRIFT) || susLang.equals(Lang.TRIX) || susLang.equals(Lang.SHACLC) || susLang.equals(Lang.TSV) || susLang.equals(Lang.CSV) || susLang.equals(Lang.RDFNULL)) { - //cannot test them as model doesn't allow them to write - continue; - } - testData.add(new Object[]{susLang}); - } - return testData; - } - - public RDFLanguageProcessorTest(Lang lang){ - this.lang = lang; - this.m = ModelFactory.createDefaultModel(); - m.add(ResourceFactory.createResource("uri://test"), ResourceFactory.createProperty("uri://prop1"), "abc"); - m.add(ResourceFactory.createResource("uri://test"), ResourceFactory.createProperty("uri://prop2"), "abc2"); - LOGGER.info("Testing Lanuage {} Content-Type: {}", lang.getName(), lang.getContentType()); - } - - @Test - public void testCorrectModel() throws IOException, ParserConfigurationException, SAXException, ParseException { - StringWriter sw = new StringWriter(); - m.write(sw, lang.getName(), null); - CloseableHttpResponse response = MockCloseableHttpResponse.buildMockResponse(sw.toString(), lang.getContentType().getContentTypeStr()); - RDFLanguageProcessor processor = new RDFLanguageProcessor(); - assertEquals(2, processor.getResultSize(response).longValue()); - } - - -} diff --git a/src/test/java/org/aksw/iguana/cc/lang/SPARQLLanguageProcessorTest.java b/src/test/java/org/aksw/iguana/cc/lang/SPARQLLanguageProcessorTest.java deleted file mode 100644 index 93b1aff5..00000000 --- a/src/test/java/org/aksw/iguana/cc/lang/SPARQLLanguageProcessorTest.java +++ /dev/null @@ -1,144 +0,0 @@ -package org.aksw.iguana.cc.lang; - -import org.aksw.iguana.cc.lang.impl.SPARQLLanguageProcessor; -import org.apache.jena.ext.com.google.common.collect.Lists; -import org.apache.jena.query.Query; -import org.apache.jena.query.QueryFactory; -import org.apache.jena.rdf.model.Model; -import org.apache.jena.rdf.model.ModelFactory; -import org.json.simple.parser.ParseException; -import org.junit.Test; -import org.xml.sax.SAXException; - -import javax.xml.parsers.ParserConfigurationException; -import java.io.ByteArrayOutputStream; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.nio.charset.StandardCharsets; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -public class SPARQLLanguageProcessorTest { - - private String jsonResult = "{\n" + - " \"head\": { \"vars\": [ \"book\" , \"title\" ]\n" + - " } ,\n" + - " \"results\": { \n" + - " \"bindings\": [\n" + - " {\n" + - " \"book\": { \"type\": \"uri\" , \"value\": \"http://example.org/book/book3\" } ,\n" + - " \"title\": { \"type\": \"literal\" , \"value\": \"Example Book 3\" }\n" + - " } ,\n" + - " {\n" + - " \"book\": { \"type\": \"uri\" , \"value\": \"http://example.org/book/book2\" } ,\n" + - " \"title\": { \"type\": \"literal\" , \"value\": \"Example Book 2\" }\n" + - " } ,\n" + - " {\n" + - " \"book\": { \"type\": \"uri\" , \"value\": \"http://example.org/book/book1\" } ,\n" + - " \"title\": { \"type\": \"literal\" , \"value\": \"Example Book 1\" }\n" + - " }\n" + - " ]\n" + - " }\n" + - "}"; - private String xmlResult = "\n" + - "\n" + - " \n" + - " \n" + - " \n" + - " \n" + - "\n" + - " \n" + - " \n" + - " test1\n" + - " ... \n" + - " \n" + - "\n" + - " \n" + - " test2\n" + - " ... \n" + - " \n" + - " \n" + - " \n" + - "\n" + - ""; - - - - - @Test - public void checkJSON() throws ParseException, IOException { - ByteArrayOutputStream bbaos = new ByteArrayOutputStream(); - bbaos.write(jsonResult.getBytes()); - assertEquals(3, SPARQLLanguageProcessor.getJsonResultSize(bbaos)); - //test if valid json response provide 0 bindings - try { - //check if invalid json throws exception - bbaos = new ByteArrayOutputStream(); - bbaos.write("{ \"a\": \"b\"}".getBytes()); - SPARQLLanguageProcessor.getJsonResultSize(bbaos); - assertTrue("Should have thrown an error", false); - }catch(Exception e){ - assertTrue(true); - } - try { - //check if invalid json throws exception - bbaos = new ByteArrayOutputStream(); - bbaos.write("{ \"a\": \"b\"".getBytes()); - SPARQLLanguageProcessor.getJsonResultSize(bbaos); - assertTrue("Should have thrown an error", false); - }catch(Exception e){ - assertTrue(true); - } - } - - @Test - public void checkXML() throws IOException, SAXException, ParserConfigurationException { - ByteArrayOutputStream bbaos = new ByteArrayOutputStream(); - bbaos.write(xmlResult.getBytes(StandardCharsets.UTF_8)); - assertEquals(2, SPARQLLanguageProcessor.getXmlResultSize(bbaos)); - //test if valid xml response provide 0 bindings - try { - //check if invalid xml throws exception - bbaos = new ByteArrayOutputStream(); - bbaos.write("b".getBytes()); - SPARQLLanguageProcessor.getJsonResultSize(bbaos); - assertTrue("Should have thrown an error", false); - }catch(Exception e){ - assertTrue(true); - } - try { - //check if invalid xml throws exception - bbaos = new ByteArrayOutputStream(); - bbaos.write("{ \"a\": \"b\"".getBytes()); - SPARQLLanguageProcessor.getJsonResultSize(bbaos); - assertTrue("Should have thrown an error", false); - }catch(Exception e){ - assertTrue(true); - } - } - - @Test - public void checkResultSize() throws IOException, ParserConfigurationException, SAXException, ParseException { - SPARQLLanguageProcessor languageProcessor = new SPARQLLanguageProcessor(); - assertEquals(3, languageProcessor.getResultSize(MockCloseableHttpResponse.buildMockResponse(jsonResult, SPARQLLanguageProcessor.QUERY_RESULT_TYPE_JSON)).longValue()); - assertEquals(2, languageProcessor.getResultSize(MockCloseableHttpResponse.buildMockResponse(xmlResult, SPARQLLanguageProcessor.QUERY_RESULT_TYPE_XML)).longValue()); - assertEquals(4, languageProcessor.getResultSize(MockCloseableHttpResponse.buildMockResponse("a\na\na\nb", "text/plain")).longValue()); - } - - - @Test - public void checkGeneratedStatsModel() throws IOException { - Query q = QueryFactory.create("SELECT * {?s ?p ?o. ?o ?q ?t. FILTER(?t = \"abc\")} GROUP BY ?s"); - QueryWrapper wrapped = new QueryWrapper(q, "abc"); - SPARQLLanguageProcessor languageProcessor = new SPARQLLanguageProcessor(); - Model actual = languageProcessor.generateTripleStats(Lists.newArrayList(wrapped),"query","1/1/2"); - Model expected = ModelFactory.createDefaultModel(); - expected.read(new FileReader("src/test/resources/querystats.nt"), null, "N-TRIPLE"); - assertEquals(expected.size(), actual.size()); - expected.remove(actual); - actual.write(new FileWriter("test2.nt"), "N-TRIPLE"); - assertEquals(0, expected.size()); - } -} diff --git a/src/test/java/org/aksw/iguana/cc/utils/ServerMock.java b/src/test/java/org/aksw/iguana/cc/utils/ServerMock.java deleted file mode 100644 index b4485ae3..00000000 --- a/src/test/java/org/aksw/iguana/cc/utils/ServerMock.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.aksw.iguana.cc.utils; - -import org.aksw.iguana.cc.lang.impl.SPARQLLanguageProcessor; -import org.apache.commons.io.FileUtils; -import org.simpleframework.http.Request; -import org.simpleframework.http.Response; -import org.simpleframework.http.Status; -import org.simpleframework.http.core.Container; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; - -/** - * Server Mock representing a TS - * - * @author f.conrads - * - */ -public class ServerMock implements Container { - - private static final Logger LOGGER = LoggerFactory.getLogger(ServerMock.class); - private String actualContent; - - - @Override - public void handle(Request request, Response resp) { - String content=null; - try { - content = request.getContent(); - } catch (IOException e) { - LOGGER.error("Got exception.", e); - } - resp.setCode(Status.OK.code); - resp.setContentType(SPARQLLanguageProcessor.QUERY_RESULT_TYPE_JSON); - try { - //write answer - String resultStr = FileUtils.readFileToString(new File("src/test/resources/sparql-json-response.json"), "UTF-8"); - resp.getOutputStream().write(resultStr.getBytes()); - resp.getOutputStream().close(); - } catch (IOException e) { - LOGGER.error("Could not close Response Output Stream"); - } - } - - -}