From 82dd89e4c6d74b1cb8fd6a1b3c825f9feb0fa13b Mon Sep 17 00:00:00 2001 From: Nick Molcanov <32801560+nck-mlcnv@users.noreply.github.com> Date: Tue, 14 Nov 2023 11:11:17 +0100 Subject: [PATCH] HTTP worker refactoring (#221) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * SPARQLProtocolWorker is a draft for a better, more reliable worker that is tailored towards SPARQL Protocol. Each worker uses a single HttpClient and handles work completion conditions itself. * Add workerId and ExecutionStats to SPARQLProtocolWorker Refactored SPARQLProtocolWorker to record workerId and execution stats for each worker. WorkerId was added to uniquely identify each worker. An ExecutionStats inner class was created to track start time, duration, HTTP status code, content length, number of bindings, and number of solutions for each worker's task. * "Refactor SPARQLProtocolWorker to handle query streams. This commit changes the query building mechanism within SPARQLProtocolWorker.java, shifting from StringBuilder to InputStream, aiming to support processing of large queries, and reduce overhead from using String for queryID. Now it reads queries directly from QueryHandler's data stream, with modifications to a number of HTTP Request methods to accommodate this change. The refactor also includes addition of new method in Query Handler which returns 'QueryHandle' record—a container for index and InputStream for a query." * Add streaming support for handling large queries Introduced InputStream support in the QueryList and QuerySource to handle large queries more efficiently. Changes have been made to IndexedQueryReader, QuerySource, QueryHandler, and several other classes to accommodate the new streaming feature. Previously, all queries were loaded into memory which might cause OutOfMemoryError for large queries. It still depends on the SPARQL worker used if queries are streamed to the client. * Refactored BigByteArrayOutputStream * Hashing and large response body support for SPARQLProtocolWorker * remove dangling javadoc comment * Scaffold ResponseBodyProcessor. This class keeps track of already handled responses to avoid repeated processing. It uses a concurrent hash map to store the responses identified by unique keys. This approach aims to improve the efficiency of handling response bodies in multi-threaded scenarios. * Use unsynchronized ByteArrayOutputStream for BigByteArrayInput/BigArrayOutputStream and complete rewrite of BigByteArrayInputStream. This should increase the performance of both streams significantly. * Add Language Processor and SparqlJsonResultCountingParser Implemented the AbstractLanguageProcessor interface to process InputStreams. A new SAX Parser (SaxSparqlJsonResultCountingParser) was introduced for SPARQL JSON results, returning solutions, bound values, and variables. * Completed ResponseBodyProcessor and integrated it into SPARQLProtocolWorker * Worker integration and removal of a lot of code * small fixes * changes to the SPARQLProtocolWorker * delegated executeQuery method * reuse bbaos if not consumed * removed assert for non-differing content-length header value and actual content length * better logging for malformed url * Add basic logging for Suite class * remove JUnit 4 and add surefire plugin The surefire plugin is used for better control over the available system resources for the test, because the BigByteArrayStream tests can take a lot of them. * update iguana-schema.json * Update config file validation and change suiteID generation This also removes some unused redundant code. The suiteID has also been changed to a string type, that consists of an epoch timestamp in seconds and the hashcode of the configuration file. * Remove CLIProcessManager.java * Update schema file and re-enable tests The validation function has also been made public, for better testing. * Remove test files for IndexQueryReader See issue #214. * Add start and end-time for each worker. Adjusted the test as well and integrate it in the StresstestResultProcessor and Storages. * Remove unused dependencies * Document possible problem with the SPARQLProtocolWorker and the connected client --------- Co-authored-by: Alexander Bigerl Co-authored-by: Alexander Bigerl --- docs/develop/extend-metrics.md | 1 - docs/develop/extend-queryhandling.md | 2 +- docs/usage/configuration.md | 4 +- example-suite.yml | 97 +-- pom.xml | 95 +-- schema/iguana-schema.json | 658 +++++++++--------- .../org/aksw/iguana/cc/config/CONSTANTS.java | 34 - .../aksw/iguana/cc/config/ConfigManager.java | 54 -- .../aksw/iguana/cc/config/IguanaConfig.java | 174 ----- .../iguana/cc/config/IguanaConfigFactory.java | 69 -- .../cc/config/elements/ConnectionConfig.java | 98 +-- .../cc/config/elements/DatasetConfig.java | 29 +- .../cc/config/elements/MetricConfig.java | 42 -- .../cc/config/elements/StorageConfig.java | 46 +- .../iguana/cc/config/elements/TaskConfig.java | 42 -- .../iguana/cc/controller/MainController.java | 109 +-- .../iguana/cc/controller/TaskController.java | 34 - .../cc/lang/AbstractLanguageProcessor.java | 65 -- .../iguana/cc/lang/LanguageProcessor.java | 90 ++- .../org/aksw/iguana/cc/lang/QueryWrapper.java | 41 -- .../cc/lang/impl/RDFLanguageProcessor.java | 112 --- .../cc/lang/impl/SPARQLLanguageProcessor.java | 190 ----- .../SaxSparqlJsonResultCountingParser.java | 263 +++++-- .../lang/impl/ThrowawayLanguageProcessor.java | 35 - .../org/aksw/iguana/cc/metrics/Metric.java | 41 ++ .../iguana/cc/metrics/ModelWritingMetric.java | 19 + .../aksw/iguana/cc/metrics/QueryMetric.java | 9 + .../aksw/iguana/cc/metrics/TaskMetric.java | 9 + .../aksw/iguana/cc/metrics/WorkerMetric.java | 9 + .../impl/AggregatedExecutionStatistics.java | 88 +++ .../aksw/iguana/cc/metrics/impl/AvgQPS.java | 45 ++ .../metrics/impl/EachExecutionStatistic.java | 56 ++ .../org/aksw/iguana/cc/metrics/impl/NoQ.java | 38 + .../aksw/iguana/cc/metrics/impl/NoQPH.java | 47 ++ .../stresstest => }/metrics/impl/PAvgQPS.java | 30 +- .../org/aksw/iguana/cc/metrics/impl/PQPS.java | 43 ++ .../org/aksw/iguana/cc/metrics/impl/QMPH.java | 49 ++ .../stresstest => }/metrics/impl/QPS.java | 19 +- .../iguana/cc/model/QueryExecutionStats.java | 15 - .../iguana/cc/model/QueryResultHashKey.java | 55 -- .../iguana/cc/query/handler/QueryHandler.java | 285 ++++---- .../aksw/iguana/cc/query/list/QueryList.java | 28 +- .../query/list/impl/FileBasedQueryList.java | 12 +- .../cc/query/list/impl/InMemQueryList.java | 24 +- .../cc/query/pattern/PatternHandler.java | 213 ------ .../cc/query/selector/QuerySelector.java | 18 +- .../selector/impl/LinearQuerySelector.java | 26 +- .../selector/impl/RandomQuerySelector.java | 18 +- .../iguana/cc/query/source/QuerySource.java | 8 +- .../source/impl/FileLineQuerySource.java | 37 +- .../source/impl/FileSeparatorQuerySource.java | 31 +- .../query/source/impl/FolderQuerySource.java | 52 +- .../org/aksw/iguana/cc/storage/Storable.java | 40 ++ .../org/aksw/iguana/cc/storage/Storage.java | 34 + .../iguana/cc/storage/impl/CSVStorage.java | 417 +++++++++++ .../cc/storage/impl/RDFFileStorage.java | 98 +++ .../storage/impl/TriplestoreStorage.java | 86 ++- .../iguana/cc/suite/IguanaSuiteParser.java | 260 +++++++ .../java/org/aksw/iguana/cc/suite/Suite.java | 103 +++ .../aksw/iguana/cc/tasks/AbstractTask.java | 63 -- .../java/org/aksw/iguana/cc/tasks/Task.java | 74 +- .../org/aksw/iguana/cc/tasks/TaskFactory.java | 18 - .../org/aksw/iguana/cc/tasks/TaskManager.java | 45 -- .../aksw/iguana/cc/tasks/impl/Stresstest.java | 111 +++ .../tasks/impl/StresstestResultProcessor.java | 280 ++++++++ .../cc/tasks/stresstest/Stresstest.java | 364 ---------- .../tasks/stresstest/StresstestMetadata.java | 24 - .../stresstest/StresstestResultProcessor.java | 230 ------ .../cc/tasks/stresstest/metrics/Metric.java | 27 - .../stresstest/metrics/MetricManager.java | 15 - .../metrics/ModelWritingMetric.java | 20 - .../tasks/stresstest/metrics/QueryMetric.java | 9 - .../tasks/stresstest/metrics/TaskMetric.java | 10 - .../stresstest/metrics/WorkerMetric.java | 10 - .../impl/AggregatedExecutionStatistics.java | 101 --- .../tasks/stresstest/metrics/impl/AvgQPS.java | 50 -- .../metrics/impl/EachExecutionStatistic.java | 52 -- .../cc/tasks/stresstest/metrics/impl/NoQ.java | 43 -- .../tasks/stresstest/metrics/impl/NoQPH.java | 53 -- .../tasks/stresstest/metrics/impl/PQPS.java | 45 -- .../tasks/stresstest/metrics/impl/QMPH.java | 54 -- .../cc/tasks/stresstest/storage/Storage.java | 19 - .../stresstest/storage/StorageManager.java | 73 -- .../storage/TripleBasedStorage.java | 29 - .../stresstest/storage/impl/CSVStorage.java | 282 -------- .../storage/impl/NTFileStorage.java | 83 --- .../storage/impl/RDFFileStorage.java | 75 -- .../iguana/cc/utils/CLIProcessManager.java | 137 ---- .../org/aksw/iguana/cc/utils/FileUtils.java | 15 +- .../iguana/cc/utils/IndexedQueryReader.java | 62 +- .../iguana/cc/utils/ResultSizeRetriever.java | 49 -- .../cc/utils/SPARQLQueryStatistics.java | 49 -- .../iguana/cc/utils/StatisticsVisitor.java | 48 -- .../aksw/iguana/cc/worker/AbstractWorker.java | 280 -------- .../org/aksw/iguana/cc/worker/HttpWorker.java | 161 +++++ .../iguana/cc/worker/LatencyStrategy.java | 27 - .../cc/worker/ResponseBodyProcessor.java | 82 +++ .../ResponseBodyProcessorInstances.java | 44 ++ .../org/aksw/iguana/cc/worker/Worker.java | 99 --- .../aksw/iguana/cc/worker/WorkerFactory.java | 14 - .../aksw/iguana/cc/worker/WorkerMetadata.java | 10 - .../cc/worker/impl/CLIInputFileWorker.java | 50 -- .../cc/worker/impl/CLIInputPrefixWorker.java | 37 - .../iguana/cc/worker/impl/CLIInputWorker.java | 124 ---- .../aksw/iguana/cc/worker/impl/CLIWorker.java | 98 --- .../iguana/cc/worker/impl/HttpGetWorker.java | 57 -- .../iguana/cc/worker/impl/HttpPostWorker.java | 61 -- .../iguana/cc/worker/impl/HttpWorker.java | 296 -------- .../worker/impl/MultipleCLIInputWorker.java | 180 ----- .../cc/worker/impl/SPARQLProtocolWorker.java | 445 ++++++++++++ .../iguana/cc/worker/impl/UPDATEWorker.java | 110 --- .../cc/worker/impl/update/UpdateTimer.java | 100 --- .../iguana/commons/annotation/Nullable.java | 12 - .../commons/annotation/ParameterNames.java | 14 - .../iguana/commons/annotation/Shorthand.java | 14 - .../aksw/iguana/commons/constants/COMMON.java | 113 --- .../iguana/commons/factory/TypedFactory.java | 293 -------- .../commons/io/BigByteArrayInputStream.java | 163 ++++- .../commons/io/BigByteArrayOutputStream.java | 216 ++++-- .../iguana/commons/numbers/NumberUtils.java | 39 -- .../org/aksw/iguana/commons/rdf/IONT.java | 9 +- .../org/aksw/iguana/commons/rdf/IPROP.java | 104 +-- .../org/aksw/iguana/commons/rdf/IRES.java | 64 +- .../commons/reflect/ShorthandMapper.java | 71 -- .../iguana/commons/script/ScriptExecutor.java | 105 --- .../aksw/iguana/commons/streams/Streams.java | 118 ---- .../aksw/iguana/commons/time/TimeUtils.java | 40 +- .../org/aksw/iguana/cc/config/ConfigTest.java | 57 -- .../aksw/iguana/cc/config/WorkflowTest.java | 123 ---- .../config/elements/ConnectionConfigTest.java | 185 +++++ .../cc/config/elements/DatasetConfigTest.java | 39 ++ .../cc/config/elements/StorageConfigTest.java | 51 ++ .../cc/lang/MockCloseableHttpResponse.java | 49 -- .../cc/lang/RDFLanguageProcessorTest.java | 65 -- .../cc/lang/SPARQLLanguageProcessorTest.java | 144 ---- .../iguana/cc/mockup/MockupConnection.java | 19 + .../iguana/cc/mockup/MockupQueryHandler.java | 41 ++ .../aksw/iguana/cc/mockup/MockupStorage.java | 18 + .../aksw/iguana/cc/mockup/MockupWorker.java | 118 ++++ .../cc/model/QueryResultHashKeyTest.java | 52 -- .../query/handler/QueryHandlerConfigTest.java | 141 ++++ .../cc/query/handler/QueryHandlerTest.java | 289 ++++---- .../iguana/cc/query/list/QueryListTest.java | 142 ++++ .../pattern/PatternBasedQueryHandlerTest.java | 141 ---- .../cc/query/pattern/PatternHandlerTest.java | 113 --- .../impl/LinearQuerySelectorTest.java | 30 +- .../impl/RandomQuerySelectorTest.java | 39 ++ .../source/impl/FileLineQuerySourceTest.java | 115 ++- .../impl/FileSeparatorQuerySourceTest.java | 143 ++-- .../source/impl/FolderQuerySourceTest.java | 99 ++- .../cc/storage/impl/CSVStorageTest.java | 140 ++++ .../cc/storage/impl/RDFFileStorageTest.java | 56 ++ .../iguana/cc/storage/impl/StorageTest.java | 120 ++++ .../storage/impl/TriplestoreStorageTest.java | 70 ++ .../cc/suite/IguanaSuiteParserTest.java | 36 + .../aksw/iguana/cc/tasks/MockupStorage.java | 14 - .../org/aksw/iguana/cc/tasks/MockupTask.java | 15 - .../org/aksw/iguana/cc/tasks/ServerMock.java | 55 -- .../cc/tasks/storage/impl/CSVStorageTest.java | 201 ------ .../tasks/storage/impl/NTFileStorageTest.java | 71 -- .../storage/impl/RDFFileStorageTest.java | 74 -- .../storage/impl/TriplestoreStorageTest.java | 84 --- .../cc/tasks/stresstest/StresstestTest.java | 134 ---- .../cc/utils/CLIProcessManagerTest.java | 64 -- .../aksw/iguana/cc/utils/FileUtilsTest.java | 196 ++---- .../cc/utils/IndexedQueryReaderTest.java | 206 ++++-- .../cc/utils/SPARQLQueryStatisticsTest.java | 62 -- .../org/aksw/iguana/cc/utils/ServerMock.java | 48 -- .../aksw/iguana/cc/worker/HTTPWorkerTest.java | 217 ------ .../aksw/iguana/cc/worker/MockupWorker.java | 56 -- .../iguana/cc/worker/UPDATEWorkerTest.java | 175 ----- .../iguana/cc/worker/WorkerServerMock.java | 145 ---- .../cc/worker/impl/CLIWorkersTests.java | 172 ----- .../cc/worker/impl/HttpPostWorkerTest.java | 51 -- .../cc/worker/impl/RequestFactoryTest.java | 110 +++ .../worker/impl/SPARQLProtocolWorkerTest.java | 258 +++++++ .../factory/AnnotatedFactorizedObject.java | 28 - .../commons/factory/FactorizedObject.java | 56 -- .../commons/factory/TypedFactoryTest.java | 152 ---- .../io/BigByteArrayInputStreamTest.java | 224 ++++++ .../io/BigByteArrayOutputStreamTest.java | 310 +++++++++ .../commons/number/NumberUtilsTest.java | 56 -- .../commons/script/ScriptExecutorTest.java | 79 --- .../script/ScriptExecutorWaitTest.java | 39 -- .../aksw/iguana/commons/utils/ServerMock.java | 55 -- .../config/mockupworkflow-no-default.yml | 2 +- src/test/resources/config/mockupworkflow.yml | 2 +- src/test/resources/controller_test.properties | 4 +- .../dataset1-triplestore1-v1-query.csv | 2 - ...ataset1-triplestore1-v1-worker-query-1.csv | 2 - ...ataset1-triplestore1-v1-worker-query-2.csv | 1 - .../dataset1-triplestore1-v1-worker.csv | 3 - .../storage/csv_test_files/tasks-overview.csv | 2 - .../suite-configs/invalid/invalid-number.yaml | 82 +++ .../suite-configs/invalid/wrong-task.yaml | 82 +++ .../suite-configs/valid/config-full.yaml | 82 +++ .../suite-123/suite-summary.csv | 3 + .../task-0/each-execution-worker-0.csv | 31 + .../task-0/each-execution-worker-1.csv | 31 + .../task-0/each-execution-worker-2.csv | 31 + .../task-0/each-execution-worker-3.csv | 31 + .../suite-123/task-0/query-summary-task.csv | 21 + .../task-0/query-summary-worker-0.csv | 11 + .../task-0/query-summary-worker-1.csv | 11 + .../task-0/query-summary-worker-2.csv | 11 + .../task-0/query-summary-worker-3.csv | 11 + .../suite-123/task-0/worker-summary.csv | 5 + .../task-1/each-execution-worker-0.csv | 16 + .../task-1/each-execution-worker-1.csv | 16 + .../task-1/each-execution-worker-2.csv | 16 + .../task-1/each-execution-worker-3.csv | 16 + .../suite-123/task-1/query-summary-task.csv | 11 + .../task-1/query-summary-worker-0.csv | 6 + .../task-1/query-summary-worker-1.csv | 6 + .../task-1/query-summary-worker-2.csv | 6 + .../task-1/query-summary-worker-3.csv | 6 + .../suite-123/task-1/worker-summary.csv | 5 + .../suite-123/task-configuration.csv | 5 + 218 files changed, 7377 insertions(+), 10250 deletions(-) delete mode 100644 src/main/java/org/aksw/iguana/cc/config/CONSTANTS.java delete mode 100644 src/main/java/org/aksw/iguana/cc/config/ConfigManager.java delete mode 100644 src/main/java/org/aksw/iguana/cc/config/IguanaConfig.java delete mode 100644 src/main/java/org/aksw/iguana/cc/config/IguanaConfigFactory.java delete mode 100644 src/main/java/org/aksw/iguana/cc/config/elements/MetricConfig.java delete mode 100644 src/main/java/org/aksw/iguana/cc/config/elements/TaskConfig.java delete mode 100644 src/main/java/org/aksw/iguana/cc/controller/TaskController.java delete mode 100644 src/main/java/org/aksw/iguana/cc/lang/AbstractLanguageProcessor.java delete mode 100644 src/main/java/org/aksw/iguana/cc/lang/QueryWrapper.java delete mode 100644 src/main/java/org/aksw/iguana/cc/lang/impl/RDFLanguageProcessor.java delete mode 100644 src/main/java/org/aksw/iguana/cc/lang/impl/SPARQLLanguageProcessor.java delete mode 100644 src/main/java/org/aksw/iguana/cc/lang/impl/ThrowawayLanguageProcessor.java create mode 100644 src/main/java/org/aksw/iguana/cc/metrics/Metric.java create mode 100644 src/main/java/org/aksw/iguana/cc/metrics/ModelWritingMetric.java create mode 100644 src/main/java/org/aksw/iguana/cc/metrics/QueryMetric.java create mode 100644 src/main/java/org/aksw/iguana/cc/metrics/TaskMetric.java create mode 100644 src/main/java/org/aksw/iguana/cc/metrics/WorkerMetric.java create mode 100644 src/main/java/org/aksw/iguana/cc/metrics/impl/AggregatedExecutionStatistics.java create mode 100644 src/main/java/org/aksw/iguana/cc/metrics/impl/AvgQPS.java create mode 100644 src/main/java/org/aksw/iguana/cc/metrics/impl/EachExecutionStatistic.java create mode 100644 src/main/java/org/aksw/iguana/cc/metrics/impl/NoQ.java create mode 100644 src/main/java/org/aksw/iguana/cc/metrics/impl/NoQPH.java rename src/main/java/org/aksw/iguana/cc/{tasks/stresstest => }/metrics/impl/PAvgQPS.java (52%) create mode 100644 src/main/java/org/aksw/iguana/cc/metrics/impl/PQPS.java create mode 100644 src/main/java/org/aksw/iguana/cc/metrics/impl/QMPH.java rename src/main/java/org/aksw/iguana/cc/{tasks/stresstest => }/metrics/impl/QPS.java (56%) delete mode 100644 src/main/java/org/aksw/iguana/cc/model/QueryExecutionStats.java delete mode 100644 src/main/java/org/aksw/iguana/cc/model/QueryResultHashKey.java delete mode 100644 src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java create mode 100644 src/main/java/org/aksw/iguana/cc/storage/Storable.java create mode 100644 src/main/java/org/aksw/iguana/cc/storage/Storage.java create mode 100644 src/main/java/org/aksw/iguana/cc/storage/impl/CSVStorage.java create mode 100644 src/main/java/org/aksw/iguana/cc/storage/impl/RDFFileStorage.java rename src/main/java/org/aksw/iguana/cc/{tasks/stresstest => }/storage/impl/TriplestoreStorage.java (56%) create mode 100644 src/main/java/org/aksw/iguana/cc/suite/IguanaSuiteParser.java create mode 100644 src/main/java/org/aksw/iguana/cc/suite/Suite.java delete mode 100644 src/main/java/org/aksw/iguana/cc/tasks/AbstractTask.java delete mode 100644 src/main/java/org/aksw/iguana/cc/tasks/TaskFactory.java delete mode 100644 src/main/java/org/aksw/iguana/cc/tasks/TaskManager.java create mode 100644 src/main/java/org/aksw/iguana/cc/tasks/impl/Stresstest.java create mode 100644 src/main/java/org/aksw/iguana/cc/tasks/impl/StresstestResultProcessor.java delete mode 100644 src/main/java/org/aksw/iguana/cc/tasks/stresstest/Stresstest.java delete mode 100644 src/main/java/org/aksw/iguana/cc/tasks/stresstest/StresstestMetadata.java delete mode 100644 src/main/java/org/aksw/iguana/cc/tasks/stresstest/StresstestResultProcessor.java delete mode 100644 src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/Metric.java delete mode 100644 src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/MetricManager.java delete mode 100644 src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/ModelWritingMetric.java delete mode 100644 src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/QueryMetric.java delete mode 100644 src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/TaskMetric.java delete mode 100644 src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/WorkerMetric.java delete mode 100644 src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/AggregatedExecutionStatistics.java delete mode 100644 src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/AvgQPS.java delete mode 100644 src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/EachExecutionStatistic.java delete mode 100644 src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/NoQ.java delete mode 100644 src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/NoQPH.java delete mode 100644 src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/PQPS.java delete mode 100644 src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/QMPH.java delete mode 100644 src/main/java/org/aksw/iguana/cc/tasks/stresstest/storage/Storage.java delete mode 100644 src/main/java/org/aksw/iguana/cc/tasks/stresstest/storage/StorageManager.java delete mode 100644 src/main/java/org/aksw/iguana/cc/tasks/stresstest/storage/TripleBasedStorage.java delete mode 100644 src/main/java/org/aksw/iguana/cc/tasks/stresstest/storage/impl/CSVStorage.java delete mode 100644 src/main/java/org/aksw/iguana/cc/tasks/stresstest/storage/impl/NTFileStorage.java delete mode 100644 src/main/java/org/aksw/iguana/cc/tasks/stresstest/storage/impl/RDFFileStorage.java delete mode 100644 src/main/java/org/aksw/iguana/cc/utils/CLIProcessManager.java delete mode 100644 src/main/java/org/aksw/iguana/cc/utils/ResultSizeRetriever.java delete mode 100644 src/main/java/org/aksw/iguana/cc/utils/SPARQLQueryStatistics.java delete mode 100644 src/main/java/org/aksw/iguana/cc/utils/StatisticsVisitor.java delete mode 100644 src/main/java/org/aksw/iguana/cc/worker/AbstractWorker.java create mode 100644 src/main/java/org/aksw/iguana/cc/worker/HttpWorker.java delete mode 100644 src/main/java/org/aksw/iguana/cc/worker/LatencyStrategy.java create mode 100644 src/main/java/org/aksw/iguana/cc/worker/ResponseBodyProcessor.java create mode 100644 src/main/java/org/aksw/iguana/cc/worker/ResponseBodyProcessorInstances.java delete mode 100644 src/main/java/org/aksw/iguana/cc/worker/Worker.java delete mode 100644 src/main/java/org/aksw/iguana/cc/worker/WorkerFactory.java delete mode 100644 src/main/java/org/aksw/iguana/cc/worker/WorkerMetadata.java delete mode 100644 src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputFileWorker.java delete mode 100644 src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputPrefixWorker.java delete mode 100644 src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputWorker.java delete mode 100644 src/main/java/org/aksw/iguana/cc/worker/impl/CLIWorker.java delete mode 100644 src/main/java/org/aksw/iguana/cc/worker/impl/HttpGetWorker.java delete mode 100644 src/main/java/org/aksw/iguana/cc/worker/impl/HttpPostWorker.java delete mode 100644 src/main/java/org/aksw/iguana/cc/worker/impl/HttpWorker.java delete mode 100644 src/main/java/org/aksw/iguana/cc/worker/impl/MultipleCLIInputWorker.java create mode 100644 src/main/java/org/aksw/iguana/cc/worker/impl/SPARQLProtocolWorker.java delete mode 100644 src/main/java/org/aksw/iguana/cc/worker/impl/UPDATEWorker.java delete mode 100644 src/main/java/org/aksw/iguana/cc/worker/impl/update/UpdateTimer.java delete mode 100644 src/main/java/org/aksw/iguana/commons/annotation/Nullable.java delete mode 100644 src/main/java/org/aksw/iguana/commons/annotation/ParameterNames.java delete mode 100644 src/main/java/org/aksw/iguana/commons/annotation/Shorthand.java delete mode 100644 src/main/java/org/aksw/iguana/commons/constants/COMMON.java delete mode 100644 src/main/java/org/aksw/iguana/commons/factory/TypedFactory.java delete mode 100644 src/main/java/org/aksw/iguana/commons/numbers/NumberUtils.java delete mode 100644 src/main/java/org/aksw/iguana/commons/reflect/ShorthandMapper.java delete mode 100644 src/main/java/org/aksw/iguana/commons/script/ScriptExecutor.java delete mode 100644 src/main/java/org/aksw/iguana/commons/streams/Streams.java delete mode 100644 src/test/java/org/aksw/iguana/cc/config/ConfigTest.java delete mode 100644 src/test/java/org/aksw/iguana/cc/config/WorkflowTest.java create mode 100644 src/test/java/org/aksw/iguana/cc/config/elements/ConnectionConfigTest.java create mode 100644 src/test/java/org/aksw/iguana/cc/config/elements/DatasetConfigTest.java create mode 100644 src/test/java/org/aksw/iguana/cc/config/elements/StorageConfigTest.java delete mode 100644 src/test/java/org/aksw/iguana/cc/lang/MockCloseableHttpResponse.java delete mode 100644 src/test/java/org/aksw/iguana/cc/lang/RDFLanguageProcessorTest.java delete mode 100644 src/test/java/org/aksw/iguana/cc/lang/SPARQLLanguageProcessorTest.java create mode 100644 src/test/java/org/aksw/iguana/cc/mockup/MockupConnection.java create mode 100644 src/test/java/org/aksw/iguana/cc/mockup/MockupQueryHandler.java create mode 100644 src/test/java/org/aksw/iguana/cc/mockup/MockupStorage.java create mode 100644 src/test/java/org/aksw/iguana/cc/mockup/MockupWorker.java delete mode 100644 src/test/java/org/aksw/iguana/cc/model/QueryResultHashKeyTest.java create mode 100644 src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerConfigTest.java create mode 100644 src/test/java/org/aksw/iguana/cc/query/list/QueryListTest.java delete mode 100644 src/test/java/org/aksw/iguana/cc/query/pattern/PatternBasedQueryHandlerTest.java delete mode 100644 src/test/java/org/aksw/iguana/cc/query/pattern/PatternHandlerTest.java create mode 100644 src/test/java/org/aksw/iguana/cc/query/selector/impl/RandomQuerySelectorTest.java create mode 100644 src/test/java/org/aksw/iguana/cc/storage/impl/CSVStorageTest.java create mode 100644 src/test/java/org/aksw/iguana/cc/storage/impl/RDFFileStorageTest.java create mode 100644 src/test/java/org/aksw/iguana/cc/storage/impl/StorageTest.java create mode 100644 src/test/java/org/aksw/iguana/cc/storage/impl/TriplestoreStorageTest.java create mode 100644 src/test/java/org/aksw/iguana/cc/suite/IguanaSuiteParserTest.java delete mode 100644 src/test/java/org/aksw/iguana/cc/tasks/MockupStorage.java delete mode 100644 src/test/java/org/aksw/iguana/cc/tasks/MockupTask.java delete mode 100644 src/test/java/org/aksw/iguana/cc/tasks/ServerMock.java delete mode 100644 src/test/java/org/aksw/iguana/cc/tasks/storage/impl/CSVStorageTest.java delete mode 100644 src/test/java/org/aksw/iguana/cc/tasks/storage/impl/NTFileStorageTest.java delete mode 100644 src/test/java/org/aksw/iguana/cc/tasks/storage/impl/RDFFileStorageTest.java delete mode 100644 src/test/java/org/aksw/iguana/cc/tasks/storage/impl/TriplestoreStorageTest.java delete mode 100644 src/test/java/org/aksw/iguana/cc/tasks/stresstest/StresstestTest.java delete mode 100644 src/test/java/org/aksw/iguana/cc/utils/CLIProcessManagerTest.java delete mode 100644 src/test/java/org/aksw/iguana/cc/utils/SPARQLQueryStatisticsTest.java delete mode 100644 src/test/java/org/aksw/iguana/cc/utils/ServerMock.java delete mode 100644 src/test/java/org/aksw/iguana/cc/worker/HTTPWorkerTest.java delete mode 100644 src/test/java/org/aksw/iguana/cc/worker/MockupWorker.java delete mode 100644 src/test/java/org/aksw/iguana/cc/worker/UPDATEWorkerTest.java delete mode 100644 src/test/java/org/aksw/iguana/cc/worker/WorkerServerMock.java delete mode 100644 src/test/java/org/aksw/iguana/cc/worker/impl/CLIWorkersTests.java delete mode 100644 src/test/java/org/aksw/iguana/cc/worker/impl/HttpPostWorkerTest.java create mode 100644 src/test/java/org/aksw/iguana/cc/worker/impl/RequestFactoryTest.java create mode 100644 src/test/java/org/aksw/iguana/cc/worker/impl/SPARQLProtocolWorkerTest.java delete mode 100644 src/test/java/org/aksw/iguana/commons/factory/AnnotatedFactorizedObject.java delete mode 100644 src/test/java/org/aksw/iguana/commons/factory/FactorizedObject.java delete mode 100644 src/test/java/org/aksw/iguana/commons/factory/TypedFactoryTest.java create mode 100644 src/test/java/org/aksw/iguana/commons/io/BigByteArrayInputStreamTest.java create mode 100644 src/test/java/org/aksw/iguana/commons/io/BigByteArrayOutputStreamTest.java delete mode 100644 src/test/java/org/aksw/iguana/commons/number/NumberUtilsTest.java delete mode 100644 src/test/java/org/aksw/iguana/commons/script/ScriptExecutorTest.java delete mode 100644 src/test/java/org/aksw/iguana/commons/script/ScriptExecutorWaitTest.java delete mode 100644 src/test/java/org/aksw/iguana/commons/utils/ServerMock.java delete mode 100644 src/test/resources/storage/csv_test_files/dataset1-triplestore1-v1-query.csv delete mode 100644 src/test/resources/storage/csv_test_files/dataset1-triplestore1-v1-worker-query-1.csv delete mode 100644 src/test/resources/storage/csv_test_files/dataset1-triplestore1-v1-worker-query-2.csv delete mode 100644 src/test/resources/storage/csv_test_files/dataset1-triplestore1-v1-worker.csv delete mode 100644 src/test/resources/storage/csv_test_files/tasks-overview.csv create mode 100644 src/test/resources/suite-configs/invalid/invalid-number.yaml create mode 100644 src/test/resources/suite-configs/invalid/wrong-task.yaml create mode 100644 src/test/resources/suite-configs/valid/config-full.yaml create mode 100644 src/test/resources/test-data/csv-storage-test/suite-123/suite-summary.csv create mode 100644 src/test/resources/test-data/csv-storage-test/suite-123/task-0/each-execution-worker-0.csv create mode 100644 src/test/resources/test-data/csv-storage-test/suite-123/task-0/each-execution-worker-1.csv create mode 100644 src/test/resources/test-data/csv-storage-test/suite-123/task-0/each-execution-worker-2.csv create mode 100644 src/test/resources/test-data/csv-storage-test/suite-123/task-0/each-execution-worker-3.csv create mode 100644 src/test/resources/test-data/csv-storage-test/suite-123/task-0/query-summary-task.csv create mode 100644 src/test/resources/test-data/csv-storage-test/suite-123/task-0/query-summary-worker-0.csv create mode 100644 src/test/resources/test-data/csv-storage-test/suite-123/task-0/query-summary-worker-1.csv create mode 100644 src/test/resources/test-data/csv-storage-test/suite-123/task-0/query-summary-worker-2.csv create mode 100644 src/test/resources/test-data/csv-storage-test/suite-123/task-0/query-summary-worker-3.csv create mode 100644 src/test/resources/test-data/csv-storage-test/suite-123/task-0/worker-summary.csv create mode 100644 src/test/resources/test-data/csv-storage-test/suite-123/task-1/each-execution-worker-0.csv create mode 100644 src/test/resources/test-data/csv-storage-test/suite-123/task-1/each-execution-worker-1.csv create mode 100644 src/test/resources/test-data/csv-storage-test/suite-123/task-1/each-execution-worker-2.csv create mode 100644 src/test/resources/test-data/csv-storage-test/suite-123/task-1/each-execution-worker-3.csv create mode 100644 src/test/resources/test-data/csv-storage-test/suite-123/task-1/query-summary-task.csv create mode 100644 src/test/resources/test-data/csv-storage-test/suite-123/task-1/query-summary-worker-0.csv create mode 100644 src/test/resources/test-data/csv-storage-test/suite-123/task-1/query-summary-worker-1.csv create mode 100644 src/test/resources/test-data/csv-storage-test/suite-123/task-1/query-summary-worker-2.csv create mode 100644 src/test/resources/test-data/csv-storage-test/suite-123/task-1/query-summary-worker-3.csv create mode 100644 src/test/resources/test-data/csv-storage-test/suite-123/task-1/worker-summary.csv create mode 100644 src/test/resources/test-data/csv-storage-test/suite-123/task-configuration.csv diff --git a/docs/develop/extend-metrics.md b/docs/develop/extend-metrics.md index 43542dd7e..76fffcc92 100644 --- a/docs/develop/extend-metrics.md +++ b/docs/develop/extend-metrics.md @@ -46,7 +46,6 @@ The following gives you an examples on how to work with the `data` parameter: } @Override - @Nonnull public Model createMetricModel(StresstestMetadata task, Map> data) { for (String queryID : task.queryIDS()) { // This list contains every query execution statistics of one query from diff --git a/docs/develop/extend-queryhandling.md b/docs/develop/extend-queryhandling.md index f4bb6150f..0a59c8e7c 100644 --- a/docs/develop/extend-queryhandling.md +++ b/docs/develop/extend-queryhandling.md @@ -66,7 +66,7 @@ implements the following methods: public class MyQuerySource extends QuerySource { public MyQuerySource(String filepath) { // your constructor - // filepath is the value, specified in the "location"-key inside the configuration file + // filepath is the value, specified in the "path"-key inside the configuration file } @Override diff --git a/docs/usage/configuration.md b/docs/usage/configuration.md index bec58f4f1..ad3ed5613 100644 --- a/docs/usage/configuration.md +++ b/docs/usage/configuration.md @@ -316,14 +316,14 @@ For example, instead of: ```yaml storages: - - className: "org.aksw.iguana.rp.storage.impl.NTFileStorage" + - className: "org.aksw.iguana.rp.storage.impl.RDFFileStorage" ``` you can use the shortname NTFileStorage: ```yaml storages: - - className: "NTFileStorage" + - className: "RDFFileStorage" ``` diff --git a/example-suite.yml b/example-suite.yml index eef144436..e81ff4b7e 100644 --- a/example-suite.yml +++ b/example-suite.yml @@ -7,6 +7,7 @@ connections: user: "dba" password: "dba" endpoint: "http://localhost:8890/sparql" + dataset: DatasetName - name: "Virtuoso6" user: "dba" password: "dba" @@ -19,49 +20,63 @@ connections: updateEndpoint: "http://localhost:3030/ds/update" tasks: - - className: "org.aksw.iguana.cc.tasks.impl.Stresstest" - configuration: - # 1 hour (time Limit is in ms) - timeLimit: 360000 - # warmup is optional - warmup: - # 1 minutes (is in ms) - timeLimit: 600000 - workers: - - threads: 1 - className: "HttpGetWorker" - queries: - location: "queries_warmup.txt" - timeOut: 180000 - workers: - - threads: 16 - className: "HttpGetWorker" - queries: - location: "queries_easy.txt" - timeOut: 180000 - - threads: 4 - className: "HttpGetWorker" - queries: - location: "queries_complex.txt" - fixedLatency: 100 - gaussianLatency: 50 - parameterName: "query" - responseType: "application/sparql-results+json" - -# both are optional and can be used to load and start as well as stop the connection before and after every task -preScriptHook: "./triplestores/{{connection}}/start.sh {{dataset.file}} {{dataset.name}} {{taskID}}" -postScriptHook: "./triplestores/{{connection}}/stop.sh" - -#optional otherwise the same metrics will be used as default -metrics: - - className: "QMPH" - - className: "QPS" - - className: "NoQPH" - - className: "AvgQPS" - - className: "NoQ" + # 1 hour (time Limit is in ms) + - type: stresstest + warmupWorkers: + # 1 minutes (is in ms) + - type: SPARQLProtocolWorker + number: 16 + queries: + path: "/home/bigerl/IdeaProjects/IGUANA/LICENSE" + timeout: 0.02s + connection: Virtuoso7 + completionTarget: + duration: 1000s + workers: + - type: "SPARQLProtocolWorker" + number: 16 + queries: + path: "/home/bigerl/IdeaProjects/IGUANA/LICENSE" + timeout: 3m + connection: Virtuoso7 + completionTarget: + duration: 1000s + requestType: get query + - number: 4 + type: "SPARQLProtocolWorker" + connection: Virtuoso7 + completionTarget: + duration: 1000s + queries: + path: "/home/bigerl/IdeaProjects/IGUANA/LICENSE" + timeout: 100s + acceptHeader: "application/sparql-results+json" + - type: stresstest + workers: + - type: "SPARQLProtocolWorker" + connection: Virtuoso7 + number: 16 + requestType: get query + queries: + path: "/home/bigerl/IdeaProjects/IGUANA/LICENSE" + timeout: 180s + completionTarget: + duration: 1000s + - number: 4 + requestType: get query + connection: Virtuoso7 + completionTarget: + duration: 1000s + type: "SPARQLProtocolWorker" + queries: + path: "/home/bigerl/IdeaProjects/IGUANA/LICENSE" + timeout: 100s + parseResults: true + acceptHeader: "application/sparql-results+json" #optional otherwise an nt file will be used storages: - - className: "NTFileStorage" + - type: "RDF file" + path: "some.ttl" #configuration: #fileName: YOUR_RESULT_FILE_NAME.nt \ No newline at end of file diff --git a/pom.xml b/pom.xml index bcb8e56ee..755e70146 100644 --- a/pom.xml +++ b/pom.xml @@ -46,7 +46,7 @@ 17 17 - 2.17.1 + 2.19.0 @@ -78,27 +78,11 @@ jena-querybuilder ${jena.version} - - junit - junit - 4.13.1 - test - org.apache.httpcomponents httpclient 4.5.13 - - commons-configuration - commons-configuration - 1.10 - - - org.apache.commons - commons-exec - 1.3 - org.apache.logging.log4j log4j-slf4j-impl @@ -119,25 +103,15 @@ log4j-1.2-api ${log4j.version} - - org.simpleframework - simple - 5.1.6 - - - org.reflections - reflections - 0.9.9 - - - commons-codec - commons-codec - 1.15 - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - 2.11.2 + 2.12.5 + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + 2.12.5 com.networknt @@ -161,17 +135,47 @@ 5.9.2 test - - org.junit.vintage - junit-vintage-engine - 5.9.2 - test - com.opencsv opencsv 5.7.1 + + org.lz4 + lz4-pure-java + 1.8.0 + + + org.apache.hbase + hbase-common + 2.5.5 + + + com.beust + jcommander + 1.82 + + + com.github.tomakehurst + wiremock-jre8-standalone + 2.35.0 + test + + + org.apache.maven.plugins + maven-surefire-plugin + 3.1.2 + + + org.springframework.data + spring-data-commons + 3.1.2 + + + org.springframework + spring-context + 6.0.11 + @@ -194,6 +198,16 @@ + + org.apache.maven.plugins + maven-surefire-plugin + 3.1.2 + + -Xmx16384M + + + + org.apache.maven.plugins maven-shade-plugin @@ -210,7 +224,8 @@ - + org.aksw.iguana.cc.controller.MainController diff --git a/schema/iguana-schema.json b/schema/iguana-schema.json index 71b45a045..d2e12eff2 100644 --- a/schema/iguana-schema.json +++ b/schema/iguana-schema.json @@ -1,367 +1,389 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - + "$schema": "http://json-schema.org/draft-06/schema#", + "$ref": "#/definitions/root", "definitions": { - "connection": { + "root": { + "title": "root", "type": "object", + "additionalProperties": false, "properties": { - "endpoint": {"type": "string"}, - "updateEndpoint": {"type": "string"}, - "user": {"type": "string"}, - "password": {"type": "string"}, - "version": {"type": "string"} + "datasets": { + "type": "array", + "items": { + "$ref": "#/definitions/Dataset" + }, + "minItems": 1 + }, + "connections": { + "type": "array", + "items": { + "$ref": "#/definitions/Connection" + }, + "minItems": 1 + }, + "tasks": { + "type": "array", + "items": { + "$ref": "#/definitions/Task" + }, + "minItems": 1 + }, + "storages": { + "type": "array", + "items": { + "$ref": "#/definitions/Storage" + }, + "minItems": 1 + }, + "responseBodyProcessors": { + "type": "array", + "items": { + "$ref": "#/definitions/ResponseBodyProcessor" + } + }, + "metrics": { + "type": "array", + "items": { + "$ref": "#/definitions/Metric" + } + } }, - "required": ["endpoint"] + "required": [ + "connections", + "datasets", + "storages", + "tasks" + ] }, - "queries": { + "Connection": { "type": "object", + "additionalProperties": false, "properties": { - "location": {"type": "string"}, - "format": { - "oneOf": [ - {"type": "string"}, - { - "type": "object", - "properties": { - "separator": {"type": "string"} - } - } - ] - }, - "caching": {"type": "boolean"}, - "order": { - "oneOf": [ - {"type": "string"}, - { - "type": "object", - "properties": { - "random": { - "type": "object", - "properties": { - "seed": {"type": "integer"} - }, - "required": ["seed"] - } - } - } - ] - }, - "pattern": { - "type": "object", - "properties": { - "endpoint": {"type": "string"}, - "outputFolder": {"type": "string"}, - "limit": {"type": "integer"} - }, - "required": ["endpoint"] + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "endpoint": { + "type": "string", + "format": "uri" + }, + "updateEndpoint": { + "type": "string", + "format": "uri" + }, + "authentication": { + "$ref": "#/definitions/Authentication" + }, + "updateAuthentication": { + "$ref": "#/definitions/Authentication" }, - "lang": {"type": "string"} + "dataset": { + "type": "string" + } }, - "required": ["location"] + "required": [ + "endpoint", + "name" + ], + "title": "Connection" }, - - "warmup": { + "Authentication": { "type": "object", + "additionalProperties": false, "properties": { - "timeLimit": {"type": "integer"}, - "workers": { - "type": "array", - "items": { - "oneOf": [{"$ref": "#/definitions/AbstractWorker"}] - } + "user": { + "type": "string" + }, + "password": { + "type": "string" } }, - "required": ["workers", "timeLimit"] + "required": [ + "password", + "user" + ], + "title": "Authentication" }, - - "stresstest": { + "Dataset": { "type": "object", + "additionalProperties": false, "properties": { - "timeLimit": {"type": "integer"}, - "noOfQueryMixes": {"type": "integer"}, - "warmup": {"$ref": "#/definitions/warmup"}, - "workers": { - "type": "array", - "items": { - "oneOf": [{"$ref": "#/definitions/AbstractWorker"}] - } + "name": { + "type": "string" + }, + "file": { + "type": "string" } }, - "required": ["workers"] + "required": [ + "name" + ], + "title": "Dataset" }, - - "AbstractWorker": { + "Metric": { "type": "object", + "additionalProperties": false, "properties": { - "className": {"type": "string"} + "type": { + "type": "string", + "enum": [ "AES", "AvgQPS", "EachQuery", "NoQ", "NoQPH", "PAvgQPS", "PQPS", "QMPH", "QPS" ] + }, + "penalty": { + "type": "integer", + "minimum": 0 + } }, - "allOf": [ - { - "if": { - "properties": { - "className" : { - "oneOf": [ {"const": "SPARQLWorker"},{"const": "org.aksw.iguana.cc.worker.impl.SPARQLWorker"}] - } - } - }, - "then": { - "properties": { - "className": {"type": "string"}, - "threads": {"type": "integer"}, - "timeOut": {"type": "integer"}, - "fixedLatency": {"type": "integer"}, - "gaussianLatency": {"type": "integer"}, - "responseType": {"type": "string"}, - "parameterName": {"type": "string"}, - "queries": {"$ref": "#/definitions/queries"} - }, - "additionalProperties": {"type": "null"}, - "required": ["className", "threads", "queries"] - } + "required": [ + "type" + ], + "title": "Metric" + }, + "ResponseBodyProcessor": { + "type": "object", + "additionalProperties": false, + "properties": { + "contentType": { + "type": "string" }, - { - "if": { - "properties": { - "className" : { - "oneOf": [{"const": "UPDATEWorker"},{"const": "org.aksw.iguana.cc.worker.impl.UPDATEWorker"}] - } - } - }, - "then": { - "properties": { - "className": {"type": "string"}, - "threads": {"type": "integer"}, - "timeOut": {"type": "integer"}, - "fixedLatency": {"type": "integer"}, - "gaussianLatency": {"type": "integer"}, - "timerStrategy": {"type": "string"}, - "queries": {"$ref": "#/definitions/queries"} - }, - "additionalProperties": {"type": "null"}, - "required": ["className", "threads", "queries"] - } + "threads": { + "type": "integer", + "minimum": 1 + } + }, + "required": [ + "contentType", + "threads" + ], + "title": "ResponseBodyProcessor" + }, + "Storage": { + "type": "object", + "oneOf": [ + { "$ref": "#/definitions/CSVStorage" }, + { "$ref": "#/definitions/RDFFileStorage" }, + { "$ref": "#/definitions/TriplestoreStorage" } + ], + "title": "Storage" + }, + "CSVStorage": { + "type": "object", + "unevaluatedProperties": false, + "properties": { + "type": { + "type": "string", + "const": "csv file" }, - { - "if": { - "properties": { - "className" : { - "oneOf": [{"const": "MultipleCLIInputWorker"}, {"const": "org.aksw.iguana.cc.worker.impl.MultipleCLIInputWorker"}] - } - }}, - "then": { - "properties": { - "className": {"type": "string"}, - "threads": {"type": "integer"}, - "timeOut": {"type": "integer"}, - "fixedLatency": {"type": "integer"}, - "gaussianLatency": {"type": "integer"}, - "queryError": {"type": "string"}, - "queryFinished": {"type": "string"}, - "initFinished": {"type": "string"}, - "numberOfProcesses": {"type": "integer"}, - "queries": {"$ref": "#/definitions/queries"} - }, - "additionalProperties": {"type": "null"}, - "required": ["className", "threads", "queryError", "queryFinished", "initFinished", "queries"] - } + "directory": { + "type": "string" + } + }, + "required": [ + "type", + "directory" + ], + "title": "CSVStorage" + }, + "RDFFileStorage": { + "type": "object", + "unevaluatedProperties": false, + "properties": { + "type": { + "type": "string", + "const": "rdf file" }, - { - "if": { - "properties": { - "className" : { - "oneOf": [{"const": "CLIInputWorker"}, {"const": "org.aksw.iguana.cc.worker.impl.CLIInputWorker"}] - } - } - }, - "then": { - "properties": { - "className": {"type": "string"}, - "threads": {"type": "integer"}, - "timeOut": {"type": "integer"}, - "fixedLatency": {"type": "integer"}, - "gaussianLatency": {"type": "integer"}, - "queryError": {"type": "string"}, - "queryFinished": {"type": "string"}, - "initFinished": {"type": "string"}, - "queries": {"$ref": "#/definitions/queries"} - }, - "additionalProperties": {"type": "null"}, - "required": ["className", "threads", "queryError", "queryFinished", "initFinished", "queries"] - } + "path": { + "type": "string" + } + }, + "required": [ + "type", + "path" + ], + "title": "RDFFileStorage" + }, + "TriplestoreStorage": { + "type": "object", + "unevaluatedProperties": false, + "properties": { + "type": { + "type": "string", + "const": "triplestore" }, - { - "if": { - "properties": { - "className": { - "oneOf": [{"const": "CLIPrefixWorker"}, {"const": "org.aksw.iguana.cc.worker.impl.CLIPrefixWorker"}] - } - } - }, - "then": { - "properties": { - "className": {"type": "string"}, - "threads": {"type": "integer"}, - "timeOut": {"type": "integer"}, - "fixedLatency": {"type": "integer"}, - "gaussianLatency": {"type": "integer"}, - "numberOfProcesses": {"type": "integer"}, - "queryError": {"type": "string"}, - "queryFinished": {"type": "string"}, - "initFinished": {"type": "string"}, - "querySuffix": {"type": "string"}, - "queryPrefix": {"type": "string"}, - "queries": {"$ref": "#/definitions/queries"} - }, - "additionalProperties": {"type": "null"}, - "required": [ - "className", - "threads", - "queryError", - "queryFinished", - "initFinished", - "queryPrefix", - "querySuffix", - "queries" - ] - } + "endpoint": { + "type": "string", + "format": "uri" }, - { - "if": { - "properties": { - "className": { - "oneOf": [{"const": "MultipleCLIInputFileWorker"}, {"const": "org.aksw.iguana.cc.worker.impl.MultipleCLIInputFileWorker"}] - } - } - }, - "then": { - "properties": { - "className": {"type": "string"}, - "threads": {"type": "integer"}, - "timeOut": {"type": "integer"}, - "fixedLatency": {"type": "integer"}, - "gaussianLatency": {"type": "integer"}, - "queryError": {"type": "string"}, - "queryFinished": {"type": "string"}, - "initFinished": {"type": "string"}, - "directory": {"type": "string"}, - "numberOfProcesses": {"type": "integer"}, - "queries": {"$ref": "#/definitions/queries"} - }, - "additionalProperties": {"type": "null"}, - "required": [ - "className", - "threads", - "directory", - "queryError", - "queryFinished", - "initFinished", - "queries" - ] - } + "user": { + "type": "string" }, - { - "if": { - "properties": { - "className": { - "oneOf": [{"const": "CLIInputFileWorker"}, {"const": "org.aksw.iguana.cc.worker.impl.CLIInputFileWorker"}] - } - } - }, - "then": { - "allOf": [ - { - "properties": { - "className": {"type": "string"}, - "threads": {"type": "integer"}, - "timeOut": {"type": "integer"}, - "fixedLatency": {"type": "integer"}, - "gaussianLatency": {"type": "integer"}, - "queryError": {"type": "string"}, - "queryFinished": {"type": "string"}, - "initFinished": {"type": "string"}, - "directory": {"type": "string"}, - "queries": {"$ref": "#/definitions/queries"} - }, - "additionalProperties": {"type": "null"} - }, - { - "required": [ - "className", - "threads", - "directory", - "queryError", - "queryFinished", - "initFinished", - "queries" - ] - } - ] - } + "password": { + "type": "string" + }, + "baseUri": { + "type": "string", + "format": "uri" } - ] + }, + "required": [ + "type", + "endpoint" + ], + "title": "TriplestoreStorage" }, - - "task": { + "Task": { "type": "object", + "oneOf": [ { "$ref": "#/definitions/Stresstest" } ], + "title": "Task" + }, + "Stresstest": { + "type": "object", + "unevaluatedProperties": false, "properties": { - "className": {"type": "string"}, - "configuration": { - "oneOf": [{"$ref": "#/definitions/stresstest"}] + "type": { + "type": "string", + "const": "stresstest" + }, + "warmupWorkers": { + "type": "array", + "items": { + "$ref": "#/definitions/Worker" + } + }, + "workers": { + "type": "array", + "items": { + "$ref": "#/definitions/Worker" + }, + "minItems": 1 } }, - "required": ["className", "configuration"] + "required": [ + "type", + "workers" + ], + "title": "Stresstest" }, - - "genericClassObject": { + "Worker": { + "type": "object", + "oneOf": [ { "$ref": "#/definitions/SPARQLWorker" } ], + "title": "Worker" + }, + "SPARQLWorker" : { "type": "object", + "unevaluatedProperties": false, "properties": { - "className": {"type": "string"}, - "configuration": { - "type": "object" + "type": { + "type": "string", + "const": "SPARQLProtocolWorker" + }, + "number": { + "type": "integer", + "minimum": 1 + }, + "requestType": { + "type": "string", + "enum": [ "post query", "get query", "post url-enc query", "post url-enc update", "post update" ] + }, + "queries": { + "$ref": "#/definitions/Queries" + }, + "timeout": { + "type": "string" + }, + "connection": { + "type": "string" + }, + "completionTarget": { + "$ref": "#/definitions/CompletionTarget" + }, + "parseResults": { + "type": "boolean" + }, + "acceptHeader": { + "type": "string" } }, - "required": ["className"] - } - }, - - "type": "object", - "properties": { - "connections": { - "type": "array", - "items": { - "$ref": "#/definitions/connection" - } + "required": [ + "type", + "completionTarget", + "connection", + "queries", + "timeout" + ], + "title": "SPARQLWorker" }, - "datasets": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name" : {"type": "string"} - }, - "required": ["name"] - } + "CompletionTarget": { + "type": "object", + "oneOf": [ + { "$ref": "#/definitions/TimeLimit" }, + { "$ref": "#/definitions/QueryMixes" } + ], + "title": "CompletionTarget" }, - "tasks": { - "type": "array", - "items": { - "$ref":"#/definitions/task" - } + "TimeLimit": { + "properties": { + "duration": { + "type": "string" + } + }, + "title": "TimeLimit", + "type": "object", + "unevaluatedProperties": false, + "required": [ + "duration" + ] }, - "preScriptHook": {"type": "string"}, - "postScriptHook": {"type": "string"}, - "metrics": { - "type": "array", - "items": { - "$ref": "#/definitions/genericClassObject" - } + "QueryMixes": { + "properties": { + "number": { + "type": "integer", + "minimum": 1 + } + }, + "title": "QueryMixes", + "type": "object", + "unevaluatedProperties": false, + "required": [ + "number" + ] }, - "storages": { - "type": "array", - "items": { - "$ref": "#/definitions/genericClassObject" - } + "Queries": { + "type": "object", + "additionalProperties": false, + "properties": { + "path": { + "type": "string" + }, + "format": { + "type": "string", + "enum": [ "one-per-line", "separator", "folder" ] + }, + "separator": { + "type": "string" + }, + "caching": { + "type": "boolean" + }, + "order": { + "type": "string", + "enum": [ "random", "linear" ] + }, + "seed": { + "type": "integer" + }, + "lang": { + "type": "string", + "enum": [ "", "SPARQL" ] + } + }, + "required": [ + "path" + + ], + "title": "Queries" } } } diff --git a/src/main/java/org/aksw/iguana/cc/config/CONSTANTS.java b/src/main/java/org/aksw/iguana/cc/config/CONSTANTS.java deleted file mode 100644 index 4b3c7ebca..000000000 --- a/src/main/java/org/aksw/iguana/cc/config/CONSTANTS.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.aksw.iguana.cc.config; - -/** - * Constants used only by the Core controller - * - * @author f.conrads - * - */ -public class CONSTANTS { - - /** - * The key to set the workerID in the Extra Meta properties - * and the properties name in the final results to get the workerID - */ - public static final String WORKER_ID_KEY = "workerID"; - - /** - * The key to set the workerType in the Extra Meta properties - * and the properties name in the final results to get the workerType - */ - public static final String WORKER_TYPE_KEY = "workerType"; - - /** - * The key to get the timeLimit parameter. - * be aware that timeLimit can be null. - */ - public static final String TIME_LIMIT = "timeLimit"; - - - public static final String NO_OF_QUERY_MIXES = "numberOfQueryMixes"; - - - public static final String WORKER_TIMEOUT_MS = "timeOutMS"; -} diff --git a/src/main/java/org/aksw/iguana/cc/config/ConfigManager.java b/src/main/java/org/aksw/iguana/cc/config/ConfigManager.java deleted file mode 100644 index b0946ba0d..000000000 --- a/src/main/java/org/aksw/iguana/cc/config/ConfigManager.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * - */ -package org.aksw.iguana.cc.config; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; - - -/** - * Manages an incoming Configuration and starts the corresponding {@link org.aksw.iguana.cc.config.IguanaConfig} - * - * @author f.conrads - * - */ -public class ConfigManager { - - private Logger LOGGER = LoggerFactory.getLogger(getClass()); - - - /** - * Will receive a JSON or YAML configuration and executes the configuration as an Iguana Suite - * @param configuration - * @param validate checks if error should be thrown if it validates the configuration given the iguana-schema.json schema - */ - public void receiveData(File configuration, Boolean validate) throws IOException { - - IguanaConfig newConfig = IguanaConfigFactory.parse(configuration, validate); - if(newConfig==null){ - return; - } - startConfig(newConfig); - } - - - - /** - * Starts the Config - */ - public void startConfig(IguanaConfig config) { - try { - config.start(); - } catch (IOException e) { - LOGGER.error("Could not start config due to an IO Exception", e); - } - - } - - - -} diff --git a/src/main/java/org/aksw/iguana/cc/config/IguanaConfig.java b/src/main/java/org/aksw/iguana/cc/config/IguanaConfig.java deleted file mode 100644 index e2d16b112..000000000 --- a/src/main/java/org/aksw/iguana/cc/config/IguanaConfig.java +++ /dev/null @@ -1,174 +0,0 @@ -package org.aksw.iguana.cc.config; - -import com.fasterxml.jackson.annotation.JsonProperty; -import org.aksw.iguana.cc.config.elements.*; -import org.aksw.iguana.cc.controller.TaskController; -import org.aksw.iguana.cc.tasks.stresstest.metrics.Metric; -import org.aksw.iguana.cc.tasks.stresstest.metrics.MetricManager; -import org.aksw.iguana.cc.tasks.stresstest.metrics.impl.*; -import org.aksw.iguana.cc.tasks.stresstest.storage.StorageManager; -import org.aksw.iguana.commons.script.ScriptExecutor; -import org.aksw.iguana.cc.tasks.stresstest.storage.Storage; -import org.aksw.iguana.cc.tasks.stresstest.storage.impl.NTFileStorage; -import org.apache.commons.exec.ExecuteException; -import org.apache.commons.lang3.SerializationUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.time.Instant; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Gets either a JSON or YAML configuration file using a json schema and will generate - * a SuiteID and ExperimentIDs as well as TaskIDs for it.
- * Afterwards it will start the taskProcessor with all specified tasks - *

- * The following order holds - *
    - *
  1. For each Dataset
    2. - *
  2. For each Connection
    3. - *
  3. For each Task
    4. - *
- * - * Further on executes the pre and post script hooks, before and after a class. - * Following values will be exchanged in the script string {{Connection}} {{Dataset.name}} {{Dataset.file}} {{taskID}} - * - * @author f.conrads - * - */ -public class IguanaConfig { - - private static final Logger LOGGER = LoggerFactory.getLogger(IguanaConfig.class); - - @JsonProperty(required = true) - private List datasets; - @JsonProperty(required = true) - private List connections; - @JsonProperty(required = true) - private List tasks; - @JsonProperty - private String preScriptHook; - @JsonProperty - private String postScriptHook; - @JsonProperty - private List metrics; - @JsonProperty - private List storages; - - private static String suiteID = generateSuiteID(); - - /** - * starts the config - * @throws IOException - * @throws ExecuteException - */ - public void start() throws ExecuteException, IOException { - initResultProcessor(); - TaskController controller = new TaskController(); - //get SuiteID - suiteID = generateSuiteID(); - //generate ExpID - int expID = 0; - - for(DatasetConfig dataset: datasets){ - expID++; - Integer taskID = 0; - for(ConnectionConfig con : connections){ - for(TaskConfig task : tasks) { - taskID++; - String[] args = new String[] {}; - if(preScriptHook!=null){ - LOGGER.info("Executing preScriptHook"); - String execScript = preScriptHook.replace("{{dataset.name}}", dataset.getName()) - .replace("{{connection}}", con.getName()) - .replace("{{connection.version}}", con.getVersion("{{connection.version}}")) - .replace("{{taskID}}", taskID+""); - LOGGER.info("Finished preScriptHook"); - if(dataset.getFile()!=null){ - execScript = execScript.replace("{{dataset.file}}", dataset.getFile()); - } - - ScriptExecutor.execSafe(execScript, args); - } - LOGGER.info("Executing Task [{}/{}: {}, {}, {}]", taskID, task.getName(), dataset.getName(), con.getName(), task.getClassName()); - controller.startTask(new String[]{suiteID, suiteID + "/" + expID, suiteID + "/" + expID + "/" + taskID}, dataset.getName(), SerializationUtils.clone(con), SerializationUtils.clone(task)); - if(postScriptHook!=null){ - String execScript = postScriptHook.replace("{{dataset.name}}", dataset.getName()) - .replace("{{connection}}", con.getName()) - .replace("{{connection.version}}", con.getVersion("{{connection.version}}")) - .replace("{{taskID}}", taskID+""); - if(dataset.getFile()!=null){ - execScript = execScript.replace("{{dataset.file}}", dataset.getFile()); - } - LOGGER.info("Executing postScriptHook {}", execScript); - ScriptExecutor.execSafe(execScript, args); - LOGGER.info("Finished postScriptHook"); - } - } - } - } - - LOGGER.info("Finished benchmark"); - } - - private void initResultProcessor() { - //If storage or metric is empty use default - if(this.storages== null || this.storages.isEmpty()){ - storages = new ArrayList<>(); - StorageConfig config = new StorageConfig(); - config.setClassName(NTFileStorage.class.getCanonicalName()); - storages.add(config); - } - if(this.metrics == null || this.metrics.isEmpty()){ - LOGGER.info("No metrics were set. Using default metrics."); - metrics = new ArrayList<>(); - MetricConfig config = new MetricConfig(); - config.setClassName(QMPH.class.getCanonicalName()); - metrics.add(config); - config = new MetricConfig(); - config.setClassName(QPS.class.getCanonicalName()); - metrics.add(config); - config = new MetricConfig(); - config.setClassName(NoQPH.class.getCanonicalName()); - metrics.add(config); - config = new MetricConfig(); - config.setClassName(AvgQPS.class.getCanonicalName()); - metrics.add(config); - config = new MetricConfig(); - config.setClassName(NoQ.class.getCanonicalName()); - metrics.add(config); - config = new MetricConfig(); - config.setClassName(AggregatedExecutionStatistics.class.getCanonicalName()); - metrics.add(config); - } - - // Create Metrics - // Metrics should be created before the Storages - List metrics = new ArrayList<>(); - for(MetricConfig config : this.metrics) { - metrics.add(config.createMetric()); - } - MetricManager.setMetrics(metrics); - - //Create Storages - List storages = new ArrayList<>(); - for(StorageConfig config : this.storages){ - storages.add(config.createStorage()); - } - StorageManager.getInstance().addStorages(storages); - } - - public static String getSuiteID() { - return suiteID; - } - - private static String generateSuiteID() { - int currentTimeMillisHashCode = Math.abs(Long.valueOf(Instant.now().getEpochSecond()).hashCode()); - return String.valueOf(currentTimeMillisHashCode); - } - -} diff --git a/src/main/java/org/aksw/iguana/cc/config/IguanaConfigFactory.java b/src/main/java/org/aksw/iguana/cc/config/IguanaConfigFactory.java deleted file mode 100644 index 68ec5de87..000000000 --- a/src/main/java/org/aksw/iguana/cc/config/IguanaConfigFactory.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.aksw.iguana.cc.config; - -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import com.networknt.schema.JsonSchema; -import com.networknt.schema.JsonSchemaFactory; -import com.networknt.schema.SpecVersion; -import com.networknt.schema.ValidationMessage; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.util.Set; - -/** - * Creates an IguanaConfig from a given JSON or YAML file, and validates the config using a JSON schema file - */ -public class IguanaConfigFactory { - - private static Logger LOGGER = LoggerFactory.getLogger(IguanaConfigFactory.class); - - private static String schemaFile = "iguana-schema.json"; - - public static IguanaConfig parse(File config) throws IOException { - return parse(config, true); - } - - public static IguanaConfig parse(File config, Boolean validate) throws IOException { - if(config.getName().endsWith(".yml") || config.getName().endsWith(".yaml")){ - return parse(config, new YAMLFactory(), validate); - } - else if(config.getName().endsWith(".json")){ - return parse(config, new JsonFactory(), validate); - } - return null; - } - private static IguanaConfig parse(File config, JsonFactory factory) throws IOException { - return parse(config, factory, true); - } - - private static IguanaConfig parse(File config, JsonFactory factory, Boolean validate) throws IOException { - final ObjectMapper mapper = new ObjectMapper(factory); - if(validate && !validateConfig(config, schemaFile, mapper)){ - return null; - } - return mapper.readValue(config, IguanaConfig.class); - } - - private static boolean validateConfig(File configuration, String schemaFile, ObjectMapper mapper) throws IOException { - JsonSchemaFactory factory = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V7); - InputStream is = Thread.currentThread().getContextClassLoader() - .getResourceAsStream(schemaFile); - JsonSchema schema = factory.getSchema(is); - JsonNode node = mapper.readTree(configuration); - Set errors = schema.validate(node); - if(errors.size()>0){ - LOGGER.error("Found {} errors in configuration file.", errors.size()); - } - for(ValidationMessage message : errors){ - LOGGER.error(message.getMessage()); - } - return errors.size()==0; - } - -} diff --git a/src/main/java/org/aksw/iguana/cc/config/elements/ConnectionConfig.java b/src/main/java/org/aksw/iguana/cc/config/elements/ConnectionConfig.java index b53d5f71b..99addab0c 100644 --- a/src/main/java/org/aksw/iguana/cc/config/elements/ConnectionConfig.java +++ b/src/main/java/org/aksw/iguana/cc/config/elements/ConnectionConfig.java @@ -1,78 +1,38 @@ package org.aksw.iguana.cc.config.elements; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import java.io.Serializable; +import java.io.IOException; +import java.net.URI; /** * A connection configuration class */ -public class ConnectionConfig implements Serializable { - - @JsonProperty(required = true) - private String name; - @JsonProperty(required = false) - private String user; - @JsonProperty(required = false) - private String password; - @JsonProperty(required = true) - private String endpoint; - @JsonProperty(required = false) - private String updateEndpoint; - @JsonProperty(required = false) - private String version; - - public String getVersion() { - return version; - } - - public String getVersion(String defaultValue) { - if(version!=null) - return version; - return defaultValue; - } - - public void setVersion(String version) { - this.version = version; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getUser() { - return user; - } - - public void setUser(String user) { - this.user = user; - } - - public String getPassword() { - return password; - } - - public void setPassword(String password) { - this.password = password; - } - - public String getEndpoint() { - return endpoint; - } - - public void setEndpoint(String endpoint) { - this.endpoint = endpoint; - } - - public String getUpdateEndpoint() { - return updateEndpoint; - } - - public void setUpdateEndpoint(String updateEndpoint) { - this.updateEndpoint = updateEndpoint; - } +public record ConnectionConfig( + @JsonProperty(required = true) + String name, + String version, + DatasetConfig dataset, + @JsonProperty(required = true) + @JsonDeserialize(using = URIDeserializer.class) + URI endpoint, + Authentication authentication, + @JsonDeserialize(using = URIDeserializer.class) + URI updateEndpoint, + Authentication updateAuthentication + +) { + public static class URIDeserializer extends JsonDeserializer { + + @Override + public URI deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { + return URI.create(p.getValueAsString()); // verifying uri doesn't work here + } + } + + public record Authentication(String user, String password) {} } diff --git a/src/main/java/org/aksw/iguana/cc/config/elements/DatasetConfig.java b/src/main/java/org/aksw/iguana/cc/config/elements/DatasetConfig.java index d067b7158..8986de3be 100644 --- a/src/main/java/org/aksw/iguana/cc/config/elements/DatasetConfig.java +++ b/src/main/java/org/aksw/iguana/cc/config/elements/DatasetConfig.java @@ -4,29 +4,10 @@ /** * The Dataset config class. - * + *

* Will set the name and if it was set in the config file the fileName */ -public class DatasetConfig { - @JsonProperty(required = true) - private String name; - - @JsonProperty - private String file; - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getFile() { - return file; - } - - public void setFile(String file) { - this.file = file; - } -} +public record DatasetConfig( + @JsonProperty(required = true) String name, + @JsonProperty String file +) {} diff --git a/src/main/java/org/aksw/iguana/cc/config/elements/MetricConfig.java b/src/main/java/org/aksw/iguana/cc/config/elements/MetricConfig.java deleted file mode 100644 index 7f5ba8521..000000000 --- a/src/main/java/org/aksw/iguana/cc/config/elements/MetricConfig.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.aksw.iguana.cc.config.elements; - -import com.fasterxml.jackson.annotation.JsonProperty; -import org.aksw.iguana.cc.tasks.stresstest.metrics.Metric; -import org.aksw.iguana.commons.factory.TypedFactory; - -import java.util.HashMap; -import java.util.Map; - -/** - * Metric Config class - */ -public class MetricConfig { - - @JsonProperty(required = true) - private String className; - - @JsonProperty() - private Map configuration = new HashMap<>(); - - - public String getClassName() { - return className; - } - - public void setClassName(String className) { - this.className = className; - } - - public Map getConfiguration() { - return configuration; - } - - public void setConfiguration(Map configuration) { - this.configuration = configuration; - } - - public Metric createMetric() { - TypedFactory factory = new TypedFactory<>(); - return factory.create(className, configuration); - } -} diff --git a/src/main/java/org/aksw/iguana/cc/config/elements/StorageConfig.java b/src/main/java/org/aksw/iguana/cc/config/elements/StorageConfig.java index 5fe6127e8..bd55cace2 100644 --- a/src/main/java/org/aksw/iguana/cc/config/elements/StorageConfig.java +++ b/src/main/java/org/aksw/iguana/cc/config/elements/StorageConfig.java @@ -1,42 +1,24 @@ package org.aksw.iguana.cc.config.elements; -import com.fasterxml.jackson.annotation.JsonProperty; -import org.aksw.iguana.commons.factory.TypedFactory; -import org.aksw.iguana.cc.tasks.stresstest.storage.Storage; - -import java.util.HashMap; -import java.util.Map; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.aksw.iguana.cc.storage.impl.CSVStorage; +import org.aksw.iguana.cc.storage.impl.RDFFileStorage; +import org.aksw.iguana.cc.storage.impl.TriplestoreStorage; /** * Storage Configuration class */ -public class StorageConfig { - - - @JsonProperty(required = true) - private String className; - - @JsonProperty - private Map configuration = new HashMap<>(); - - public String getClassName() { - return className; - } - public void setClassName(String className) { - this.className = className; - } +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type") +@JsonSubTypes({ + @JsonSubTypes.Type(value = TriplestoreStorage.Config.class, name = "triplestore"), + @JsonSubTypes.Type(value = RDFFileStorage.Config.class, name = "rdf file"), + @JsonSubTypes.Type(value = CSVStorage.Config.class, name = "csv file") +}) +public interface StorageConfig {} - public Map getConfiguration() { - return configuration; - } - public void setConfiguration(Map configuration) { - this.configuration = configuration; - } - public Storage createStorage() { - TypedFactory factory = new TypedFactory<>(); - return factory.create(className, configuration); - } -} diff --git a/src/main/java/org/aksw/iguana/cc/config/elements/TaskConfig.java b/src/main/java/org/aksw/iguana/cc/config/elements/TaskConfig.java deleted file mode 100644 index 5b09b3e45..000000000 --- a/src/main/java/org/aksw/iguana/cc/config/elements/TaskConfig.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.aksw.iguana.cc.config.elements; - -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; - -/** - * The task configuration class, sets the class name and it's configuration - */ -public class TaskConfig implements Serializable { - - @JsonProperty(required = true) - private Map configuration = new HashMap<>(); - - @JsonProperty(required = true) - private String className; - - @JsonProperty() - private String name=null; - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public Map getConfiguration() { - return configuration; - } - - public String getClassName() { - return className; - } - - public void setClassName(String className) { - this.className = className; - } -} diff --git a/src/main/java/org/aksw/iguana/cc/controller/MainController.java b/src/main/java/org/aksw/iguana/cc/controller/MainController.java index 9dd1b63ca..e9a03e70e 100644 --- a/src/main/java/org/aksw/iguana/cc/controller/MainController.java +++ b/src/main/java/org/aksw/iguana/cc/controller/MainController.java @@ -1,68 +1,73 @@ package org.aksw.iguana.cc.controller; -import org.aksw.iguana.cc.config.ConfigManager; +import com.beust.jcommander.*; +import org.aksw.iguana.cc.suite.IguanaSuiteParser; +import org.aksw.iguana.cc.suite.Suite; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.io.IOException; +import java.nio.file.Path; + /** - * The main controller for the core. - * Will execute the Config Manager and the consuming for configurations. - * - * @author f.conrads - * + * The MainController class is responsible for executing the IGUANA program. */ public class MainController { - - private static final Logger LOGGER = LoggerFactory - .getLogger(MainController.class); - - /** - * main method for standalone controlling. - * If the TaskController should run standalone instead of in the core itself - * - * @param argc - * @throws IOException - */ - public static void main(String[] argc) throws IOException{ - if(argc.length != 1 && argc.length !=2){ - System.out.println("java -jar iguana.jar [--ignore-schema] suite.yml \n\tsuite.yml - The suite containing the benchmark configuration\n\t--ignore-schema - Will not validate configuration using the internal json schema\n"); - return; - } - MainController controller = new MainController(); - String config =argc[0]; - Boolean validate = true; - if(argc.length==2){ - if(argc[0].equals("--ignore-schema")){ - validate=false; - } - config = argc[1]; - } - controller.start(config, validate); - LOGGER.info("Stopping Iguana"); - //System.exit(0); - } + public static class Args { + public class PathConverter implements IStringConverter { + @Override + public Path convert(String value) { + return Path.of(value); + } + } + + + @Parameter(names = {"--ignore-schema", "-is"}, description = "Do not check the schema before parsing the suite file.") + private boolean ignoreShema = false; + + @Parameter(names = "--help", help = true) + private boolean help; + + @Parameter(description = "suite file {yml,yaml,json}", arity = 1, required = true, converter = PathConverter.class) + private Path suitePath; + } + + + private static final Logger LOGGER = LoggerFactory.getLogger(MainController.class); - /** - * Starts a configuration using the config file an states if Iguana should validate it using a json-schema - * - * @param configFile the Iguana config file - * @param validate should the config file be validated using a json-schema - * @throws IOException - */ - public void start(String configFile, Boolean validate) throws IOException{ - ConfigManager cmanager = new ConfigManager(); - File f = new File(configFile); - if (f.length()!=0) { - cmanager.receiveData(f, validate); - } else { - LOGGER.error("Empty configuration."); + /** + * The main method for executing IGUANA + * + * @param argc The command line arguments that are passed to the program. + */ + public static void main(String[] argc) { + var args = new Args(); + JCommander jc = JCommander.newBuilder() + .addObject(args) + .build(); + try { + jc.parse(argc); + } catch (ParameterException e) { + System.err.println(e.getLocalizedMessage()); + jc.usage(); + System.exit(0); + } + if (args.help) { + jc.usage(); + System.exit(1); + } - } + try { + Suite parse = IguanaSuiteParser.parse(args.suitePath, !args.ignoreShema); + parse.run(); + } catch (IOException e) { + LOGGER.error("Error while reading the configuration file.", e); + System.exit(0); + } + System.exit(0); + } - } } diff --git a/src/main/java/org/aksw/iguana/cc/controller/TaskController.java b/src/main/java/org/aksw/iguana/cc/controller/TaskController.java deleted file mode 100644 index 26f9652b7..000000000 --- a/src/main/java/org/aksw/iguana/cc/controller/TaskController.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.aksw.iguana.cc.controller; - -import org.aksw.iguana.cc.config.elements.ConnectionConfig; -import org.aksw.iguana.cc.config.elements.TaskConfig; -import org.aksw.iguana.cc.tasks.TaskFactory; -import org.aksw.iguana.cc.tasks.TaskManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.concurrent.TimeoutException; - - -/** - * Task Controlling, will start the actual benchmark tasks and its {@link org.aksw.iguana.cc.tasks.TaskManager} - * - * @author f.conrads - */ -public class TaskController { - - private static final Logger LOGGER = LoggerFactory.getLogger(TaskController.class); - - public void startTask(String[] ids, String dataset, ConnectionConfig con, TaskConfig task) { - TaskManager tmanager = new TaskManager(); - String className = task.getClassName(); - TaskFactory factory = new TaskFactory(); - tmanager.setTask(factory.create(className, task.getConfiguration())); - try { - tmanager.startTask(ids, dataset, con, task.getName()); - } catch (IOException | TimeoutException e) { - LOGGER.error("Could not start Task " + className, e); - } - } -} diff --git a/src/main/java/org/aksw/iguana/cc/lang/AbstractLanguageProcessor.java b/src/main/java/org/aksw/iguana/cc/lang/AbstractLanguageProcessor.java deleted file mode 100644 index 10cce5b06..000000000 --- a/src/main/java/org/aksw/iguana/cc/lang/AbstractLanguageProcessor.java +++ /dev/null @@ -1,65 +0,0 @@ -package org.aksw.iguana.cc.lang; - -import org.aksw.iguana.commons.constants.COMMON; -import org.aksw.iguana.commons.rdf.IONT; -import org.aksw.iguana.commons.rdf.IPROP; -import org.aksw.iguana.commons.streams.Streams; -import org.apache.http.Header; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.jena.rdf.model.Model; -import org.apache.jena.rdf.model.ModelFactory; -import org.apache.jena.rdf.model.Resource; -import org.apache.jena.rdf.model.ResourceFactory; -import org.apache.jena.vocabulary.RDF; -import org.apache.jena.vocabulary.RDFS; -import org.json.simple.parser.ParseException; -import org.xml.sax.SAXException; - -import javax.xml.parsers.ParserConfigurationException; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.time.Instant; -import java.util.List; -import java.util.concurrent.TimeoutException; - -public abstract class AbstractLanguageProcessor implements LanguageProcessor { - - @Override - public String getQueryPrefix() { - return "query"; - } - - @Override - public Model generateTripleStats(List queries, String resourcePrefix, String taskID) { - Model model = ModelFactory.createDefaultModel(); - for(QueryWrapper wrappedQuery : queries) { - Resource subject = ResourceFactory.createResource(COMMON.RES_BASE_URI + resourcePrefix + "/" + wrappedQuery.getId()); - model.add(subject, RDF.type, IONT.query); - // TODO: fix this - model.add(subject, IPROP.queryID, ResourceFactory.createTypedLiteral(wrappedQuery.getId())); - model.add(subject, RDFS.label, wrappedQuery.getQuery().toString()); - } - return model; - } - - @Override - public Long getResultSize(CloseableHttpResponse response) throws ParserConfigurationException, SAXException, ParseException, IOException { - return response.getEntity().getContentLength(); - } - - @Override - public Long getResultSize(Header contentTypeHeader, ByteArrayOutputStream content, long contentLength) throws ParserConfigurationException, SAXException, ParseException, IOException { - return Long.valueOf(content.size()); - } - - @Override - public long readResponse(InputStream inputStream, ByteArrayOutputStream responseBody) throws IOException, TimeoutException { - return Streams.inputStream2ByteArrayOutputStream(inputStream, responseBody); - } - - //@Override - public long readResponse(InputStream inputStream, Instant startTime, Double timeOut, ByteArrayOutputStream responseBody) throws IOException, TimeoutException { - return Streams.inputStream2ByteArrayOutputStream(inputStream, startTime, timeOut, responseBody); - } -} diff --git a/src/main/java/org/aksw/iguana/cc/lang/LanguageProcessor.java b/src/main/java/org/aksw/iguana/cc/lang/LanguageProcessor.java index 211d50aa8..bd902dd82 100644 --- a/src/main/java/org/aksw/iguana/cc/lang/LanguageProcessor.java +++ b/src/main/java/org/aksw/iguana/cc/lang/LanguageProcessor.java @@ -1,55 +1,69 @@ package org.aksw.iguana.cc.lang; -import org.apache.http.Header; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.jena.rdf.model.Model; -import org.json.simple.parser.ParseException; -import org.xml.sax.SAXException; - -import javax.xml.parsers.ParserConfigurationException; -import java.io.ByteArrayOutputStream; -import java.io.IOException; +import org.aksw.iguana.cc.storage.Storable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.util.AnnotatedTypeScanner; + import java.io.InputStream; -import java.time.Instant; -import java.util.List; -import java.util.concurrent.TimeoutException; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.lang.reflect.InvocationTargetException; +import java.util.HashMap; +import java.util.Map; + /** - * Language Processor tells how to handle Http responses as well as how to analyze queries and generate stats. + * Interface for abstract language processors that work on InputStreams. */ -public interface LanguageProcessor { +public abstract class LanguageProcessor { /** - * Returns the prefix used for the queries (e.g. sparql, query or document) - * @return + * Provides the content type that a LanguageProcessor consumes. */ - String getQueryPrefix(); + @Retention(RetentionPolicy.RUNTIME) + @Target(ElementType.TYPE) + public @interface ContentType { + String value(); + } - /** - * Method to generate Triple Statistics for provided queries - * - * - * @param taskID - * @return Model with the triples to add to the results - */ - Model generateTripleStats(List queries, String resourcePrefix, String taskID); + public interface LanguageProcessingData extends Storable { + long hash(); + Class processor(); + } + public abstract LanguageProcessingData process(InputStream inputStream, long hash); - /** - * Gets the result size of a given HTTP response - * - * @param response - * @return - * @throws ParserConfigurationException - * @throws SAXException - * @throws ParseException - * @throws IOException - */ - Long getResultSize(CloseableHttpResponse response) throws ParserConfigurationException, SAXException, ParseException, IOException; + final private static Map> processors = new HashMap<>(); - Long getResultSize(Header contentTypeHeader, ByteArrayOutputStream content, long contentLength) throws ParserConfigurationException, SAXException, ParseException, IOException; + final private static Logger LOGGER = LoggerFactory.getLogger(LanguageProcessor.class); + static { + final var scanner = new AnnotatedTypeScanner(false, ContentType.class); + final var langProcessors = scanner.findTypes("org.aksw.iguana.cc.lang"); + for (Class langProcessor : langProcessors) { + String contentType = langProcessor.getAnnotation(ContentType.class).value(); + if (LanguageProcessor.class.isAssignableFrom(langProcessor)) { + processors.put(contentType, (Class) langProcessor); + } else { + LOGGER.error("Found a class with the ContentType annotation, that doesn't inherit from the class LanguageProcessor: {}", langProcessor.getName()); + } + } + } - long readResponse(InputStream inputStream, ByteArrayOutputStream responseBody) throws IOException, TimeoutException; + public static LanguageProcessor getInstance(String contentType) { + Class processorClass = processors.get(contentType); + if (processorClass != null) { + try { + return processorClass.getDeclaredConstructor().newInstance(); + } catch (InstantiationException | IllegalAccessException | InvocationTargetException | + NoSuchMethodException e) { + throw new RuntimeException(e); + } + } + throw new IllegalArgumentException("No LanguageProcessor for ContentType " + contentType); + } } diff --git a/src/main/java/org/aksw/iguana/cc/lang/QueryWrapper.java b/src/main/java/org/aksw/iguana/cc/lang/QueryWrapper.java deleted file mode 100644 index 07ca5fb73..000000000 --- a/src/main/java/org/aksw/iguana/cc/lang/QueryWrapper.java +++ /dev/null @@ -1,41 +0,0 @@ -package org.aksw.iguana.cc.lang; - -import java.math.BigInteger; - -/** - * Util class to wrap a Query of what ever class it may be and it's id - */ -public class QueryWrapper { - private final Object query; - private final int id; - private final String fullId; - - public QueryWrapper(Object query, String fullId) { - this.query = query; - int i = fullId.length(); - while (i > 0 && Character.isDigit(fullId.charAt(i - 1))) { - i--; - } - - this.id = Integer.parseInt(fullId.substring(i)); - this.fullId = fullId; - } - - public QueryWrapper(Object query, String prefix, int id) { - this.query = query; - this.id = id; - this.fullId = prefix + id; - } - - public Object getQuery() { - return query; - } - - public BigInteger getId() { - return BigInteger.valueOf(id); - } - - public String getFullId() { - return fullId; - } -} diff --git a/src/main/java/org/aksw/iguana/cc/lang/impl/RDFLanguageProcessor.java b/src/main/java/org/aksw/iguana/cc/lang/impl/RDFLanguageProcessor.java deleted file mode 100644 index 80e68e1dc..000000000 --- a/src/main/java/org/aksw/iguana/cc/lang/impl/RDFLanguageProcessor.java +++ /dev/null @@ -1,112 +0,0 @@ -package org.aksw.iguana.cc.lang.impl; - -import org.aksw.iguana.cc.lang.AbstractLanguageProcessor; -import org.aksw.iguana.cc.lang.LanguageProcessor; -import org.aksw.iguana.cc.lang.QueryWrapper; -import org.aksw.iguana.commons.annotation.Shorthand; -import org.aksw.iguana.commons.constants.COMMON; -import org.aksw.iguana.commons.rdf.IONT; -import org.aksw.iguana.commons.rdf.IPROP; -import org.apache.http.Header; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.jena.rdf.model.Model; -import org.apache.jena.rdf.model.ModelFactory; -import org.apache.jena.rdf.model.Resource; -import org.apache.jena.rdf.model.ResourceFactory; -import org.apache.jena.riot.Lang; -import org.apache.jena.vocabulary.RDF; -import org.apache.jena.vocabulary.RDFS; -import org.json.simple.parser.ParseException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.xml.sax.SAXException; - -import javax.xml.parsers.ParserConfigurationException; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.lang.reflect.Field; -import java.util.List; - -/** - * Language for everything which returns RDF in any rdf format. - * - * Counts triples returned as ResultSize - */ -@Shorthand("lang.RDF") -public class RDFLanguageProcessor extends AbstractLanguageProcessor implements LanguageProcessor { - - private static Logger LOGGER = LoggerFactory.getLogger(RDFLanguageProcessor.class); - protected String queryPrefix="document"; - - @Override - public String getQueryPrefix() { - return this.queryPrefix; - } - - @Override - public Model generateTripleStats(List queries, String resourcePrefix, String taskID) { - Model model = ModelFactory.createDefaultModel(); - for(QueryWrapper wrappedQuery : queries) { - Resource subject = ResourceFactory.createResource(COMMON.RES_BASE_URI + resourcePrefix + "/" + wrappedQuery.getId()); - model.add(subject, RDF.type, IONT.query); - // TODO: fix this - model.add(subject, IPROP.queryID, ResourceFactory.createTypedLiteral(wrappedQuery.getId())); - model.add(subject, RDFS.label, wrappedQuery.getQuery().toString()); - } - return model; - } - - @Override - public Long getResultSize(CloseableHttpResponse response) throws ParserConfigurationException, SAXException, ParseException, IOException { - Model m; - try { - Header contentTypeHeader = response.getEntity().getContentType(); - InputStream inputStream = response.getEntity().getContent(); - m = getModel(contentTypeHeader, inputStream); - } catch (IllegalAccessException e) { - LOGGER.error("Could not read response as model", e); - return -1L; - } - return countSize(m); - } - - @Override - public Long getResultSize(Header contentTypeHeader, ByteArrayOutputStream content, long contentLength) throws IOException { - Model m; - try { - //TODO BBAIS - InputStream inputStream = new ByteArrayInputStream(content.toByteArray()); - m = getModel(contentTypeHeader, inputStream); - } catch (IllegalAccessException e) { - LOGGER.error("Could not read response as model", e); - return -1L; - } - return countSize(m); - } - - protected Long countSize(Model m) { - return m.size(); - } - - private Model getModel(Header contentTypeHeader, InputStream contentInputStream) throws IOException, IllegalAccessException { - Model m = ModelFactory.createDefaultModel(); - Lang lang = null; - // get actual content type - String contentType = contentTypeHeader.getValue(); - // use reflect to iterate over all static Lang fields of the Lang.class - for (Field langField : Lang.class.getFields()) { - //create the Language of the field - Lang susLang = (Lang) langField.get(Lang.class); - //if they are the same we have our language - if (contentType.equals(susLang.getContentType().getContentTypeStr())) { - lang = susLang; - break; - } - } - if (lang != null) - m.read(contentInputStream, null, lang.getName()); - return m; - } -} diff --git a/src/main/java/org/aksw/iguana/cc/lang/impl/SPARQLLanguageProcessor.java b/src/main/java/org/aksw/iguana/cc/lang/impl/SPARQLLanguageProcessor.java deleted file mode 100644 index 5711f87d2..000000000 --- a/src/main/java/org/aksw/iguana/cc/lang/impl/SPARQLLanguageProcessor.java +++ /dev/null @@ -1,190 +0,0 @@ -package org.aksw.iguana.cc.lang.impl; - -import org.aksw.iguana.cc.lang.AbstractLanguageProcessor; -import org.aksw.iguana.cc.lang.LanguageProcessor; -import org.aksw.iguana.cc.lang.QueryWrapper; -import org.aksw.iguana.cc.utils.SPARQLQueryStatistics; -import org.aksw.iguana.commons.annotation.Shorthand; -import org.aksw.iguana.commons.constants.COMMON; -import org.aksw.iguana.commons.rdf.IONT; -import org.aksw.iguana.commons.rdf.IPROP; -import org.apache.http.Header; -import org.apache.http.HeaderElement; -import org.apache.http.HttpEntity; -import org.apache.http.NameValuePair; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.jena.ext.com.google.common.hash.HashCode; -import org.apache.jena.ext.com.google.common.hash.Hashing; -import org.apache.jena.ext.com.google.common.io.BaseEncoding; -import org.apache.jena.query.Query; -import org.apache.jena.query.QueryFactory; -import org.apache.jena.rdf.model.Model; -import org.apache.jena.rdf.model.ModelFactory; -import org.apache.jena.rdf.model.Resource; -import org.apache.jena.rdf.model.ResourceFactory; -import org.apache.jena.vocabulary.OWL; -import org.apache.jena.vocabulary.RDF; -import org.apache.jena.vocabulary.RDFS; -import org.json.simple.parser.JSONParser; -import org.json.simple.parser.ParseException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.w3c.dom.Document; -import org.w3c.dom.NodeList; -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import java.io.*; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.util.List; - -import static org.aksw.iguana.commons.streams.Streams.inputStream2String; - -/** - * SPARQL Language Processor. - * Tries to analyze Queries as SPARQL queries and checks http response for either application/sparql-results+json - * or application/sparql-results+xml to count the result size correctly. Otherwise assumes it record per line and counts the returning lines. - */ -@Shorthand("lang.SPARQL") -public class SPARQLLanguageProcessor extends AbstractLanguageProcessor implements LanguageProcessor { - - private static Logger LOGGER = LoggerFactory.getLogger(SPARQLLanguageProcessor.class); - - public static final String XML_RESULT_ELEMENT_NAME = "result"; - public static final String XML_RESULT_ROOT_ELEMENT_NAME = "results"; - public static final String QUERY_RESULT_TYPE_JSON = "application/sparql-results+json"; - public static final String QUERY_RESULT_TYPE_XML = "application/sparql-results+xml"; - private static final String LSQ_RES = "http://lsq.aksw.org/res/q-"; - - @Override - public String getQueryPrefix() { - return "sparql"; - } - - @Override - public Model generateTripleStats(List queries, String resourcePrefix, String taskID) { - Model model = ModelFactory.createDefaultModel(); - for(QueryWrapper wrappedQuery : queries) { - Resource subject = ResourceFactory.createResource(COMMON.RES_BASE_URI + resourcePrefix + "/" + wrappedQuery.getId()); - model.add(subject, RDF.type, IONT.query); - // TODO: queryID is already used in a different context - model.add(subject, IPROP.queryID, ResourceFactory.createTypedLiteral(wrappedQuery.getId())); - model.add(subject, RDFS.label, wrappedQuery.getQuery().toString()); - try { - Query q = QueryFactory.create(wrappedQuery.getQuery().toString()); - SPARQLQueryStatistics qs2 = new SPARQLQueryStatistics(); - qs2.getStatistics(q); - - model.add(subject, IPROP.aggregations, model.createTypedLiteral(qs2.aggr==1)); - model.add(subject, IPROP.filter, model.createTypedLiteral(qs2.filter==1)); - model.add(subject, IPROP.groupBy, model.createTypedLiteral(qs2.groupBy==1)); - model.add(subject, IPROP.having, model.createTypedLiteral(qs2.having==1)); - model.add(subject, IPROP.triples, model.createTypedLiteral(qs2.triples)); - model.add(subject, IPROP.offset, model.createTypedLiteral(qs2.offset==1)); - model.add(subject, IPROP.optional, model.createTypedLiteral(qs2.optional==1)); - model.add(subject, IPROP.orderBy, model.createTypedLiteral(qs2.orderBy==1)); - model.add(subject, IPROP.union, model.createTypedLiteral(qs2.union==1)); - model.add(subject, OWL.sameAs, getLSQHash(q)); - }catch(Exception e){ - LOGGER.warn("Query statistics could not be created. Not using SPARQL?"); - } - } - return model; - } - - private Resource getLSQHash(Query query){ - HashCode hashCode = Hashing.sha256().hashString(query.toString(), StandardCharsets.UTF_8); - String result = BaseEncoding.base64Url().omitPadding().encode(hashCode.asBytes()); - return ResourceFactory.createResource(LSQ_RES+result); - } - - - public static String getContentTypeVal(Header header) { - for (HeaderElement el : header.getElements()) { - NameValuePair cTypePair = el.getParameterByName("Content-Type"); - - if (cTypePair != null && !cTypePair.getValue().isEmpty()) { - return cTypePair.getValue(); - } - } - int index = header.toString().indexOf("Content-Type"); - if (index >= 0) { - String ret = header.toString().substring(index + "Content-Type".length() + 1); - if (ret.contains(";")) { - return ret.substring(0, ret.indexOf(";")).trim(); - } - return ret.trim(); - } - return "application/sparql-results+json"; - } - - public static long getJsonResultSize(ByteArrayOutputStream res) throws ParseException, UnsupportedEncodingException { - JSONParser parser = new JSONParser(); - SaxSparqlJsonResultCountingParser handler = new SaxSparqlJsonResultCountingParser(); - parser.parse(res.toString(StandardCharsets.UTF_8), handler, true); - return handler.getNoBindings(); - } - - public static long getXmlResultSize(ByteArrayOutputStream res) throws ParserConfigurationException, IOException, SAXException { - DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); - - ByteArrayInputStream bbais = new ByteArrayInputStream(res.toByteArray()); - Document doc = dBuilder.parse(bbais); - NodeList childNodes = doc.getDocumentElement().getElementsByTagName(XML_RESULT_ROOT_ELEMENT_NAME).item(0).getChildNodes(); - - long size = 0; - for (int i = 0; i < childNodes.getLength(); i++) { - if (XML_RESULT_ELEMENT_NAME.equalsIgnoreCase(childNodes.item(i).getNodeName())) { - size++; - } - } - return size; - - } - - @Override - public Long getResultSize(CloseableHttpResponse response) throws ParserConfigurationException, SAXException, ParseException, IOException { - HttpEntity httpResponse = response.getEntity(); - Header contentTypeHeader = response.getEntity().getContentType(); - - ByteArrayOutputStream entity; - try (InputStream inputStream = httpResponse.getContent()) { - - entity = inputStream2String(inputStream); - } catch (IOException e) { - LOGGER.error("Query result could not be read.", e); - throw e; - } - return getResultSize(contentTypeHeader, entity, entity.size()); - } - - @Override - public Long getResultSize(Header contentTypeHeader, ByteArrayOutputStream content, long contentLength) throws ParserConfigurationException, SAXException, ParseException, IOException { - try { - switch (getContentTypeVal(contentTypeHeader)) { - case QUERY_RESULT_TYPE_JSON: - return getJsonResultSize(content); - - case QUERY_RESULT_TYPE_XML: - return getXmlResultSize(content); - default: - //return content.countMatches('\n')+1; - long matches=0; - for(byte b: content.toByteArray()){ - if(b=='\n'){ - matches++; - } - } - return matches+1; - } - } catch (ParseException | ParserConfigurationException | IOException | SAXException e) { - LOGGER.error("Query results could not be parsed: ", e); - throw e; - } - } -} diff --git a/src/main/java/org/aksw/iguana/cc/lang/impl/SaxSparqlJsonResultCountingParser.java b/src/main/java/org/aksw/iguana/cc/lang/impl/SaxSparqlJsonResultCountingParser.java index d4c1f3e29..42a8a4eaf 100644 --- a/src/main/java/org/aksw/iguana/cc/lang/impl/SaxSparqlJsonResultCountingParser.java +++ b/src/main/java/org/aksw/iguana/cc/lang/impl/SaxSparqlJsonResultCountingParser.java @@ -1,114 +1,223 @@ package org.aksw.iguana.cc.lang.impl; +import org.aksw.iguana.cc.lang.LanguageProcessor; +import org.aksw.iguana.cc.storage.Storable; +import org.aksw.iguana.commons.rdf.IPROP; +import org.aksw.iguana.commons.rdf.IRES; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; +import org.apache.jena.rdf.model.Resource; +import org.apache.jena.rdf.model.ResourceFactory; import org.json.simple.parser.ContentHandler; +import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; import static org.json.simple.parser.ParseException.ERROR_UNEXPECTED_EXCEPTION; /** * SAX Parser for SPARQL JSON Results. - * For correct SPARQL JSON Results it returns the correct size. + * For correct SPARQL JSON Results it returns the number of solutions, bound values and the names of the variables. * For malformed results it may or may not fail. For malformed JSON it fails if the underlying json.simple.parser fails. */ -class SaxSparqlJsonResultCountingParser implements ContentHandler { +@LanguageProcessor.ContentType("application/sparql-results+json") +public class SaxSparqlJsonResultCountingParser extends LanguageProcessor { - private boolean headFound = false; + @Override + public LanguageProcessingData process(InputStream inputStream, long hash) { + var parser = new JSONParser(); + var handler = new SaxSparqlJsonResultContentHandler(); + try { + parser.parse(new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)), handler); + return new SaxSparqlJsonResultData(hash, handler.solutions(), handler.boundValues(), handler.variables(), null); + } catch (IOException e) { + throw new RuntimeException(e); + } catch (ParseException e) { + return new SaxSparqlJsonResultData(hash, -1, -1, null, e); + } + } - private int objectDepth = 0; - private boolean inResults = false; - private boolean inBindings = false; - private boolean inBindingsArray = false; + record SaxSparqlJsonResultData( + long hash, + long results, + long bindings, + List variables, + Exception exception + ) implements LanguageProcessingData, Storable.AsCSV, Storable.AsRDF { + final static String[] header = new String[]{ "responseBodyHash", "results", "bindings", "variables", "exception" }; + + @Override + public Class processor() { + return SaxSparqlJsonResultCountingParser.class; + } - private long noBindings = 0; + @Override + public CSVData toCSV() { + String variablesString = ""; + String exceptionString = ""; + if (variables != null) + variablesString = String.join("; ", variables); + if (exception != null) + exceptionString = exception().toString(); + + String[] content = new String[]{ String.valueOf(hash), String.valueOf(results), String.valueOf(bindings), variablesString, exceptionString}; + String[][] data = new String[][]{ header, content }; + + String folderName = "application-sparql+json"; + List files = List.of(new CSVData.CSVFileData("sax-sparql-result-data.csv", data)); + return new Storable.CSVData(folderName, files); + } - public long getNoBindings() { - return noBindings; - } + @Override + public Model toRDF() { + Model m = ModelFactory.createDefaultModel(); + Resource responseBodyRes = IRES.getResponsebodyResource(this.hash); + m.add(responseBodyRes, IPROP.results, ResourceFactory.createTypedLiteral(this.results)) + .add(responseBodyRes, IPROP.bindings, ResourceFactory.createTypedLiteral(this.bindings)); - @Override - public void startJSON() { - } + if (this.variables != null) { + for (String variable : this.variables) { + m.add(responseBodyRes, IPROP.variable, ResourceFactory.createTypedLiteral(variable)); + } + } + if (this.exception != null) { + m.add(responseBodyRes, IPROP.exception, ResourceFactory.createTypedLiteral(this.exception.toString())); + } - @Override - public void endJSON() throws ParseException { - if (inResults || inBindings || inBindingsArray || !headFound || objectDepth != 0) - throw new ParseException(ERROR_UNEXPECTED_EXCEPTION, "SPARQL Json Response was malformed."); + return m; + } } - @Override - public boolean startObject() { - objectDepth += 1; - if (objectDepth == 3 && inBindingsArray) { - noBindings += 1; + private static class SaxSparqlJsonResultContentHandler implements ContentHandler { + // TODO: add support for ask queries and link + // TODO: code is unnecessary complicated + + private boolean headFound = false; + + private int objectDepth = 0; + private boolean inResults = false; + private boolean inBindings = false; + private boolean inBindingsArray = false; + private boolean inVars = false; + + private long boundValues = 0; + + private long solutions = 0; + + private final List variables = new ArrayList<>(); + + + @Override + public void startJSON() { } - return true; - } - @Override - public boolean endObject() { - switch (objectDepth) { - case 1: - if (inResults) - inResults = false; - break; - case 2: - if (inBindings) { - inBindings = false; + @Override + public void endJSON() throws ParseException { + if (inResults || inBindings || inBindingsArray || !headFound || objectDepth != 0) + throw new ParseException(ERROR_UNEXPECTED_EXCEPTION, "SPARQL Json Response was malformed."); + } + + @Override + public boolean startObject() { + objectDepth += 1; + if (inBindingsArray) { + switch (objectDepth) { + case 3 -> solutions += 1; + case 4 -> boundValues += 1; } - break; + } + return true; } - objectDepth -= 1; - return true; - } - @Override - public boolean startArray() { - if (objectDepth == 2 && inResults && inBindings && !inBindingsArray) { - inBindingsArray = true; + @Override + public boolean endObject() { + switch (objectDepth) { + case 1: + if (inResults) + inResults = false; + break; + case 2: + if (inBindings) { + inBindings = false; + } + break; + } + objectDepth -= 1; + return true; } - return true; - } - @Override - public boolean endArray() { - if (objectDepth == 2 && inResults && inBindings && inBindingsArray) { - inBindingsArray = false; + @Override + public boolean startArray() { + if (objectDepth == 2 && inResults && inBindings && !inBindingsArray) { + inBindingsArray = true; + } + return true; } - return true; - } - @Override - public boolean startObjectEntry(String key) { - switch (objectDepth) { - case 1: - switch (key) { - case "head": - headFound = true; - break; - case "results": - if (headFound) - inResults = true; - break; + @Override + public boolean endArray() { + if (inVars) + inVars = false; + if (objectDepth == 2 && inResults && inBindings && inBindingsArray) { + inBindingsArray = false; + } + return true; + } + + + @Override + public boolean startObjectEntry(String key) { + switch (objectDepth) { + case 1 -> { + switch (key) { + case "head" -> headFound = true; + case "results" -> { + if (headFound) + inResults = true; + } + } } - break; - case 2: - if ("bindings".compareTo(key) == 0) { - inBindings = true; + case 2 -> { + if ("bindings".compareTo(key) == 0) { + inBindings = true; + } + if ("vars".compareTo(key) == 0) { + inVars = true; + } } - break; + } + return true; } - return true; - } - @Override - public boolean endObjectEntry() { - return true; - } + @Override + public boolean endObjectEntry() { + return true; + } - public boolean primitive(Object value) { - return true; - } + public boolean primitive(Object value) { + if (inVars) + variables.add(value.toString()); + return true; + } + public long boundValues() { + return boundValues; + } + + public long solutions() { + return solutions; + } + + public List variables() { + return variables; + } + } } \ No newline at end of file diff --git a/src/main/java/org/aksw/iguana/cc/lang/impl/ThrowawayLanguageProcessor.java b/src/main/java/org/aksw/iguana/cc/lang/impl/ThrowawayLanguageProcessor.java deleted file mode 100644 index 5f2267936..000000000 --- a/src/main/java/org/aksw/iguana/cc/lang/impl/ThrowawayLanguageProcessor.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.aksw.iguana.cc.lang.impl; - -import org.aksw.iguana.cc.lang.AbstractLanguageProcessor; -import org.aksw.iguana.commons.annotation.Shorthand; -import org.aksw.iguana.commons.streams.Streams; -import org.apache.http.Header; -import org.json.simple.parser.ParseException; -import org.xml.sax.SAXException; - -import javax.xml.parsers.ParserConfigurationException; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.time.Instant; -import java.util.concurrent.TimeoutException; - -@Shorthand("lang.SIMPLE") -public class ThrowawayLanguageProcessor extends AbstractLanguageProcessor { - - @Override - public long readResponse(InputStream inputStream, ByteArrayOutputStream responseBody) throws IOException, TimeoutException { - return Streams.inputStream2Length(inputStream, Instant.now(), 0); - } - - @Override - public long readResponse(InputStream inputStream, Instant startTime, Double timeOut, ByteArrayOutputStream responseBody) throws IOException, TimeoutException { - return Streams.inputStream2Length(inputStream, startTime, timeOut); - } - - @Override - public Long getResultSize(Header contentTypeHeader, ByteArrayOutputStream content, long contentLength) throws ParserConfigurationException, SAXException, ParseException, IOException { - return contentLength; - } - -} diff --git a/src/main/java/org/aksw/iguana/cc/metrics/Metric.java b/src/main/java/org/aksw/iguana/cc/metrics/Metric.java new file mode 100644 index 000000000..0f4bc15fa --- /dev/null +++ b/src/main/java/org/aksw/iguana/cc/metrics/Metric.java @@ -0,0 +1,41 @@ +package org.aksw.iguana.cc.metrics; + +import com.fasterxml.jackson.annotation.*; +import org.aksw.iguana.cc.metrics.impl.*; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") +@JsonSubTypes({ + @JsonSubTypes.Type(value = AggregatedExecutionStatistics.class, name = "AES"), + @JsonSubTypes.Type(value = AvgQPS.class, name = "AvgQPS"), + @JsonSubTypes.Type(value = EachExecutionStatistic.class, name = "EachQuery"), + @JsonSubTypes.Type(value = NoQ.class, name = "NoQ"), + @JsonSubTypes.Type(value = NoQPH.class, name = "NoQPH"), + @JsonSubTypes.Type(value = PAvgQPS.class, name = "PAvgQPS"), + @JsonSubTypes.Type(value = PQPS.class, name = "PQPS"), + @JsonSubTypes.Type(value = QMPH.class, name = "QMPH"), + @JsonSubTypes.Type(value = QPS.class, name = "QPS") +}) +public abstract class Metric { + private final String name; + private final String abbreviation; + private final String description; + + public Metric(String name, String abbreviation, String description) { + this.name = name; + this.abbreviation = abbreviation; + this.description = description; + } + + + public String getDescription(){ + return this.description; + } + + public String getName(){ + return this.name; + } + + public String getAbbreviation(){ + return this.abbreviation; + } +} diff --git a/src/main/java/org/aksw/iguana/cc/metrics/ModelWritingMetric.java b/src/main/java/org/aksw/iguana/cc/metrics/ModelWritingMetric.java new file mode 100644 index 000000000..9debe1481 --- /dev/null +++ b/src/main/java/org/aksw/iguana/cc/metrics/ModelWritingMetric.java @@ -0,0 +1,19 @@ +package org.aksw.iguana.cc.metrics; + +import org.aksw.iguana.cc.worker.HttpWorker; +import org.aksw.iguana.commons.rdf.IRES; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; + +import java.util.List; +import java.util.Map; + +public interface ModelWritingMetric { + default Model createMetricModel(List workers, List[][] data, IRES.Factory iresFactory) { + return ModelFactory.createDefaultModel(); + } + + default Model createMetricModel(List workers, Map> data, IRES.Factory iresFactory) { + return ModelFactory.createDefaultModel(); + } +} diff --git a/src/main/java/org/aksw/iguana/cc/metrics/QueryMetric.java b/src/main/java/org/aksw/iguana/cc/metrics/QueryMetric.java new file mode 100644 index 000000000..9b771a570 --- /dev/null +++ b/src/main/java/org/aksw/iguana/cc/metrics/QueryMetric.java @@ -0,0 +1,9 @@ +package org.aksw.iguana.cc.metrics; + +import org.aksw.iguana.cc.worker.HttpWorker; + +import java.util.List; + +public interface QueryMetric { + Number calculateQueryMetric(List data); +} diff --git a/src/main/java/org/aksw/iguana/cc/metrics/TaskMetric.java b/src/main/java/org/aksw/iguana/cc/metrics/TaskMetric.java new file mode 100644 index 000000000..8b4360306 --- /dev/null +++ b/src/main/java/org/aksw/iguana/cc/metrics/TaskMetric.java @@ -0,0 +1,9 @@ +package org.aksw.iguana.cc.metrics; + +import org.aksw.iguana.cc.worker.HttpWorker; + +import java.util.List; + +public interface TaskMetric { + Number calculateTaskMetric(List workers, List[][] data); +} diff --git a/src/main/java/org/aksw/iguana/cc/metrics/WorkerMetric.java b/src/main/java/org/aksw/iguana/cc/metrics/WorkerMetric.java new file mode 100644 index 000000000..1fe5b763f --- /dev/null +++ b/src/main/java/org/aksw/iguana/cc/metrics/WorkerMetric.java @@ -0,0 +1,9 @@ +package org.aksw.iguana.cc.metrics; + +import org.aksw.iguana.cc.worker.HttpWorker; + +import java.util.List; + +public interface WorkerMetric { + Number calculateWorkerMetric(HttpWorker.Config worker, List[] data); +} diff --git a/src/main/java/org/aksw/iguana/cc/metrics/impl/AggregatedExecutionStatistics.java b/src/main/java/org/aksw/iguana/cc/metrics/impl/AggregatedExecutionStatistics.java new file mode 100644 index 000000000..e0942dba9 --- /dev/null +++ b/src/main/java/org/aksw/iguana/cc/metrics/impl/AggregatedExecutionStatistics.java @@ -0,0 +1,88 @@ +package org.aksw.iguana.cc.metrics.impl; + +import org.aksw.iguana.cc.metrics.Metric; +import org.aksw.iguana.cc.metrics.ModelWritingMetric; +import org.aksw.iguana.cc.worker.HttpWorker; +import org.aksw.iguana.commons.rdf.IONT; +import org.aksw.iguana.commons.rdf.IPROP; +import org.aksw.iguana.commons.rdf.IRES; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; +import org.apache.jena.rdf.model.Resource; +import org.apache.jena.rdf.model.ResourceFactory; +import org.apache.jena.vocabulary.RDF; + +import java.math.BigInteger; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.aksw.iguana.commons.time.TimeUtils.toXSDDurationInSeconds; + +public class AggregatedExecutionStatistics extends Metric implements ModelWritingMetric { + + public AggregatedExecutionStatistics() { + super("Aggregated Execution Statistics", "AES", "Sums up the statistics of each query execution for each query a worker and task has. The result size only contains the value of the last execution."); + } + + @Override + public Model createMetricModel(List workers, List[][] data, IRES.Factory iresFactory) { + Model m = ModelFactory.createDefaultModel(); + for (var worker : workers) { + for (int i = 0; i < worker.config().queries().getQueryCount(); i++) { + Resource queryRes = iresFactory.getWorkerQueryResource(worker, i); + m.add(createAggregatedModel(data[(int) worker.getWorkerID()][i], queryRes)); + } + } + return m; + } + + @Override + public Model createMetricModel(List workers, Map> data, IRES.Factory iresFactory) { + Model m = ModelFactory.createDefaultModel(); + for (String queryID : data.keySet()) { + Resource queryRes = iresFactory.getTaskQueryResource(queryID); + m.add(createAggregatedModel(data.get(queryID), queryRes)); + } + return m; + } + + private static Model createAggregatedModel(List data, Resource queryRes) { + Model m = ModelFactory.createDefaultModel(); + BigInteger succeeded = BigInteger.ZERO; + BigInteger failed = BigInteger.ZERO; + Optional resultSize = Optional.empty(); + BigInteger wrongCodes = BigInteger.ZERO; + BigInteger timeOuts = BigInteger.ZERO; + BigInteger unknownExceptions = BigInteger.ZERO; + Duration totalTime = Duration.ZERO; + + for (HttpWorker.ExecutionStats exec : data) { + switch (exec.endState()) { + case SUCCESS -> succeeded = succeeded.add(BigInteger.ONE); + case TIMEOUT -> timeOuts = timeOuts.add(BigInteger.ONE); + case HTTP_ERROR -> wrongCodes = wrongCodes.add(BigInteger.ONE); + case MISCELLANEOUS_EXCEPTION -> unknownExceptions = unknownExceptions.add(BigInteger.ONE); + } + + if (!exec.successful()) + failed = failed.add(BigInteger.ONE); + + totalTime = totalTime.plus(exec.duration()); + if (exec.contentLength().isPresent()) + resultSize = Optional.of(BigInteger.valueOf(exec.contentLength().getAsLong())); + } + + m.add(queryRes, IPROP.succeeded, ResourceFactory.createTypedLiteral(succeeded)); + m.add(queryRes, IPROP.failed, ResourceFactory.createTypedLiteral(failed)); + m.add(queryRes, IPROP.resultSize, ResourceFactory.createTypedLiteral(resultSize.orElse(BigInteger.valueOf(-1)))); + m.add(queryRes, IPROP.timeOuts, ResourceFactory.createTypedLiteral(timeOuts)); + m.add(queryRes, IPROP.wrongCodes, ResourceFactory.createTypedLiteral(wrongCodes)); + m.add(queryRes, IPROP.unknownException, ResourceFactory.createTypedLiteral(unknownExceptions)); + m.add(queryRes, IPROP.totalTime, ResourceFactory.createTypedLiteral(toXSDDurationInSeconds(totalTime))); + m.add(queryRes, RDF.type, IONT.executedQuery); + + return m; + } +} diff --git a/src/main/java/org/aksw/iguana/cc/metrics/impl/AvgQPS.java b/src/main/java/org/aksw/iguana/cc/metrics/impl/AvgQPS.java new file mode 100644 index 000000000..cb27e55b4 --- /dev/null +++ b/src/main/java/org/aksw/iguana/cc/metrics/impl/AvgQPS.java @@ -0,0 +1,45 @@ +package org.aksw.iguana.cc.metrics.impl; + +import org.aksw.iguana.cc.metrics.Metric; +import org.aksw.iguana.cc.metrics.TaskMetric; +import org.aksw.iguana.cc.metrics.WorkerMetric; +import org.aksw.iguana.cc.worker.HttpWorker; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.List; + +public class AvgQPS extends Metric implements TaskMetric, WorkerMetric { + + public AvgQPS() { + super("Average Queries per Second", "AvgQPS", "This metric calculates the average QPS between all queries."); + } + + @Override + public Number calculateTaskMetric(List workers, List[][] data) { + final var sum = workers.stream() + .map(worker -> (BigDecimal) this.calculateWorkerMetric(worker.config(), data[(int) worker.getWorkerID()])) + .reduce(BigDecimal.ZERO, BigDecimal::add); + + try { + return sum.divide(BigDecimal.valueOf(data.length), 10, RoundingMode.HALF_UP); + } catch (ArithmeticException e) { + return BigDecimal.ZERO; + } + } + + @Override + public Number calculateWorkerMetric(HttpWorker.Config worker, List[] data) { + BigDecimal sum = BigDecimal.ZERO; + QPS qpsmetric = new QPS(); + for (List datum : data) { + sum = sum.add((BigDecimal) qpsmetric.calculateQueryMetric(datum)); + } + + try { + return sum.divide(BigDecimal.valueOf(data.length), 10, RoundingMode.HALF_UP); + } catch (ArithmeticException e) { + return BigDecimal.ZERO; + } + } +} diff --git a/src/main/java/org/aksw/iguana/cc/metrics/impl/EachExecutionStatistic.java b/src/main/java/org/aksw/iguana/cc/metrics/impl/EachExecutionStatistic.java new file mode 100644 index 000000000..d8b267f56 --- /dev/null +++ b/src/main/java/org/aksw/iguana/cc/metrics/impl/EachExecutionStatistic.java @@ -0,0 +1,56 @@ +package org.aksw.iguana.cc.metrics.impl; + +import org.aksw.iguana.cc.metrics.Metric; +import org.aksw.iguana.cc.metrics.ModelWritingMetric; +import org.aksw.iguana.cc.worker.HttpWorker; +import org.aksw.iguana.commons.rdf.IPROP; +import org.aksw.iguana.commons.rdf.IRES; +import org.aksw.iguana.commons.time.TimeUtils; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; +import org.apache.jena.rdf.model.Resource; +import org.apache.jena.rdf.model.ResourceFactory; + +import java.math.BigInteger; +import java.util.List; + +public class EachExecutionStatistic extends Metric implements ModelWritingMetric { + + public EachExecutionStatistic() { + super("Each Query Execution Statistic", "EachQuery", "This metric saves the statistics of each query execution."); + } + + @Override + public Model createMetricModel(List workers, List[][] data, IRES.Factory iresFactory) { + Model m = ModelFactory.createDefaultModel(); + for (var worker : workers) { + for (int i = 0; i < worker.config().queries().getQueryCount(); i++) { + Resource workerQueryResource = iresFactory.getWorkerQueryResource(worker, i); + Resource queryRes = IRES.getResource(worker.config().queries().getQueryId(i)); + BigInteger run = BigInteger.ONE; + for (HttpWorker.ExecutionStats exec : data[(int) worker.getWorkerID()][i]) { + Resource runRes = iresFactory.getWorkerQueryRunResource(worker, i, run); + m.add(workerQueryResource, IPROP.queryExecution, runRes); + m.add(runRes, IPROP.time, TimeUtils.createTypedDurationLiteral(exec.duration())); + m.add(runRes, IPROP.startTime, TimeUtils.createTypedInstantLiteral(exec.startTime())); + m.add(runRes, IPROP.success, ResourceFactory.createTypedLiteral(exec.successful())); + m.add(runRes, IPROP.run, ResourceFactory.createTypedLiteral(run)); + m.add(runRes, IPROP.code, ResourceFactory.createTypedLiteral(exec.endState().value)); + m.add(runRes, IPROP.resultSize, ResourceFactory.createTypedLiteral(exec.contentLength().orElse(-1))); + m.add(runRes, IPROP.queryID, queryRes); + if (exec.responseBodyHash().isPresent()) { + Resource responseBodyRes = IRES.getResponsebodyResource(exec.responseBodyHash().getAsLong()); + m.add(runRes, IPROP.responseBody, responseBodyRes); + m.add(responseBodyRes, IPROP.responseBodyHash, ResourceFactory.createTypedLiteral(exec.responseBodyHash().getAsLong())); + } + if (exec.error().isPresent()) + m.add(runRes, IPROP.exception, ResourceFactory.createTypedLiteral(exec.error().get().toString())); + if (exec.httpStatusCode().isPresent()) + m.add(runRes, IPROP.httpCode, ResourceFactory.createTypedLiteral(exec.httpStatusCode().get().toString())); + run = run.add(BigInteger.ONE); + } + } + } + return m; + } +} diff --git a/src/main/java/org/aksw/iguana/cc/metrics/impl/NoQ.java b/src/main/java/org/aksw/iguana/cc/metrics/impl/NoQ.java new file mode 100644 index 000000000..411f73ca9 --- /dev/null +++ b/src/main/java/org/aksw/iguana/cc/metrics/impl/NoQ.java @@ -0,0 +1,38 @@ +package org.aksw.iguana.cc.metrics.impl; + +import org.aksw.iguana.cc.metrics.Metric; +import org.aksw.iguana.cc.metrics.TaskMetric; +import org.aksw.iguana.cc.metrics.WorkerMetric; +import org.aksw.iguana.cc.worker.HttpWorker; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.List; + +public class NoQ extends Metric implements TaskMetric, WorkerMetric { + + public NoQ() { + super("Number of Queries", "NoQ", "This metric calculates the number of successfully executed queries."); + } + + @Override + public Number calculateTaskMetric(List workers, List[][] data) { + final var sum = workers.stream() + .map(worker -> (BigInteger) this.calculateWorkerMetric(worker.config(), data[(int) worker.getWorkerID()])) + .reduce(BigInteger.ZERO, BigInteger::add); + return sum; + } + + @Override + public Number calculateWorkerMetric(HttpWorker.Config worker, List[] data) { + BigInteger sum = BigInteger.ZERO; + for (List datum : data) { + for (HttpWorker.ExecutionStats exec : datum) { + if (exec.successful()) { + sum = sum.add(BigInteger.ONE); + } + } + } + return sum; + } +} diff --git a/src/main/java/org/aksw/iguana/cc/metrics/impl/NoQPH.java b/src/main/java/org/aksw/iguana/cc/metrics/impl/NoQPH.java new file mode 100644 index 000000000..790f17a89 --- /dev/null +++ b/src/main/java/org/aksw/iguana/cc/metrics/impl/NoQPH.java @@ -0,0 +1,47 @@ +package org.aksw.iguana.cc.metrics.impl; + +import org.aksw.iguana.cc.metrics.Metric; +import org.aksw.iguana.cc.metrics.TaskMetric; +import org.aksw.iguana.cc.metrics.WorkerMetric; +import org.aksw.iguana.cc.worker.HttpWorker; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.RoundingMode; +import java.time.Duration; +import java.util.List; + +public class NoQPH extends Metric implements TaskMetric, WorkerMetric { + + public NoQPH() { + super("Number of Queries per Hour", "NoQPH", "This metric calculates the number of successfully executed queries per hour."); + } + @Override + public Number calculateTaskMetric(List workers, List[][] data) { + final var sum = workers.stream() + .map(worker -> (BigDecimal) this.calculateWorkerMetric(worker.config(), data[(int) worker.getWorkerID()])) + .reduce(BigDecimal.ZERO, BigDecimal::add); + return sum; + } + + @Override + public Number calculateWorkerMetric(HttpWorker.Config worker, List[] data) { + BigDecimal successes = BigDecimal.ZERO; + Duration totalTime = Duration.ZERO; + for (List datum : data) { + for (HttpWorker.ExecutionStats exec : datum) { + if (exec.successful()) { + successes = successes.add(BigDecimal.ONE); + totalTime = totalTime.plus(exec.duration()); + } + } + } + BigDecimal tt = (new BigDecimal(BigInteger.valueOf(totalTime.toNanos()), 9)).divide(BigDecimal.valueOf(3600), 20, RoundingMode.HALF_UP); + + try { + return successes.divide(tt, 10, RoundingMode.HALF_UP); + } catch (ArithmeticException e) { + return BigDecimal.ZERO; + } + } +} diff --git a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/PAvgQPS.java b/src/main/java/org/aksw/iguana/cc/metrics/impl/PAvgQPS.java similarity index 52% rename from src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/PAvgQPS.java rename to src/main/java/org/aksw/iguana/cc/metrics/impl/PAvgQPS.java index f71d42ae3..d22472a55 100644 --- a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/PAvgQPS.java +++ b/src/main/java/org/aksw/iguana/cc/metrics/impl/PAvgQPS.java @@ -1,33 +1,29 @@ -package org.aksw.iguana.cc.tasks.stresstest.metrics.impl; +package org.aksw.iguana.cc.metrics.impl; -import org.aksw.iguana.cc.model.QueryExecutionStats; -import org.aksw.iguana.cc.tasks.stresstest.StresstestMetadata; -import org.aksw.iguana.cc.worker.WorkerMetadata; -import org.aksw.iguana.cc.tasks.stresstest.metrics.Metric; -import org.aksw.iguana.cc.tasks.stresstest.metrics.TaskMetric; -import org.aksw.iguana.cc.tasks.stresstest.metrics.WorkerMetric; -import org.aksw.iguana.commons.annotation.Shorthand; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.aksw.iguana.cc.metrics.Metric; +import org.aksw.iguana.cc.metrics.TaskMetric; +import org.aksw.iguana.cc.metrics.WorkerMetric; +import org.aksw.iguana.cc.worker.HttpWorker; import java.math.BigDecimal; import java.math.RoundingMode; import java.util.List; -@Shorthand("PAvgQPS") public class PAvgQPS extends Metric implements TaskMetric, WorkerMetric { private final int penalty; - public PAvgQPS(Integer penalty) { + public PAvgQPS(@JsonProperty("penalty") Integer penalty) { super("Penalized Average Queries per Second", "PAvgQPS", "This metric calculates the average QPS between all queries. Failed executions receive a time penalty."); this.penalty = penalty; } @Override - public Number calculateTaskMetric(StresstestMetadata task, List[][] data) { - BigDecimal sum = BigDecimal.ZERO; - for (WorkerMetadata worker : task.workers()) { - sum = sum.add((BigDecimal) this.calculateWorkerMetric(worker, data[worker.workerID()])); - } + public Number calculateTaskMetric(List workers, List[][] data) { + final var sum = workers.stream() + .map(worker -> (BigDecimal) this.calculateWorkerMetric(worker.config(), data[(int) worker.getWorkerID()])) + .reduce(BigDecimal.ZERO, BigDecimal::add); try { return sum.divide(BigDecimal.valueOf(data.length), 10, RoundingMode.HALF_UP); @@ -37,10 +33,10 @@ public Number calculateTaskMetric(StresstestMetadata task, List[] data) { + public Number calculateWorkerMetric(HttpWorker.Config worker, List[] data) { BigDecimal sum = BigDecimal.ZERO; PQPS pqpsmetric = new PQPS(penalty); - for (List datum : data) { + for (List datum : data) { sum = sum.add((BigDecimal) pqpsmetric.calculateQueryMetric(datum)); } if (data.length == 0) { diff --git a/src/main/java/org/aksw/iguana/cc/metrics/impl/PQPS.java b/src/main/java/org/aksw/iguana/cc/metrics/impl/PQPS.java new file mode 100644 index 000000000..78b237c5e --- /dev/null +++ b/src/main/java/org/aksw/iguana/cc/metrics/impl/PQPS.java @@ -0,0 +1,43 @@ +package org.aksw.iguana.cc.metrics.impl; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.aksw.iguana.cc.metrics.Metric; +import org.aksw.iguana.cc.metrics.QueryMetric; +import org.aksw.iguana.cc.worker.HttpWorker; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.RoundingMode; +import java.time.Duration; +import java.util.List; + +public class PQPS extends Metric implements QueryMetric { + + private final int penalty; + + public PQPS(@JsonProperty("penalty") Integer penalty) { + super("Penalized Queries per Second", "PQPS", "This metric calculates for each query the amount of executions per second. Failed executions receive a time penalty."); + this.penalty = penalty; + } + + @Override + public Number calculateQueryMetric(List data) { + BigDecimal numberOfExecutions = BigDecimal.ZERO; + Duration totalTime = Duration.ZERO; + for (HttpWorker.ExecutionStats exec : data) { + numberOfExecutions = numberOfExecutions.add(BigDecimal.ONE); + if (exec.successful()) { + totalTime = totalTime.plus(exec.duration()); + } else { + totalTime = totalTime.plusMillis(penalty); + } + } + BigDecimal tt = (new BigDecimal(BigInteger.valueOf(totalTime.toNanos()), 9)); + + try { + return numberOfExecutions.divide(tt, 10, RoundingMode.HALF_UP); + } catch (ArithmeticException e) { + return BigDecimal.ZERO; + } + } +} diff --git a/src/main/java/org/aksw/iguana/cc/metrics/impl/QMPH.java b/src/main/java/org/aksw/iguana/cc/metrics/impl/QMPH.java new file mode 100644 index 000000000..d2ae19143 --- /dev/null +++ b/src/main/java/org/aksw/iguana/cc/metrics/impl/QMPH.java @@ -0,0 +1,49 @@ +package org.aksw.iguana.cc.metrics.impl; + +import org.aksw.iguana.cc.metrics.Metric; +import org.aksw.iguana.cc.metrics.TaskMetric; +import org.aksw.iguana.cc.metrics.WorkerMetric; +import org.aksw.iguana.cc.worker.HttpWorker; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.RoundingMode; +import java.time.Duration; +import java.util.List; + +public class QMPH extends Metric implements TaskMetric, WorkerMetric { + + public QMPH() { + super("Query Mixes per Hour", "QMPH", "This metric calculates the amount of query mixes (a given set of queries) that are executed per hour."); + } + + @Override + public Number calculateTaskMetric(List workers, List[][] data) { + final var sum = workers.stream() + .map(worker -> (BigDecimal) this.calculateWorkerMetric(worker.config(), data[(int) worker.getWorkerID()])) + .reduce(BigDecimal.ZERO, BigDecimal::add); + return sum; + } + + @Override + public Number calculateWorkerMetric(HttpWorker.Config worker, List[] data) { + BigDecimal successes = BigDecimal.ZERO; + BigDecimal noq = BigDecimal.valueOf(worker.queries().getQueryCount()); + Duration totalTime = Duration.ZERO; + for (List datum : data) { + for (HttpWorker.ExecutionStats exec : datum) { + if (exec.successful()) { + successes = successes.add(BigDecimal.ONE); + totalTime = totalTime.plus(exec.duration()); + } + } + } + BigDecimal tt = (new BigDecimal(BigInteger.valueOf(totalTime.toNanos()), 9)).divide(BigDecimal.valueOf(3600), 20, RoundingMode.HALF_UP); + + try { + return successes.divide(tt, 10, RoundingMode.HALF_UP).divide(noq, 10, RoundingMode.HALF_UP); + } catch (ArithmeticException e) { + return BigDecimal.ZERO; + } + } +} diff --git a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/QPS.java b/src/main/java/org/aksw/iguana/cc/metrics/impl/QPS.java similarity index 56% rename from src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/QPS.java rename to src/main/java/org/aksw/iguana/cc/metrics/impl/QPS.java index 888c7bb49..b20e2d84d 100644 --- a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/QPS.java +++ b/src/main/java/org/aksw/iguana/cc/metrics/impl/QPS.java @@ -1,10 +1,8 @@ -package org.aksw.iguana.cc.tasks.stresstest.metrics.impl; +package org.aksw.iguana.cc.metrics.impl; -import org.aksw.iguana.cc.model.QueryExecutionStats; -import org.aksw.iguana.cc.tasks.stresstest.metrics.Metric; -import org.aksw.iguana.cc.tasks.stresstest.metrics.QueryMetric; -import org.aksw.iguana.commons.annotation.Shorthand; -import org.aksw.iguana.commons.constants.COMMON; +import org.aksw.iguana.cc.metrics.Metric; +import org.aksw.iguana.cc.metrics.QueryMetric; +import org.aksw.iguana.cc.worker.HttpWorker; import java.math.BigDecimal; import java.math.BigInteger; @@ -12,7 +10,6 @@ import java.time.Duration; import java.util.List; -@Shorthand("QPS") public class QPS extends Metric implements QueryMetric { public QPS() { @@ -20,13 +17,13 @@ public QPS() { } @Override - public Number calculateQueryMetric(List data) { + public Number calculateQueryMetric(List data) { BigDecimal successes = BigDecimal.ZERO; Duration totalTime = Duration.ZERO; - for (QueryExecutionStats exec : data) { - if (exec.responseCode() == COMMON.QUERY_SUCCESS) { + for (HttpWorker.ExecutionStats exec : data) { + if (exec.successful()) { successes = successes.add(BigDecimal.ONE); - totalTime = totalTime.plusNanos((long) exec.executionTime() * 1000000); + totalTime = totalTime.plus(exec.duration()); } } BigDecimal tt = (new BigDecimal(BigInteger.valueOf(totalTime.toNanos()), 9)); diff --git a/src/main/java/org/aksw/iguana/cc/model/QueryExecutionStats.java b/src/main/java/org/aksw/iguana/cc/model/QueryExecutionStats.java deleted file mode 100644 index c15a6b478..000000000 --- a/src/main/java/org/aksw/iguana/cc/model/QueryExecutionStats.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.aksw.iguana.cc.model; - -/** - * Wrapper for a query execution. - */ -public record QueryExecutionStats ( - String queryID, - long responseCode, - double executionTime, - long resultSize -) { - public QueryExecutionStats(String queryID, long responseCode, double executionTime) { - this(queryID, responseCode, executionTime, 0); - } -} diff --git a/src/main/java/org/aksw/iguana/cc/model/QueryResultHashKey.java b/src/main/java/org/aksw/iguana/cc/model/QueryResultHashKey.java deleted file mode 100644 index 21ad255c6..000000000 --- a/src/main/java/org/aksw/iguana/cc/model/QueryResultHashKey.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.aksw.iguana.cc.model; - -import java.util.Objects; - -/** - * Creates a Result Hash key for a query, thus a result size only has to be checked once and it will be cached using this key - */ -public class QueryResultHashKey { - - private String queryId; - private long uniqueKey; - - public QueryResultHashKey(String queryId, long uniqueKey) { - this.queryId = queryId; - this.uniqueKey = uniqueKey; - } - - public String getQueryId() { - return queryId; - } - - public void setQueryId(String queryId) { - this.queryId = queryId; - } - - public long getUniqueKey() { - return uniqueKey; - } - - public void setUniqueKey(long uniqueKey) { - this.uniqueKey = uniqueKey; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - QueryResultHashKey that = (QueryResultHashKey) o; - return uniqueKey == that.uniqueKey && - queryId.equals(that.queryId); - } - - @Override - public int hashCode() { - return Objects.hash(queryId, uniqueKey); - } - - @Override - public String toString() { - return "QueryResultHashKey{" + - "queryId='" + queryId + '\'' + - ", uniqueKey=" + uniqueKey + - '}'; - } -} diff --git a/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java b/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java index dee5bfab1..1c9ac2eee 100644 --- a/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java +++ b/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java @@ -1,28 +1,28 @@ package org.aksw.iguana.cc.query.handler; -import org.aksw.iguana.cc.lang.LanguageProcessor; -import org.aksw.iguana.cc.lang.QueryWrapper; -import org.aksw.iguana.cc.query.pattern.PatternHandler; +import com.fasterxml.jackson.annotation.*; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; import org.aksw.iguana.cc.query.selector.QuerySelector; import org.aksw.iguana.cc.query.selector.impl.LinearQuerySelector; import org.aksw.iguana.cc.query.selector.impl.RandomQuerySelector; import org.aksw.iguana.cc.query.list.QueryList; import org.aksw.iguana.cc.query.list.impl.FileBasedQueryList; import org.aksw.iguana.cc.query.list.impl.InMemQueryList; -import org.aksw.iguana.cc.query.source.QuerySource; import org.aksw.iguana.cc.query.source.impl.FileLineQuerySource; import org.aksw.iguana.cc.query.source.impl.FileSeparatorQuerySource; import org.aksw.iguana.cc.query.source.impl.FolderQuerySource; -import org.aksw.iguana.commons.factory.TypedFactory; -import org.apache.jena.rdf.model.Model; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.ArrayList; +import java.io.InputStream; +import java.nio.file.Path; import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.Objects; /** * The QueryHandler is used by every worker that extends the AbstractWorker. @@ -32,185 +32,176 @@ * * @author frensing */ +@JsonDeserialize(using = QueryHandler.Deserializer.class) public class QueryHandler { + static class Deserializer extends StdDeserializer { + final HashMap queryHandlers = new HashMap<>(); + protected Deserializer(Class vc) { + super(vc); + } - protected final Logger LOGGER = LoggerFactory.getLogger(QueryHandler.class); + protected Deserializer() { + this(null); + } - protected Map config; - protected Integer workerID; - protected String location; - protected int hashcode; + @Override + public QueryHandler deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException { + QueryHandler.Config queryHandlerConfig = ctxt.readValue(jp, QueryHandler.Config.class); + if (!queryHandlers.containsKey(queryHandlerConfig)) + queryHandlers.put(queryHandlerConfig, new QueryHandler(queryHandlerConfig)); - protected boolean caching; + return queryHandlers.get(queryHandlerConfig); + } + } - protected QuerySelector querySelector; + public record Config ( + String path, + Format format, + String separator, + Boolean caching, + Order order, + Long seed, + Language lang + ) { + public Config(@JsonProperty(required = true) String path, Format format, String separator, Boolean caching, Order order, Long seed, Language lang) { + this.path = path; + this.format = (format == null ? Format.ONE_PER_LINE : format); + this.caching = (caching == null || caching); + this.order = (order == null ? Order.LINEAR : order); + this.seed = (seed == null ? 0 : seed); + this.lang = (lang == null ? Language.SPARQL : lang); + this.separator = (separator == null ? "" : separator); + } - protected QueryList queryList; + public enum Format { + @JsonEnumDefaultValue ONE_PER_LINE("one-per-line"), + SEPARATOR("separator"), + FOLDER("folder"); - protected LanguageProcessor langProcessor; + final String value; - public QueryHandler(Map config, Integer workerID) { - this.config = config; - this.workerID = workerID; + Format(String value) { + this.value = Objects.requireNonNullElse(value, "one-per-line"); + } - this.location = (String) config.get("location"); + @JsonValue + public String value() { + return value; + } + } - initQuerySet(); + public enum Order { + @JsonEnumDefaultValue LINEAR("linear"), + RANDOM("random"); - initQuerySelector(); - initLanguageProcessor(); + final String value; - this.hashcode = this.queryList.hashCode(); - } + Order(String value) { + this.value = value; + } - public void getNextQuery(StringBuilder queryStr, StringBuilder queryID) throws IOException { - int queryIndex = this.querySelector.getNextIndex(); - queryStr.append(this.queryList.getQuery(queryIndex)); - queryID.append(getQueryId(queryIndex)); - } + @JsonValue + public String value() { + return value; + } + } + + public enum Language { + @JsonEnumDefaultValue SPARQL("SPARQL"), + UNSPECIFIED("unspecified"); - public Model getTripleStats(String taskID) { - List queries = new ArrayList<>(this.queryList.size()); - for (int i = 0; i < this.queryList.size(); i++) { - try { - queries.add(new QueryWrapper(this.queryList.getQuery(i), getQueryId(i))); - } catch (Exception e) { - LOGGER.error("Could not parse query " + this.queryList.getName() + ":" + i, e); + final String value; + + Language(String value) { + this.value = value; + } + + @JsonValue + public String value() { + return value; } } - return this.langProcessor.generateTripleStats(queries, "" + this.hashcode, taskID); } - @Override - public int hashCode() { - return this.hashcode; - } + public record QueryStringWrapper(int index, String query) {} + public record QueryStreamWrapper(int index, InputStream queryInputStream) {} - public int getQueryCount() { - return this.queryList.size(); - } - public LanguageProcessor getLanguageProcessor() { - return this.langProcessor; - } + protected final Logger LOGGER = LoggerFactory.getLogger(QueryHandler.class); - /** - * This method initializes the PatternHandler if a pattern config is given, therefore - * this.config.get("pattern") should return an appropriate pattern configuration and not - * null. The PatternHandler uses the original query source to generate a new query source and list with - * the instantiated queries. - */ - private void initPatternQuerySet() { - Map patternConfig = (Map) this.config.get("pattern"); - PatternHandler patternHandler = new PatternHandler(patternConfig, createQuerySource()); + @JsonValue + final protected Config config; - initQuerySet(patternHandler.generateQuerySource()); - } + final protected QueryList queryList; + + private int workerCount = 0; // give every worker inside the same worker config an offset seed + + final protected int hashCode; /** - * Will initialize the QueryList. - * If caching is not set or set to true, the InMemQueryList will be used. Otherwise the FileBasedQueryList. - * - * @param querySource The QuerySource which contains the queries. + * Empty Constructor for Testing purposes. + * TODO: look for an alternative */ - private void initQuerySet(QuerySource querySource) { - this.caching = (Boolean) this.config.getOrDefault("caching", true); + protected QueryHandler() { + config = null; + queryList = null; + hashCode = 0; + } - if (this.caching) { - this.queryList = new InMemQueryList(this.location, querySource); - } else { - this.queryList = new FileBasedQueryList(this.location, querySource); - } + @JsonCreator + public QueryHandler(Config config) throws IOException { + final var querySource = switch (config.format()) { + case ONE_PER_LINE -> new FileLineQuerySource(Path.of(config.path())); + case SEPARATOR -> new FileSeparatorQuerySource(Path.of(config.path()), config.separator); + case FOLDER -> new FolderQuerySource(Path.of(config.path())); + }; + + queryList = (config.caching()) ? + new InMemQueryList(querySource) : + new FileBasedQueryList(querySource); + + this.config = config; + hashCode = queryList.hashCode(); } - /** - * This method initializes the QueryList for the QueryHandler. If a pattern configuration is specified, this method - * will execute initPatternQuerySet to create the QueryList. - */ - private void initQuerySet() { - if(this.config.containsKey("pattern")) { - initPatternQuerySet(); - } - else { - initQuerySet(createQuerySource()); + public QuerySelector getQuerySelectorInstance() { + switch (config.order()) { + case LINEAR -> { return new LinearQuerySelector(queryList.size()); } + case RANDOM -> { return new RandomQuerySelector(queryList.size(), config.seed() + workerCount++); } } + + throw new IllegalStateException("Unknown query selection order: " + config.order()); } - /** - * Will initialize the QuerySource. - * Depending on the format configuration, the FileLineQuerySource, - * FileSeparatorQuerySource or FolderQuerySource will be used. - * The FileSeparatorQuerySource can be further configured with a separator. - * - * @return The QuerySource which contains the queries. - */ - private QuerySource createQuerySource() { - Object formatObj = this.config.getOrDefault("format", "one-per-line"); - if (formatObj instanceof Map) { - Map format = (Map) formatObj; - if (format.containsKey("separator")) { - return new FileSeparatorQuerySource(this.location, (String) format.get("separator")); - } - } else { - switch ((String) formatObj) { - case "one-per-line": - return new FileLineQuerySource(this.location); - case "separator": - return new FileSeparatorQuerySource(this.location); - case "folder": - return new FolderQuerySource(this.location); - } - } - LOGGER.error("Could not create QuerySource for format {}", formatObj); - return null; + public QueryStringWrapper getNextQuery(QuerySelector querySelector) throws IOException { + final var queryIndex = querySelector.getNextIndex(); + return new QueryStringWrapper(queryIndex, queryList.getQuery(queryIndex)); } - /** - * Will initialize the QuerySelector that provides the next query index during the benchmark execution. - *

- * currently linear or random (with seed) are implemented - */ - private void initQuerySelector() { - Object orderObj = this.config.getOrDefault("order", "linear"); - - if (orderObj instanceof String) { - String order = (String) orderObj; - if (order.equals("linear")) { - this.querySelector = new LinearQuerySelector(this.queryList.size()); - return; - } - if (order.equals("random")) { - this.querySelector = new RandomQuerySelector(this.queryList.size(), this.workerID); - return; - } + public QueryStreamWrapper getNextQueryStream(QuerySelector querySelector) throws IOException { + final var queryIndex = querySelector.getNextIndex(); + return new QueryStreamWrapper(queryIndex, this.queryList.getQueryStream(queryIndex)); + } - LOGGER.error("Unknown order: " + order); - } - if (orderObj instanceof Map) { - Map order = (Map) orderObj; - if (order.containsKey("random")) { - Map random = (Map) order.get("random"); - Integer seed = (Integer) random.get("seed"); - this.querySelector = new RandomQuerySelector(this.queryList.size(), seed); - return; - } - LOGGER.error("Unknown order: " + order); - } + @Override + public int hashCode() { + return hashCode; } - private void initLanguageProcessor() { - Object langObj = this.config.getOrDefault("lang", "lang.SPARQL"); - if (langObj instanceof String) { - this.langProcessor = new TypedFactory().create((String) langObj, new HashMap<>()); - } else { - LOGGER.error("Unknown language: " + langObj); - } + public int getQueryCount() { + return this.queryList.size(); } public String getQueryId(int i) { - return this.queryList.getName() + ":" + i; + return this.queryList.hashCode() + ":" + i; } + /** + * Returns every query id in the format: queryListHash:index
+ * The index of a query inside the returned array is the same as the index inside the string. + * + * @return String[] of query ids + */ public String[] getAllQueryIds() { String[] out = new String[queryList.size()]; for (int i = 0; i < queryList.size(); i++) { diff --git a/src/main/java/org/aksw/iguana/cc/query/list/QueryList.java b/src/main/java/org/aksw/iguana/cc/query/list/QueryList.java index 39b0961cb..4006e917e 100644 --- a/src/main/java/org/aksw/iguana/cc/query/list/QueryList.java +++ b/src/main/java/org/aksw/iguana/cc/query/list/QueryList.java @@ -3,6 +3,7 @@ import org.aksw.iguana.cc.query.source.QuerySource; import java.io.IOException; +import java.io.InputStream; /** * The abstract class for a QueryList. A query list provides the queries to the QueryHandler. @@ -11,14 +12,12 @@ */ public abstract class QueryList { - /** This is the QuerySource from which the queries should be retrieved. */ - protected QuerySource querySource; - - /** A name for the query list. This is a part of the queryIDs. */ - protected String name; + /** + * This is the QuerySource from which the queries should be retrieved. + */ + final protected QuerySource querySource; - public QueryList(String name, QuerySource querySource) { - this.name = name; + public QueryList(QuerySource querySource) { this.querySource = querySource; } @@ -28,16 +27,7 @@ public QueryList(String name, QuerySource querySource) { * @return The amount of queries in the query list */ public int size() { - return this.querySource.size(); - } - - /** - * This method returns the name of the query list. - * - * @return The name of the query list - */ - public String getName() { - return this.name; + return querySource.size(); } /** @@ -47,7 +37,7 @@ public String getName() { */ @Override public int hashCode() { - return this.querySource.hashCode(); + return querySource.hashCode(); } /** @@ -57,4 +47,6 @@ public int hashCode() { * @return The query at the given index */ public abstract String getQuery(int index) throws IOException; + + public abstract InputStream getQueryStream(int index) throws IOException; } diff --git a/src/main/java/org/aksw/iguana/cc/query/list/impl/FileBasedQueryList.java b/src/main/java/org/aksw/iguana/cc/query/list/impl/FileBasedQueryList.java index 76d1ef459..f01c3ab63 100644 --- a/src/main/java/org/aksw/iguana/cc/query/list/impl/FileBasedQueryList.java +++ b/src/main/java/org/aksw/iguana/cc/query/list/impl/FileBasedQueryList.java @@ -4,6 +4,7 @@ import org.aksw.iguana.cc.query.source.QuerySource; import java.io.IOException; +import java.io.InputStream; /** * A query list which reads the queries directly from a file. @@ -12,12 +13,17 @@ */ public class FileBasedQueryList extends QueryList { - public FileBasedQueryList(String name, QuerySource querySource) { - super(name, querySource); + public FileBasedQueryList(QuerySource querySource) { + super(querySource); } @Override public String getQuery(int index) throws IOException { - return this.querySource.getQuery(index); + return querySource.getQuery(index); + } + + @Override + public InputStream getQueryStream(int index) throws IOException { + return querySource.getQueryStream(index); } } diff --git a/src/main/java/org/aksw/iguana/cc/query/list/impl/InMemQueryList.java b/src/main/java/org/aksw/iguana/cc/query/list/impl/InMemQueryList.java index d2cfb86b3..7e6d30a37 100644 --- a/src/main/java/org/aksw/iguana/cc/query/list/impl/InMemQueryList.java +++ b/src/main/java/org/aksw/iguana/cc/query/list/impl/InMemQueryList.java @@ -5,7 +5,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.List; /** @@ -18,24 +21,21 @@ public class InMemQueryList extends QueryList { private static final Logger LOGGER = LoggerFactory.getLogger(InMemQueryList.class); - private List queries; + private final List queries; - public InMemQueryList(String name, QuerySource querySource) { - super(name, querySource); - loadQueries(); + public InMemQueryList(QuerySource querySource) throws IOException { + super(querySource); + queries = this.querySource.getAllQueries().stream().map(s -> s.getBytes(StandardCharsets.UTF_8)).toList(); } - private void loadQueries() { - try { - this.queries = this.querySource.getAllQueries(); - } catch (IOException e) { - LOGGER.error("Could not read queries"); - } + @Override + public String getQuery(int index) { + return new String(this.queries.get(index), StandardCharsets.UTF_8); } @Override - public String getQuery(int index) { - return this.queries.get(index); + public InputStream getQueryStream(int index) { + return new ByteArrayInputStream(this.queries.get(index)); } @Override diff --git a/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java b/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java deleted file mode 100644 index 63cb1a907..000000000 --- a/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java +++ /dev/null @@ -1,213 +0,0 @@ -package org.aksw.iguana.cc.query.pattern; - -import org.aksw.iguana.cc.query.source.QuerySource; -import org.aksw.iguana.cc.query.source.impl.FileLineQuerySource; -import org.apache.jena.query.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.PrintWriter; -import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * This class is used to instantiate SPARQL pattern queries.
- * It will create and execute a SPARQL query against the provided SPARQL endpoint, that will retrieve fitting values for - * the variables in the pattern query. - *

- * The instantiated queries are located in a text file, which is created at the given location. - * If a fitting query file is already present, the queries will not be instantiated again. - * - * @author frensing - */ -public class PatternHandler { - private static final Logger LOGGER = LoggerFactory.getLogger(PatternHandler.class); - - private final Map config; - private final QuerySource querySource; - private String endpoint; - private Long limit; - private String outputFolder; - - public PatternHandler(Map config, QuerySource querySource) { - this.config = config; - this.querySource = querySource; - - init(); - } - - /** - * This method will generate the queries from the given patterns, write them - * to a file, and return a QuerySource based on that file. - * The QuerySource is then used in the QueryHandler to get the queries. - * - * @return QuerySource containing the instantiated queries - */ - public QuerySource generateQuerySource() { - File cacheFile = new File(this.outputFolder + File.separator + this.querySource.hashCode()); - if (cacheFile.exists()) { - - LOGGER.warn("Output file already exists. Will not generate queries again. To generate them new remove the {{}} file", cacheFile.getAbsolutePath()); - - } else { - LOGGER.info("Generating queries for pattern queries"); - File outFolder = new File(this.outputFolder); - if (!outFolder.exists()) { - if(!outFolder.mkdirs()) { - LOGGER.error("Failed to create folder for the generated queries"); - } - } - - try (PrintWriter pw = new PrintWriter(cacheFile)) { - for (int i = 0; i < this.querySource.size(); i++) { - for (String query : generateQueries(this.querySource.getQuery(i))) { - pw.println(query); - } - } - } catch (Exception e) { - LOGGER.error("Could not write to file", e); - } - } - - return new FileLineQuerySource(cacheFile.getAbsolutePath()); - } - - /** - * Initializes the PatternHandler - * Sets up the output folder, the endpoint and the limit. - */ - private void init() { - this.endpoint = (String) this.config.get("endpoint"); - if (this.endpoint == null) { - LOGGER.error("No endpoint given for pattern handler"); - } - - this.outputFolder = (String) this.config.getOrDefault("outputFolder", "queryCache"); - - Object limitObj = this.config.getOrDefault("limit", 2000L); - if (limitObj instanceof Number) { - this.limit = ((Number) limitObj).longValue(); - } else if (limitObj instanceof String) { - this.limit = Long.parseLong((String) limitObj); - } else { - LOGGER.error("could not parse limit"); - } - } - - /** - * This method generates a list of queries for a given pattern query. - * - * @param query String of the pattern query - * @return List of generated queries as strings - */ - protected List generateQueries(String query) { - List queries = new LinkedList<>(); - - try { - // if query is already an instance, we do not need to generate anything - QueryFactory.create(query); - LOGGER.debug("Query is already an instance: {{}}", query); - queries.add(query); - return queries; - } catch (Exception ignored) { - } - - // Replace the pattern variables with real variables and store them to the Set varNames - Set varNames = new HashSet<>(); - String command = replaceVars(query, varNames); - - // Generate parameterized sparql string to construct final queries - ParameterizedSparqlString pss = new ParameterizedSparqlString(); - pss.setCommandText(command); - - ResultSet exchange = getInstanceVars(pss, varNames); - - // exchange vars in PSS - if (!exchange.hasNext()) { - //no solution - LOGGER.warn("Pattern query has no solution, will use variables instead of var instances: {{}}", pss); - queries.add(command); - } - while (exchange.hasNext()) { - QuerySolution solution = exchange.next(); - for (String var : exchange.getResultVars()) { - //exchange variable with - pss.clearParam(var); - pss.setParam(var, solution.get(var)); - } - queries.add(pss.toString()); - } - LOGGER.debug("Found instances {}", queries); - - return queries; - } - - /** - * Replaces the pattern variables of the pattern query with actual variables and returns it. - * The names of the replaced variables will be stored in the set. - * - * @param queryStr String of the pattern query - * @param varNames This set will be extended by the strings of the replaced variable names - * @return The pattern query with the actual variables instead of pattern variables - */ - protected String replaceVars(String queryStr, Set varNames) { - String command = queryStr; - Pattern pattern = Pattern.compile("%%var[0-9]+%%"); - Matcher m = pattern.matcher(queryStr); - while (m.find()) { - String patternVariable = m.group(); - String var = patternVariable.replace("%", ""); - command = command.replace(patternVariable, "?" + var); - varNames.add(var); - } - return command; - } - - /** - * Generates valid values for the given variables in the query. - * - * @param pss The query, whose variables should be instantiated - * @param varNames The set of variables in the query that should be instantiated - * @return ResultSet that contains valid values for the given variables of the query - */ - protected ResultSet getInstanceVars(ParameterizedSparqlString pss, Set varNames) { - QueryExecution exec = QueryExecutionFactory.createServiceRequest(this.endpoint, convertToSelect(pss, varNames)); - //return result set - return exec.execSelect(); - } - - /** - * Creates a new query that can find valid values for the variables in the original query. - * The variables, that should be instantiated, are named by the set. - * - * @param pss The query whose variables should be instantiated - * @param varNames The set of variables in the given query that should be instantiated - * @return Query that can evaluate valid values for the given variables of the original query - */ - protected Query convertToSelect(ParameterizedSparqlString pss, Set varNames) { - Query queryCpy; - try { - if (varNames.isEmpty()) { - return pss.asQuery(); - } - queryCpy = pss.asQuery(); - } catch (Exception e) { - LOGGER.error("The pattern query is not a valid SELECT query (is it perhaps an UPDATE query?): {{}}", pss.toString(), e); - return null; - } - - StringBuilder queryStr = new StringBuilder("SELECT DISTINCT "); - for (String varName : varNames) { - queryStr.append("?").append(varName).append(" "); - } - queryStr.append(queryCpy.getQueryPattern()); - ParameterizedSparqlString pssSelect = new ParameterizedSparqlString(); - pssSelect.setCommandText(queryStr.toString()); - pssSelect.setNsPrefixes(pss.getNsPrefixMap()); - pssSelect.append(" LIMIT "); - pssSelect.append(this.limit); - return pssSelect.asQuery(); - } -} diff --git a/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelector.java b/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelector.java index e66a954bc..824643213 100644 --- a/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelector.java +++ b/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelector.java @@ -1,17 +1,31 @@ package org.aksw.iguana.cc.query.selector; +import static java.text.MessageFormat.format; + /** * The QuerySelector provides a method to retrieve the index of a query, that should be executed next.
* It is used by the QueryHandler to get the next query. * * @author frensing */ -public interface QuerySelector { +public abstract class QuerySelector { + + protected int index = 0; + + protected final int size; + + public QuerySelector(int size) { + if (size <= 0) + throw new IllegalArgumentException(format("{0} size must be >0.", QuerySelector.class.getSimpleName())); + this.size = size; + } /** * This method gives the next query index that should be used. * * @return the next query index */ - int getNextIndex(); + public abstract int getNextIndex(); + + public abstract int getCurrentIndex(); } diff --git a/src/main/java/org/aksw/iguana/cc/query/selector/impl/LinearQuerySelector.java b/src/main/java/org/aksw/iguana/cc/query/selector/impl/LinearQuerySelector.java index a8a5abe37..3d3faad32 100644 --- a/src/main/java/org/aksw/iguana/cc/query/selector/impl/LinearQuerySelector.java +++ b/src/main/java/org/aksw/iguana/cc/query/selector/impl/LinearQuerySelector.java @@ -10,21 +10,29 @@ * * @author frensing */ -public class LinearQuerySelector implements QuerySelector { - - protected int querySelector; - - private int size; +public class LinearQuerySelector extends QuerySelector { public LinearQuerySelector(int size) { - this.size = size; + super(size); + index = -1; } @Override public int getNextIndex() { - if (this.querySelector >= this.size) { - this.querySelector = 0; + index++; + if (index >= this.size) { + index = 0; } - return this.querySelector++; + return index; + } + + /** + * Return the current index. This is the index of the last returned query. If no query was returned yet, it returns + * -1. + * @return + */ + @Override + public int getCurrentIndex() { + return index; } } diff --git a/src/main/java/org/aksw/iguana/cc/query/selector/impl/RandomQuerySelector.java b/src/main/java/org/aksw/iguana/cc/query/selector/impl/RandomQuerySelector.java index fdff4a248..80b18d51c 100644 --- a/src/main/java/org/aksw/iguana/cc/query/selector/impl/RandomQuerySelector.java +++ b/src/main/java/org/aksw/iguana/cc/query/selector/impl/RandomQuerySelector.java @@ -11,19 +11,23 @@ * * @author frensing */ -public class RandomQuerySelector implements QuerySelector { +public class RandomQuerySelector extends QuerySelector { - protected Random querySelector; - - private int size; + final protected Random indexGenerator; + int currentIndex; public RandomQuerySelector(int size, long seed) { - this.size = size; - this.querySelector = new Random(seed); + super(size); + indexGenerator = new Random(seed); } @Override public int getNextIndex() { - return this.querySelector.nextInt(this.size); + return currentIndex = this.indexGenerator.nextInt(this.size); + } + + @Override + public int getCurrentIndex() { + return currentIndex; } } diff --git a/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java b/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java index f505a8da6..38bdf966f 100644 --- a/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java +++ b/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java @@ -3,6 +3,8 @@ import org.aksw.iguana.cc.utils.FileUtils; import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; import java.util.List; /** @@ -14,9 +16,9 @@ public abstract class QuerySource { /** This string represents the path of the file or folder, that contains the queries. */ - protected String path; + final protected Path path; - public QuerySource(String path) { + public QuerySource(Path path) { this.path = path; } @@ -36,6 +38,8 @@ public QuerySource(String path) { */ public abstract String getQuery(int index) throws IOException; + public abstract InputStream getQueryStream(int index) throws IOException; + /** * This method returns all queries in the source as a list of Strings. * diff --git a/src/main/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySource.java b/src/main/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySource.java index 60185e0fc..992df4384 100644 --- a/src/main/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySource.java +++ b/src/main/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySource.java @@ -1,45 +1,18 @@ package org.aksw.iguana.cc.query.source.impl; -import org.aksw.iguana.cc.query.source.QuerySource; -import org.aksw.iguana.cc.utils.IndexedQueryReader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.aksw.iguana.cc.utils.FileUtils; import java.io.IOException; -import java.util.List; +import java.nio.file.Path; /** * The FileLineQuerySource reads queries from a file with one query per line. * * @author frensing */ -public class FileLineQuerySource extends QuerySource { - private static final Logger LOGGER = LoggerFactory.getLogger(FileLineQuerySource.class); - - private IndexedQueryReader iqr; - - public FileLineQuerySource(String path) { - super(path); - - try { - iqr = IndexedQueryReader.make(path); - } catch (IOException e) { - LOGGER.error("Failed to read this file for the queries: " + path + "\n" + e); - } - } - - @Override - public int size() { - return iqr.size(); - } - - @Override - public String getQuery(int index) throws IOException { - return iqr.readQuery(index); +public class FileLineQuerySource extends FileSeparatorQuerySource { + public FileLineQuerySource(Path filepath) throws IOException { + super(filepath, FileUtils.getLineEnding(filepath)); } - @Override - public List getAllQueries() throws IOException { - return iqr.readQueries(); - } } diff --git a/src/main/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySource.java b/src/main/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySource.java index 5b846be29..a0b07b10b 100644 --- a/src/main/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySource.java +++ b/src/main/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySource.java @@ -6,6 +6,8 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; import java.util.List; /** @@ -19,7 +21,7 @@ public class FileSeparatorQuerySource extends QuerySource { private static final String DEFAULT_SEPARATOR = "###"; - private IndexedQueryReader iqr; + final protected IndexedQueryReader iqr; /** * This constructor indexes the queries inside the given file. It assumes, that the queries inside the file are @@ -27,8 +29,9 @@ public class FileSeparatorQuerySource extends QuerySource { * * @param path path to the queries-file */ - public FileSeparatorQuerySource(String path) { - this(path, DEFAULT_SEPARATOR); + public FileSeparatorQuerySource(Path path) throws IOException { + super(path); + iqr = getIqr(path, DEFAULT_SEPARATOR); } /** @@ -39,19 +42,14 @@ public FileSeparatorQuerySource(String path) { * @param path path to the queries-file * @param separator string with which the queries inside the file are separated */ - public FileSeparatorQuerySource(String path, String separator) { + public FileSeparatorQuerySource(Path path, String separator) throws IOException { super(path); + iqr = getIqr(path, separator); + + } - try { - if(separator.isBlank()) { - iqr = IndexedQueryReader.makeWithEmptyLines(path); - } - else { - iqr = IndexedQueryReader.makeWithStringSeparator(path, separator); - } - } catch (IOException e) { - LOGGER.error("Failed to read this file for the queries: " + path + "\n" + e); - } + private static IndexedQueryReader getIqr(Path path, String separator) throws IOException { + return (separator.isEmpty()) ? IndexedQueryReader.makeWithEmptyLines(path) : IndexedQueryReader.makeWithStringSeparator(path, separator); } @Override @@ -64,6 +62,11 @@ public String getQuery(int index) throws IOException { return iqr.readQuery(index); } + @Override + public InputStream getQueryStream(int index) throws IOException { + return iqr.streamQuery(index); + } + @Override public List getAllQueries() throws IOException { return iqr.readQueries(); diff --git a/src/main/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySource.java b/src/main/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySource.java index 980c61ca1..04ae5fd12 100644 --- a/src/main/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySource.java +++ b/src/main/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySource.java @@ -2,14 +2,19 @@ import org.aksw.iguana.cc.query.source.QuerySource; import org.aksw.iguana.cc.utils.FileUtils; +import org.apache.commons.io.input.AutoCloseInputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; +import java.util.stream.Stream; + +import static java.text.MessageFormat.format; /** * The FileSeparatorQuerySource reads queries from a folder with query files. @@ -21,30 +26,26 @@ public class FolderQuerySource extends QuerySource { protected static final Logger LOGGER = LoggerFactory.getLogger(FolderQuerySource.class); - protected File[] files; + protected Path[] files; - public FolderQuerySource(String path) { + public FolderQuerySource(Path path) throws IOException { super(path); - indexFolder(); - } - - private void indexFolder() { - File dir = new File(this.path); - if (!dir.exists()) { - LOGGER.error("Folder does not exist"); - return; + if (!Files.isDirectory(path)) { + final var message = format("Folder does not exist {0}.", path); + LOGGER.error(message); + throw new IOException(message); } - if (!dir.isDirectory()) { - LOGGER.error("Path is not a folder"); - return; + + LOGGER.info("Indexing folder {}.", path); + + try (Stream pathStream = Files.list(path);) { + files = pathStream + .filter(p -> Files.isReadable(p) && Files.isRegularFile(p)) + .sorted() + .toArray(Path[]::new); } - LOGGER.info("indexing folder {}", this.path); - this.files = dir.listFiles(File::isFile); - if (this.files == null) - this.files = new File[]{}; - Arrays.sort(this.files); } @Override @@ -54,7 +55,12 @@ public int size() { @Override public String getQuery(int index) throws IOException { - return FileUtils.readFile(files[index].getAbsolutePath()); + return Files.readString(files[index], StandardCharsets.UTF_8); + } + + @Override + public InputStream getQueryStream(int index) throws IOException { + return new AutoCloseInputStream(new BufferedInputStream(new FileInputStream(files[index].toFile()))); } @Override @@ -68,6 +74,6 @@ public List getAllQueries() throws IOException { @Override public int hashCode() { - return FileUtils.getHashcodeFromFileContent(this.files[0].getAbsolutePath()); + return FileUtils.getHashcodeFromFileContent(this.files[0]); } } diff --git a/src/main/java/org/aksw/iguana/cc/storage/Storable.java b/src/main/java/org/aksw/iguana/cc/storage/Storable.java new file mode 100644 index 000000000..e45bcc976 --- /dev/null +++ b/src/main/java/org/aksw/iguana/cc/storage/Storable.java @@ -0,0 +1,40 @@ +package org.aksw.iguana.cc.storage; + +import org.apache.jena.rdf.model.Model; + +import java.util.List; + +/** + * This interface provides the functionality to store data in different formats. The data can be stored in CSV files + * or in RDF models. + */ +public interface Storable { + + record CSVData ( + String folderName, + List files + ) { + public record CSVFileData(String filename, String[][] data) {} + } + + interface AsCSV extends Storable { + + /** + * Converts the data into CSV files. The key of the map contains the file name for the linked entries. + * + * @return CSVFileData list which contains all the files and their data that should be created and stored + */ + CSVData toCSV(); + } + + interface AsRDF extends Storable { + + /** + * Converts the data into an RDF model, which will be added to the appropriate storages. + * + * @return RDF model that contains the data + */ + Model toRDF(); + } + +} diff --git a/src/main/java/org/aksw/iguana/cc/storage/Storage.java b/src/main/java/org/aksw/iguana/cc/storage/Storage.java new file mode 100644 index 000000000..06d1c2234 --- /dev/null +++ b/src/main/java/org/aksw/iguana/cc/storage/Storage.java @@ -0,0 +1,34 @@ +package org.aksw.iguana.cc.storage; + +import org.apache.jena.rdf.model.Model; + +/** + * Interface for the Result Storages + * + * @author f.conrads + * + */ +public interface Storage { + + /** + * Stores the task result into the storage. This method will be executed after a task has finished. + * Depending on the storages format, the storage class will need convert the data into the appropriate format. + * + * @param data the given result model + */ + void storeResult(Model data); + + /** + * General purpose method to store data into the storage. + * This method will mostly be used by the language processors to store their already formatted data.
+ * The default implementation will call the {@link #storeResult(Model)} method. This might not be the best solution + * for storages, that do not use RDF as their format. + * + * @param data the data to store + */ + default void storeData(Storable data) { + if (data instanceof Storable.AsRDF) { + storeResult(((Storable.AsRDF) data).toRDF()); + } + } +} diff --git a/src/main/java/org/aksw/iguana/cc/storage/impl/CSVStorage.java b/src/main/java/org/aksw/iguana/cc/storage/impl/CSVStorage.java new file mode 100644 index 000000000..4b37c439e --- /dev/null +++ b/src/main/java/org/aksw/iguana/cc/storage/impl/CSVStorage.java @@ -0,0 +1,417 @@ +package org.aksw.iguana.cc.storage.impl; + +import com.opencsv.CSVReader; +import com.opencsv.CSVWriter; +import com.opencsv.CSVWriterBuilder; +import com.opencsv.exceptions.CsvValidationException; +import org.aksw.iguana.cc.config.elements.StorageConfig; +import org.aksw.iguana.cc.metrics.*; +import org.aksw.iguana.cc.metrics.impl.AggregatedExecutionStatistics; +import org.aksw.iguana.cc.metrics.impl.EachExecutionStatistic; +import org.aksw.iguana.cc.storage.Storable; +import org.aksw.iguana.cc.storage.Storage; +import org.aksw.iguana.commons.rdf.IONT; +import org.aksw.iguana.commons.rdf.IPROP; +import org.apache.jena.arq.querybuilder.SelectBuilder; +import org.apache.jena.arq.querybuilder.WhereBuilder; +import org.apache.jena.query.*; +import org.apache.jena.rdf.model.*; +import org.apache.jena.sparql.lang.sparql_11.ParseException; +import org.apache.jena.vocabulary.RDF; +import org.apache.jena.vocabulary.RDFS; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.nio.file.*; +import java.util.*; +import java.util.function.Predicate; + +public class CSVStorage implements Storage { + + /** This private record is used to store information about the connections used in a task. */ + private record ConnectionInfo(String connection, String version, String dataset) {} + + public record Config(String directory) implements StorageConfig { + public Config(String directory) { + if (directory == null) { + directory = "results"; + } + Path path = Paths.get(directory); + if (Files.exists(path) && !Files.isDirectory(path)) { + throw new IllegalArgumentException("The given path is not a directory."); + } + this.directory = directory; + } + } + + private static final Logger LOGGER = LoggerFactory.getLogger(CSVStorage.class); + + private final List metrics; + + private final Path suiteFolder; + private Path currentFolder; + private final Path taskFile; + private final Path taskConfigFile; + + private List workerResources; + private Resource taskRes; + List connections; + + public CSVStorage(Config config, List metrics, String suiteID) { + this(config.directory(), metrics, suiteID); + } + + public CSVStorage(String folderPath, List metrics, String suiteID) { + this.metrics = metrics; + + Path parentFolder; + try { + parentFolder = Paths.get(folderPath); + } catch (InvalidPathException e) { + LOGGER.error("Can't store csv files, the given path is invalid.", e); + this.suiteFolder = null; + this.taskFile = null; + this.taskConfigFile = null; + return; + } + + this.suiteFolder = parentFolder.resolve("suite-" + suiteID); + this.taskFile = this.suiteFolder.resolve("suite-summary.csv"); + this.taskConfigFile = this.suiteFolder.resolve("task-configuration.csv"); + + if (Files.notExists(suiteFolder)) { + try { + Files.createDirectories(suiteFolder); + } catch (IOException e) { + LOGGER.error("Can't store csv files, directory could not be created.", e); + return; + } + } + + try { + Files.createFile(taskFile); + } catch (IOException e) { + LOGGER.error("Couldn't create the file: " + taskFile.toAbsolutePath(), e); + return; + } + + try { + Files.createFile(taskConfigFile); + } catch (IOException e) { + LOGGER.error("Couldn't create the file: " + taskFile.toAbsolutePath(), e); + return; + } + + // write headers for the suite-summary.csv file + try (CSVWriter csvWriter = getCSVWriter(taskFile)) { + Metric[] taskMetrics = metrics.stream().filter(x -> TaskMetric.class.isAssignableFrom(x.getClass())).toArray(Metric[]::new); + List headerList = new LinkedList<>(); + // headerList.addAll(List.of("connection", "dataset", "startDate", "endDate", "noOfWorkers")); + headerList.addAll(List.of("taskID", "startDate", "endDate", "noOfWorkers")); + headerList.addAll(Arrays.stream(taskMetrics).map(Metric::getAbbreviation).toList()); + String[] header = headerList.toArray(String[]::new); + csvWriter.writeNext(header, true); + } catch (IOException e) { + LOGGER.error("Error while writing to file: " + taskFile.toAbsolutePath(), e); + } + + // write headers for the task-configuration.csv file + try (CSVWriter csvWriter = getCSVWriter(taskConfigFile)) { + csvWriter.writeNext(new String[]{"taskID", "connection", "version", "dataset"}, true); + } catch (IOException e) { + LOGGER.error("Error while writing to file: " + taskConfigFile.toAbsolutePath(), e); + } + } + + /** + * Stores the task result into the storage. This method will be executed after a task has finished. + * + * @param data the given result model + */ + @Override + public void storeResult(Model data) { + try { + setObjectAttributes(data); + } catch (NoSuchElementException e) { + LOGGER.error("Error while querying the result model. The given model is probably incorrect.", e); + return; + } + + this.currentFolder = this.suiteFolder.resolve("task-" + retrieveTaskID(this.taskRes)); + try { + Files.createDirectory(this.currentFolder); + } catch (IOException e) { + LOGGER.error("Error while storing the task result in a csv file.", e); + } + + try { + storeTaskInfo(); + storeTaskResults(data); + } catch (IOException e) { + LOGGER.error("Error while storing the task result in a csv file.", e); + } catch (NoSuchElementException | ParseException e) { + LOGGER.error("Error while storing the task result in a csv file. The given model is probably incorrect.", e); + } + + try { + Path temp = createCSVFile("worker", "summary"); + storeWorkerResults(this.taskRes, temp, data, this.metrics); + for (Resource workerRes : workerResources) { + String workerID = data.listObjectsOfProperty(workerRes, IPROP.workerID).next().asLiteral().getLexicalForm(); + try { + Path file = createCSVFile("query", "summary", "worker", workerID); + Path file2 = createCSVFile("each", "execution", "worker", workerID); + storeSummarizedQueryResults(workerRes, file, data, this.metrics); + storeEachQueryResults(workerRes, file2, data, this.metrics); + } catch (IOException e) { + LOGGER.error("Error while storing the query results of a worker in a csv file.", e); + } catch (NoSuchElementException e) { + LOGGER.error("Error while storing the query results of a worker in a csv file. The given model is probably incorrect.", e); + } + } + } catch (IOException e) { + LOGGER.error("Error while storing the worker results in a csv file.", e); + } catch (NoSuchElementException e) { + LOGGER.error("Error while storing the worker results in a csv file. The given model is probably incorrect.", e); + } + + try { + Path file = createCSVFile("query", "summary", "task"); + storeSummarizedQueryResults(taskRes, file, data, this.metrics); + } catch (IOException e) { + LOGGER.error("Error while storing the query results of a task result in a csv file.", e); + } catch (NoSuchElementException e) { + LOGGER.error("Error while storing the query results of a task result in a csv file. The given model is probably incorrect.", e); + } + } + + @Override + public void storeData(Storable data) { + if (!(data instanceof Storable.AsCSV)) return; // dismiss data if it can't be stored as csv + Storable.CSVData csvdata = ((Storable.AsCSV) data).toCSV(); + + Path responseTypeDir = Path.of(csvdata.folderName()); + responseTypeDir = this.currentFolder.resolve(responseTypeDir); + + try { + Files.createDirectory(responseTypeDir); + } catch (FileAlreadyExistsException ignored) { + } catch (IOException e) { + LOGGER.error("Error while creating the directory for the language processor results. ", e); + return; + } + + for (var csvFile : csvdata.files()) { + // check for file extension + String filename = csvFile.filename().endsWith(".csv") ? csvFile.filename() : csvFile.filename() + ".csv"; + Path file = responseTypeDir.resolve(filename); + + int i = 1; // skip the header by default + + if (Files.notExists(file)) { + try { + Files.createFile(file); + } catch (IOException e) { + LOGGER.error("Error while creating a csv file for language processor results. The storing of language processor results will be skipped.", e); + return; + } + i = 0; // include header if file is new + } + + try (CSVWriter writer = getCSVWriter(file)) { + for (; i < csvFile.data().length; i++) { + writer.writeNext(csvFile.data()[i], true); + } + } catch (IOException e) { + LOGGER.error("Error while writing the data into a csv file for language processor results. The storing of language processor results will be skipped.", e); + return; + } + } + } + + /** + * This method sets the objects attributes by querying the given model. + * + * @param data the result model + * @throws NoSuchElementException might be thrown if the model is incorrect + */ + private void setObjectAttributes(Model data) throws NoSuchElementException { + // obtain connection information of task + this.connections = new ArrayList<>(); + ResIterator resIterator = data.listSubjectsWithProperty(RDF.type, IONT.connection); + while (resIterator.hasNext()) { + Resource connectionRes = resIterator.nextResource(); + NodeIterator nodeIterator = data.listObjectsOfProperty(connectionRes, RDFS.label); + String conString = nodeIterator.next().asLiteral().getLexicalForm(); + + // obtain connection version + String conVersionString = ""; + nodeIterator = data.listObjectsOfProperty(connectionRes, IPROP.version); + if (nodeIterator.hasNext()) { + conVersionString = nodeIterator.next().toString(); + } + + // obtain dataset + String conDatasetString = ""; + nodeIterator = data.listObjectsOfProperty(connectionRes, IPROP.dataset); + if (nodeIterator.hasNext()) { + conDatasetString = nodeIterator.next().toString(); + } + this.connections.add(new ConnectionInfo(conString, conVersionString, conDatasetString)); + } + + // obtain task type + resIterator = data.listSubjectsWithProperty(RDF.type, IONT.task); + this.taskRes = resIterator.nextResource(); + + // obtain worker resources + NodeIterator nodeIterator = data.listObjectsOfProperty(this.taskRes, IPROP.workerResult); + this.workerResources = nodeIterator.toList().stream().map(RDFNode::asResource).toList(); + } + + /** + * Creates a CSV file with the given name values that will be located inside the parent folder. The name value are + * joined together with the character '-'. Empty values will be ignored. + * + * @param nameValues strings that build up the name of the file + * @throws IOException if an I/O error occurs + * @return path object to the created CSV file + */ + private Path createCSVFile(String... nameValues) throws IOException { + // remove empty string values + nameValues = Arrays.stream(nameValues).filter(Predicate.not(String::isEmpty)).toArray(String[]::new); + String filename = String.join("-", nameValues) + ".csv"; + Path file = this.currentFolder.resolve(filename); + Files.createFile(file); + return file; + } + + private static void storeSummarizedQueryResults(Resource parentRes, Path file, Model data, List metrics) throws IOException, NoSuchElementException { + boolean containsAggrStats = !metrics.stream().filter(AggregatedExecutionStatistics.class::isInstance).toList().isEmpty(); + Metric[] queryMetrics = metrics.stream().filter(x -> QueryMetric.class.isAssignableFrom(x.getClass())).toArray(Metric[]::new); + + SelectBuilder sb = new SelectBuilder(); + sb.addWhere(parentRes, IPROP.query, "?eQ"); + queryProperties(sb, "?eQ", IPROP.queryID); + if (containsAggrStats) { + queryProperties(sb, "?eQ", IPROP.succeeded, IPROP.failed, IPROP.totalTime, IPROP.resultSize, IPROP.wrongCodes, IPROP.timeOuts, IPROP.unknownException); + } + queryMetrics(sb, "?eQ", queryMetrics); + + executeAndStoreQuery(sb, file, data); + } + + private static void storeEachQueryResults(Resource parentRes, Path file, Model data, List metrics) throws IOException { + boolean containsEachStats = !metrics.stream().filter(EachExecutionStatistic.class::isInstance).toList().isEmpty(); + if (!containsEachStats) { + return; + } + + SelectBuilder sb = new SelectBuilder(); + sb.addWhere(parentRes, IPROP.query, "?eQ") // variable name should be different from property names + .addWhere("?eQ", IPROP.queryExecution, "?exec") + .addOptional(new WhereBuilder().addWhere("?exec", IPROP.responseBody, "?rb").addWhere("?rb", IPROP.responseBodyHash, "?responseBodyHash")) + .addOptional(new WhereBuilder().addWhere("?exec", IPROP.exception, "?exception")) + .addOptional(new WhereBuilder().addWhere("?exec", IPROP.httpCode, "?httpCode")); + queryProperties(sb, "?exec", IPROP.queryID, IPROP.run, IPROP.success, IPROP.startTime, IPROP.time, IPROP.resultSize, IPROP.code); + sb.addVar("httpCode").addVar("exception").addVar("responseBodyHash"); + executeAndStoreQuery(sb, file, data); + } + + /** + * Stores the current task information into the task configuration file. + */ + private void storeTaskInfo() { + try (CSVWriter csvWriter = getCSVWriter(taskConfigFile)) { + for (ConnectionInfo connectionInfo : connections) { + csvWriter.writeNext(new String[]{this.taskRes.toString(), connectionInfo.connection(), connectionInfo.version(), connectionInfo.dataset()}, true); + } + } catch (IOException e) { + LOGGER.error("Error while writing to file: " + taskConfigFile.toAbsolutePath(), e); + } + } + + private void storeTaskResults(Model data) throws IOException, NoSuchElementException, ParseException { + Metric[] taskMetrics = metrics.stream().filter(x -> TaskMetric.class.isAssignableFrom(x.getClass())).toArray(Metric[]::new); + + SelectBuilder sb = new SelectBuilder(); + queryProperties(sb, String.format("<%s>", this.taskRes.toString()), IPROP.startDate, IPROP.endDate, IPROP.noOfWorkers); + queryMetrics(sb, String.format("<%s>", this.taskRes.toString()), taskMetrics); + + try (QueryExecution exec = QueryExecutionFactory.create(sb.build(), data); + CSVWriter csvWriter = getCSVWriter(taskFile); + ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + ResultSet results = exec.execSelect(); + ResultSetFormatter.outputAsCSV(baos, results); + + // workaround to remove the created header from the ResultSetFormatter + CSVReader reader = new CSVReader(new StringReader(baos.toString())); + try { + reader.readNext(); + + // inject connection and dataset information + String[] row = reader.readNext(); + String[] newRow = new String[row.length + 1]; + newRow[0] = this.taskRes.getURI(); + // newRow[0] = connection; + // newRow[1] = dataset; + System.arraycopy(row, 0, newRow, 1, row.length); + csvWriter.writeNext(newRow, true); + } catch (CsvValidationException ignored) { + // shouldn't happen + } + } + } + + private static void storeWorkerResults(Resource taskRes, Path file, Model data, List metrics) throws IOException, NoSuchElementException { + Metric[] workerMetrics = metrics.stream().filter(x -> WorkerMetric.class.isAssignableFrom(x.getClass())).toArray(Metric[]::new); + + SelectBuilder sb = new SelectBuilder(); + sb.addWhere(taskRes, IPROP.workerResult, "?worker"); + queryProperties(sb, "?worker", IPROP.workerID, IPROP.workerType, IPROP.noOfQueries, IPROP.timeOut, IPROP.startDate, IPROP.endDate); + queryMetrics(sb, "?worker", workerMetrics); + + executeAndStoreQuery(sb, file, data); + } + + private static CSVWriter getCSVWriter(Path file) throws IOException { + return (CSVWriter) new CSVWriterBuilder(new FileWriter(file.toAbsolutePath().toString(), true)) + .withQuoteChar('\"') + .withSeparator(',') + .withLineEnd("\n") + .build(); + } + + private static void queryProperties(SelectBuilder sb, String variable, Property... properties) { + for (Property prop : properties) { + sb.addVar(prop.getLocalName()).addWhere(variable, prop, "?" + prop.getLocalName()); + } + } + + private static void queryMetrics(SelectBuilder sb, String variable, Metric[] metrics) { + for (Metric m : metrics) { + // Optional, in case metric isn't created, because of failed executions + sb.addVar(m.getAbbreviation()).addOptional(variable, IPROP.createMetricProperty(m), "?" + m.getAbbreviation()); + } + } + + private static void executeAndStoreQuery(SelectBuilder sb, Path file, Model data) throws IOException { + try(QueryExecution exec = QueryExecutionFactory.create(sb.build(), data); + FileOutputStream fos = new FileOutputStream(file.toFile())) { + ResultSet results = exec.execSelect(); + ResultSetFormatter.outputAsCSV(fos, results); + } + } + + /** + * Retrieves the task ID from the given task resource. The current model doesn't save the task ID as a property of + * the task resource. Therefore, the task ID is extracted from the URI of the task resource. + * + * @param taskRes the task resource + * @return the task ID + */ + private static String retrieveTaskID(Resource taskRes) { + return taskRes.getURI().substring(taskRes.getURI().lastIndexOf("/") + 1); + } +} diff --git a/src/main/java/org/aksw/iguana/cc/storage/impl/RDFFileStorage.java b/src/main/java/org/aksw/iguana/cc/storage/impl/RDFFileStorage.java new file mode 100644 index 000000000..e3cce2801 --- /dev/null +++ b/src/main/java/org/aksw/iguana/cc/storage/impl/RDFFileStorage.java @@ -0,0 +1,98 @@ +package org.aksw.iguana.cc.storage.impl; + +import com.github.jsonldjava.shaded.com.google.common.base.Supplier; +import org.aksw.iguana.cc.config.elements.StorageConfig; +import org.aksw.iguana.cc.storage.Storage; +import org.apache.commons.io.FilenameUtils; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.riot.Lang; +import org.apache.jena.riot.RDFDataMgr; +import org.apache.jena.riot.RDFLanguages; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Calendar; +import java.util.Optional; + +public class RDFFileStorage implements Storage { + public record Config(String path) implements StorageConfig {} + + private static final Logger LOGGER = LoggerFactory.getLogger(RDFFileStorage.class.getName()); + + protected static Supplier defaultFileNameSupplier = () -> { + var now = Calendar.getInstance(); + return String.format("%d-%02d-%02d_%02d-%02d.%03d", + now.get(Calendar.YEAR), + now.get(Calendar.MONTH) + 1, + now.get(Calendar.DAY_OF_MONTH), + now.get(Calendar.HOUR_OF_DAY), + now.get(Calendar.MINUTE), + now.get(Calendar.MILLISECOND)); + }; + + final private Lang lang; + private Path path; + + public RDFFileStorage(Config config) { + this(config.path()); + } + + /** + * Uses a generated file called results_{DD}-{MM}-{YYYY}_{HH}-{mm}.ttl + */ + public RDFFileStorage() { + this(""); + } + + /** + * Uses the provided filename. If the filename is null or empty, a generated file called + * results_{DD}-{MM}-{YYYY}_{HH}-{mm}.ttl is used. The file extension determines the file format. + * + * @param fileName the filename to use + */ + public RDFFileStorage(String fileName) { + if (fileName == null || Optional.of(fileName).orElse("").isBlank()) { + path = Paths.get("").resolve(defaultFileNameSupplier.get() + ".ttl"); + } + else { + path = Paths.get(fileName); + if (Files.exists(path) && Files.isDirectory(path)) { + path = path.resolve(defaultFileNameSupplier.get() + ".ttl"); + } else if (Files.exists(path)) { + path = Paths.get(FilenameUtils.removeExtension(fileName) + "_" + defaultFileNameSupplier.get() + ".ttl"); // we're just going to assume that that's enough to make it unique + } + } + final var parentDir = path.toAbsolutePath().getParent(); + try { + Files.createDirectories(parentDir); + } catch (IOException e) { + LOGGER.error("Could not create parent directories for RDFFileStorage. ", e); + } + + this.lang = RDFLanguages.filenameToLang(path.toString(), Lang.TTL); + } + + @Override + public void storeResult(Model data){ + try (OutputStream os = new FileOutputStream(path.toString(), true)) { + RDFDataMgr.write(os, data, this.lang); + } catch (IOException e) { + LOGGER.error("Could not write to RDFFileStorage using lang: " + lang, e); + } + } + + @Override + public String toString() { + return this.getClass().getSimpleName(); + } + + public String getFileName() { + return this.path.toString(); + } +} diff --git a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/storage/impl/TriplestoreStorage.java b/src/main/java/org/aksw/iguana/cc/storage/impl/TriplestoreStorage.java similarity index 56% rename from src/main/java/org/aksw/iguana/cc/tasks/stresstest/storage/impl/TriplestoreStorage.java rename to src/main/java/org/aksw/iguana/cc/storage/impl/TriplestoreStorage.java index 3623fc39a..994c24af2 100644 --- a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/storage/impl/TriplestoreStorage.java +++ b/src/main/java/org/aksw/iguana/cc/storage/impl/TriplestoreStorage.java @@ -1,10 +1,8 @@ -/** - * - */ -package org.aksw.iguana.cc.tasks.stresstest.storage.impl; +package org.aksw.iguana.cc.storage.impl; -import org.aksw.iguana.cc.tasks.stresstest.storage.TripleBasedStorage; -import org.aksw.iguana.commons.annotation.Shorthand; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.aksw.iguana.cc.config.elements.StorageConfig; +import org.aksw.iguana.cc.storage.Storage; import org.apache.http.auth.AuthScope; import org.apache.http.auth.Credentials; import org.apache.http.auth.UsernamePasswordCredentials; @@ -25,70 +23,66 @@ /** * This Storage will save all the metric results into a specified triple store - * + * * @author f.conrads * */ -@Shorthand("TriplestoreStorage") -public class TriplestoreStorage extends TripleBasedStorage { - - private UpdateRequest blockRequest = UpdateFactory.create(); +public class TriplestoreStorage implements Storage { - - private final String updateEndpoint; + public record Config( + @JsonProperty(required = true) String endpoint, + String user, + String password, + String baseUri + ) implements StorageConfig {} + + private UpdateRequest blockRequest = UpdateFactory.create(); private final String endpoint; - private String user; - private String pwd; + private final String user; + private final String password; + private final String baseUri; - - public TriplestoreStorage(String endpoint, String updateEndpoint, String user, String pwd, String baseUri){ - this.endpoint=endpoint; - this.updateEndpoint=updateEndpoint; - this.user=user; - this.pwd=pwd; - if(baseUri!=null && !baseUri.isEmpty()){ - this.baseUri=baseUri; - } + public TriplestoreStorage(Config config) { + endpoint = config.endpoint(); + user = config.user(); + password = config.password(); + baseUri = config.baseUri(); } - - public TriplestoreStorage(String endpoint, String updateEndpoint, String baseUri){ - this.endpoint=endpoint; - this.updateEndpoint=updateEndpoint; - if(baseUri!=null && !baseUri.isEmpty()){ - this.baseUri=baseUri; - } + + + public TriplestoreStorage(String endpoint, String user, String pwd, String baseUri) { + this.endpoint = endpoint; + this.user = user; + this.password = pwd; + this.baseUri = baseUri; } - - public TriplestoreStorage(String endpoint, String updateEndpoint){ - this.endpoint=endpoint; - this.updateEndpoint=updateEndpoint; + + public TriplestoreStorage(String endpoint) { + this.endpoint = endpoint; + this.user = null; + this.password = null; + this.baseUri = null; } @Override public void storeResult(Model data) { - super.storeResult(data); - if (metricResults.size() == 0) - return; - StringWriter results = new StringWriter(); - RDFDataMgr.write(results, metricResults, Lang.NT); + RDFDataMgr.write(results, data, Lang.NT); String update = "INSERT DATA {" + results.toString() + "}"; //Create Update Request from block blockRequest.add(update); //submit Block to Triple Store UpdateProcessor processor = UpdateExecutionFactory - .createRemote(blockRequest, updateEndpoint, createHttpClient()); + .createRemote(blockRequest, endpoint, createHttpClient()); processor.execute(); blockRequest = new UpdateRequest(); } - - - private HttpClient createHttpClient(){ + private HttpClient createHttpClient() { CredentialsProvider credsProvider = new BasicCredentialsProvider(); - if(user !=null && pwd !=null){ - Credentials credentials = new UsernamePasswordCredentials(user, pwd); + if(user != null && password != null){ + Credentials credentials = new UsernamePasswordCredentials(user, password); credsProvider.setCredentials(AuthScope.ANY, credentials); } HttpClient httpclient = HttpClients.custom() diff --git a/src/main/java/org/aksw/iguana/cc/suite/IguanaSuiteParser.java b/src/main/java/org/aksw/iguana/cc/suite/IguanaSuiteParser.java new file mode 100644 index 000000000..8a297772d --- /dev/null +++ b/src/main/java/org/aksw/iguana/cc/suite/IguanaSuiteParser.java @@ -0,0 +1,260 @@ +package org.aksw.iguana.cc.suite; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.networknt.schema.JsonSchema; +import com.networknt.schema.JsonSchemaFactory; +import com.networknt.schema.SpecVersion; +import com.networknt.schema.ValidationMessage; +import org.aksw.iguana.cc.config.elements.ConnectionConfig; +import org.aksw.iguana.cc.config.elements.DatasetConfig; +import org.apache.commons.io.FilenameUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.text.MessageFormat; +import java.time.Duration; +import java.time.Instant; +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * Creates an IguanaConfig from a given JSON or YAML file, and validates the config using a JSON schema file + */ +public class IguanaSuiteParser { + private static final Logger LOGGER = LoggerFactory.getLogger(IguanaSuiteParser.class); + + private static final String SCHEMA_FILE = "./schema/iguana-schema.json"; + + enum DataFormat { + YAML, JSON; + + public static DataFormat getFormat(Path file) { + final var extension = FilenameUtils.getExtension(file.toString()); + switch (extension) { + case "yml", "yaml" -> { + return YAML; + } + case "json" -> { + return JSON; + } + default -> throw new IllegalStateException("Unexpected suite file extension: " + extension); + } + } + } + + /** + * Parses an IGUANA configuration file and optionally validates it against a JSON schema file, before parsing. + * + * @param config the path to the configuration file. + * @param validate whether to validate the configuration file against the JSON schema file. + * @return a Suite object containing the parsed configuration. + * @throws IOException if there is an error during IO. + * @throws IllegalStateException if the configuration file is invalid. + */ + public static Suite parse(Path config, boolean validate) throws IOException { + final var format = DataFormat.getFormat(config); + JsonFactory factory = switch (format) { + case YAML -> new YAMLFactory(); + case JSON -> new JsonFactory(); + }; + + if (validate && !validateConfig(config)) { + throw new IllegalStateException("Invalid config file"); + } + + try (var stream = new FileInputStream(config.toFile())) { + return parse(stream, factory); + } + } + + /** + * Validates an IGUANA configuration file against a JSON schema file. + * + * @param config the path to the configuration file. + * @return true if the configuration file is valid, false otherwise. + * @throws IOException if there is an error during IO. + */ + public static boolean validateConfig(Path config) throws IOException { + final var format = DataFormat.getFormat(config); + JsonFactory factory = switch (format) { + case YAML -> new YAMLFactory(); + case JSON -> new JsonFactory(); + }; + final var mapper = new ObjectMapper(factory); + + JsonSchemaFactory schemaFactory = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V6); + InputStream is = new FileInputStream(SCHEMA_FILE); + JsonSchema schema = schemaFactory.getSchema(is); + JsonNode node = mapper.readTree(config.toFile()); + Set errors = schema.validate(node); + if (!errors.isEmpty()) { + LOGGER.error("Found {} errors in configuration file.", errors.size()); + } + for (ValidationMessage message : errors) { + LOGGER.error(message.getMessage()); + } + return errors.isEmpty(); + } + + /** + * Parses an IGUANA configuration file.

+ * + * This involves two steps: First, datasets and connections are parsed and stored. In a second step, the rest of the + * file is parsed. If the names of datasets and connections are used, they are replaced with the respective + * configurations that were parsed in the first step. + * + * @param inputStream the input stream containing the configuration file content. + * @param factory the JsonFactory instance used for parsing the configuration file. + * @return a Suite object containing the parsed configuration. + * @throws IOException if there is an error during IO. + */ + private static Suite parse(InputStream inputStream, JsonFactory factory) throws IOException { + ObjectMapper mapper = new ObjectMapper(factory); + + final var input = new String(inputStream.readAllBytes(), StandardCharsets.UTF_8); + final var datasets = preparseDataset(mapper, input); + + class DatasetDeserializer extends StdDeserializer { + public DatasetDeserializer() { + this(null); + } + + protected DatasetDeserializer(Class vc) { + super(vc); + } + + @Override + public DatasetConfig deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException { + JsonNode node = jp.getCodec().readTree(jp); + if (node.isTextual()) { + final var datasetName = node.asText(); + if (!datasets.containsKey(datasetName)) + throw new IllegalStateException(MessageFormat.format("Unknown dataset name: {0}", datasetName)); + return datasets.get(datasetName); + } else { + DatasetConfig datasetConfig = ctxt.readValue(jp, DatasetConfig.class); + if (datasets.containsKey(datasetConfig.name())) + assert datasets.get(datasetConfig.name()) == datasetConfig; + else datasets.put(datasetConfig.name(), datasetConfig); + return datasetConfig; + } + } + } + mapper = new ObjectMapper(factory).registerModule(new SimpleModule() + .addDeserializer(DatasetConfig.class, new DatasetDeserializer())); + + final var connections = preparseConnections(mapper, input); + + class ConnectionDeserializer extends StdDeserializer { + + public ConnectionDeserializer() { + this(null); + } + + protected ConnectionDeserializer(Class vc) { + super(vc); + } + + @Override + public ConnectionConfig deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException { + JsonNode node = jp.getCodec().readTree(jp); + if (node.isTextual()) { + final var connectionName = node.asText(); + if (!connections.containsKey(connectionName)) + throw new IllegalStateException(MessageFormat.format("Unknown connection name: {0}", connectionName)); + return connections.get(connectionName); + } else { + ConnectionConfig connectionConfig = ctxt.readValue(jp, ConnectionConfig.class); + if (connections.containsKey(connectionConfig.name())) + assert connections.get(connectionConfig.name()) == connectionConfig; + else connections.put(connectionConfig.name(), connectionConfig); + return connectionConfig; + } + } + } + + class HumanReadableDurationDeserializer extends StdDeserializer { + + public HumanReadableDurationDeserializer() { + this(null); + } + + protected HumanReadableDurationDeserializer(Class vc) { + super(vc); + } + + @Override + public Duration deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException { + var durationString = jp.getValueAsString() + .toLowerCase() + .replaceAll("\\s+", "") + .replace("years", "y") + .replace("year", "y") + .replace("months", "m") + .replace("month", "m") + .replace("weeks", "w") + .replace("week", "w") + .replace("days", "d") + .replace("day", "d") + .replace("mins", "m") + .replace("min", "m") + .replace("hrs", "h") + .replace("hr", "h") + .replace("secs", "s") + .replace("sec", "s") + .replaceFirst("(\\d+d)", "P$1T"); + if ((durationString.charAt(0) != 'P')) durationString = "PT" + durationString; + return Duration.parse(durationString); + } + } + + mapper = new ObjectMapper(factory).registerModule(new JavaTimeModule()) + .registerModule(new SimpleModule() + .addDeserializer(DatasetConfig.class, new DatasetDeserializer()) + .addDeserializer(ConnectionConfig.class, new ConnectionDeserializer()) + .addDeserializer(Duration.class, new HumanReadableDurationDeserializer())); + + final String suiteID = Instant.now().getEpochSecond() + "-" + Integer.toUnsignedString(input.hashCode()); // convert to unsigned, so that there is no double -- minus in the string + return new Suite(suiteID, mapper.readValue(input, Suite.Config.class)); + } + + /** + * Preparses the datasets field in a IGUANA configuration file and adds a custom Deserializer to mapper to enable retrieving already parsed datasets by name. + * + * @param mapper The ObjectMapper instance used for parsing the configuration file. + * @param input The input String containing the configuration file content. + * @return A Map of DatasetConfig objects, where the key is the dataset name and the value is the corresponding DatasetConfig object. + * @throws JsonProcessingException If there is an error during JSON processing. + */ + private static Map preparseDataset(ObjectMapper mapper, String input) throws JsonProcessingException { + @JsonIgnoreProperties(ignoreUnknown = true) + record PreparsingDatasets(@JsonProperty(required = true) List datasets) {} + final var preparsingDatasets = mapper.readValue(input, PreparsingDatasets.class); + + return preparsingDatasets.datasets().stream().collect(Collectors.toMap(DatasetConfig::name, Function.identity())); + } + + private static Map preparseConnections(ObjectMapper mapper, String input) throws JsonProcessingException { + @JsonIgnoreProperties(ignoreUnknown = true) + record PreparsingConnections(@JsonProperty(required = true) List connections) {} + final var preparsingConnections = mapper.readValue(input, PreparsingConnections.class); + + return preparsingConnections.connections().stream().collect(Collectors.toMap(ConnectionConfig::name, Function.identity())); + } + +} diff --git a/src/main/java/org/aksw/iguana/cc/suite/Suite.java b/src/main/java/org/aksw/iguana/cc/suite/Suite.java new file mode 100644 index 000000000..1cb38acb5 --- /dev/null +++ b/src/main/java/org/aksw/iguana/cc/suite/Suite.java @@ -0,0 +1,103 @@ +package org.aksw.iguana.cc.suite; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.aksw.iguana.cc.config.elements.ConnectionConfig; +import org.aksw.iguana.cc.config.elements.DatasetConfig; +import org.aksw.iguana.cc.config.elements.StorageConfig; +import org.aksw.iguana.cc.metrics.Metric; +import org.aksw.iguana.cc.metrics.impl.*; +import org.aksw.iguana.cc.storage.Storage; +import org.aksw.iguana.cc.storage.impl.CSVStorage; +import org.aksw.iguana.cc.storage.impl.RDFFileStorage; +import org.aksw.iguana.cc.storage.impl.TriplestoreStorage; +import org.aksw.iguana.cc.tasks.impl.Stresstest; +import org.aksw.iguana.cc.tasks.Task; +import org.aksw.iguana.cc.worker.ResponseBodyProcessor; +import org.aksw.iguana.cc.worker.ResponseBodyProcessorInstances; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +public class Suite { + + public record Config( + @JsonIgnore + List datasets, /* Will already be consumed and ignored herein */ + @JsonIgnore + List connections, /* Will already be consumed and ignored herein */ + @JsonProperty(required = true) + List tasks, + List storages, + List metrics, + @JsonProperty List responseBodyProcessors + ) {} + + + private final String suiteId; + private final Config config; + private final ResponseBodyProcessorInstances responseBodyProcessorInstances; + + private final static Logger LOGGER = LoggerFactory.getLogger(Suite.class); + + private final List tasks = new ArrayList<>(); + + Suite(String suiteId, Config config) { + this.suiteId = suiteId; + this.config = config; + long taskID = 0; + + responseBodyProcessorInstances = new ResponseBodyProcessorInstances(config.responseBodyProcessors); + List metrics = initialiseMetrics(this.config.metrics); + List storages = initialiseStorages(this.config.storages, metrics, this.suiteId); + + for (Task.Config task : config.tasks()) { + if (task instanceof Stresstest.Config) { + tasks.add(new Stresstest(this.suiteId, taskID++, (Stresstest.Config) task, responseBodyProcessorInstances, storages, metrics)); + } + } + } + + private static List initialiseMetrics(List metrics) { + if (metrics != null && !metrics.isEmpty()) { + return metrics; + } + + final List out = new ArrayList<>(); + out.add(new QPS()); + out.add(new AvgQPS()); + out.add(new NoQPH()); + out.add(new AggregatedExecutionStatistics()); + out.add(new EachExecutionStatistic()); + out.add(new NoQ()); + out.add(new QMPH()); + return out; + } + + private static List initialiseStorages(List configs, List metrics, String suiteID) { + List out = new ArrayList<>(); + for (var storageConfig : configs) { + if (storageConfig instanceof CSVStorage.Config) { + out.add(new CSVStorage((CSVStorage.Config) storageConfig, metrics, suiteID)); + } + else if (storageConfig instanceof TriplestoreStorage.Config) { + out.add(new TriplestoreStorage((TriplestoreStorage.Config) storageConfig)); + } + else if (storageConfig instanceof RDFFileStorage.Config) { + out.add(new RDFFileStorage((RDFFileStorage.Config) storageConfig)); + } + } + return out; + } + + public void run() { + for (int i = 0; i < tasks.size(); i++) { + tasks.get(i).run(); + LOGGER.info("Task/{} {} finished.", tasks.get(i).getTaskName(), i); + } + } +} + + diff --git a/src/main/java/org/aksw/iguana/cc/tasks/AbstractTask.java b/src/main/java/org/aksw/iguana/cc/tasks/AbstractTask.java deleted file mode 100644 index ae9de8d43..000000000 --- a/src/main/java/org/aksw/iguana/cc/tasks/AbstractTask.java +++ /dev/null @@ -1,63 +0,0 @@ -/** - * - */ -package org.aksw.iguana.cc.tasks; - -import org.aksw.iguana.cc.config.elements.ConnectionConfig; -import java.util.Properties; - -/** - * Abstract Task to help create a Task. - * Will do the background work - * - * @author f.conrads - * - */ -public abstract class AbstractTask implements Task { - - protected String taskID; - protected ConnectionConfig con; - - protected String expID; - protected String suiteID; - protected String datasetID; - protected String conID; - protected String taskName; - - /** - * Creates an AbstractTask with the TaskID - */ - public AbstractTask() { - - } - - - /* - * (non-Javadoc) - * - * @see org.aksw.iguana.tp.tasks.Task#init() - */ - @Override - public void init(String[] ids, String dataset, ConnectionConfig con, String taskName) { - this.suiteID=ids[0]; - this.expID=ids[1]; - this.taskID=ids[2]; - this.taskName=taskName; - this.datasetID=dataset; - this.conID=con.getName(); - this.con=con; - } - - @Override - public void start() {} - - @Override - public void sendResults(Properties data) {} - - @Override - public void close() {} - - @Override - public void addMetaData() {} - -} diff --git a/src/main/java/org/aksw/iguana/cc/tasks/Task.java b/src/main/java/org/aksw/iguana/cc/tasks/Task.java index 3bd6b0b7a..5c5901fcc 100644 --- a/src/main/java/org/aksw/iguana/cc/tasks/Task.java +++ b/src/main/java/org/aksw/iguana/cc/tasks/Task.java @@ -1,66 +1,18 @@ -/** - * - */ package org.aksw.iguana.cc.tasks; -import org.aksw.iguana.cc.config.elements.ConnectionConfig; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.aksw.iguana.cc.tasks.impl.Stresstest; -import java.io.IOException; -import java.util.Properties; - -/** - * A simple Task to execute - * - * @author f.conrads - * - */ public interface Task { - - /** - * Will execute the Task - */ - public void execute(); - - /** - * Will start the Task (sending the rabbitMQ start flag) - */ - public void start(); - - /** - * Will send the results to the result processing. - * @param data - * @throws IOException - */ - void sendResults(Properties data) throws IOException; - - - /** - * Will close the Task and post process everything (e.g. send the end flag to the rabbit mq queue) - */ - void close(); - - /** - * Will add the Meta data for the start which then can be saved into the triple based storages - */ - void addMetaData(); - - - /** - * Will initialize the task - * @param ids normally the suiteID, experimentID, taskID - * @param dataset the dataset name - * @param con the current connection to execute the task against - * @param taskName the taskName - */ - void init(String[] ids, String dataset, ConnectionConfig con, String taskName); - - /** - * Will initialize the task - * @param ids normally the suiteID, experimentID, taskID - * @param dataset the dataset name - * @param con the current connection to execute the task against - */ - default void init(String[] ids, String dataset, ConnectionConfig con){ - init(ids, dataset, con, null); - } + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type") + @JsonSubTypes({ + @JsonSubTypes.Type(value = Stresstest.Config.class, name = "stresstest"), + }) + interface Config {} + + void run(); + String getTaskName(); } diff --git a/src/main/java/org/aksw/iguana/cc/tasks/TaskFactory.java b/src/main/java/org/aksw/iguana/cc/tasks/TaskFactory.java deleted file mode 100644 index bf753ef1b..000000000 --- a/src/main/java/org/aksw/iguana/cc/tasks/TaskFactory.java +++ /dev/null @@ -1,18 +0,0 @@ -/** - * - */ -package org.aksw.iguana.cc.tasks; - -import org.aksw.iguana.commons.factory.TypedFactory; - - -/** - * Factory to create Tasks. see {@link TypedFactory} for more information. - * - * @author f.conrads - * - */ -public class TaskFactory extends TypedFactory{ - - -} diff --git a/src/main/java/org/aksw/iguana/cc/tasks/TaskManager.java b/src/main/java/org/aksw/iguana/cc/tasks/TaskManager.java deleted file mode 100644 index 22e8b8605..000000000 --- a/src/main/java/org/aksw/iguana/cc/tasks/TaskManager.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * - */ -package org.aksw.iguana.cc.tasks; - -import org.aksw.iguana.cc.config.elements.ConnectionConfig; - -import java.io.IOException; -import java.util.concurrent.TimeoutException; - -/** - * Will manage the Tasks - * - * @author f.conrads - * - */ -public class TaskManager { - - private Task task; - - /** - * Will simply set the Task to execute - * @param task - */ - public void setTask(Task task){ - this.task = task; - } - - /** - * Will initialize and start the Task - * - * @throws IOException - * @throws TimeoutException - */ - public void startTask(String[] ids, String dataset, ConnectionConfig con, String taskName) throws IOException, TimeoutException{ - this.task.init(ids, dataset, con, taskName); - this.task.addMetaData(); - this.task.start(); - this.task.execute(); - this.task.close(); - } - - - -} diff --git a/src/main/java/org/aksw/iguana/cc/tasks/impl/Stresstest.java b/src/main/java/org/aksw/iguana/cc/tasks/impl/Stresstest.java new file mode 100644 index 000000000..cfa03d243 --- /dev/null +++ b/src/main/java/org/aksw/iguana/cc/tasks/impl/Stresstest.java @@ -0,0 +1,111 @@ +package org.aksw.iguana.cc.tasks.impl; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.aksw.iguana.cc.metrics.Metric; +import org.aksw.iguana.cc.storage.Storage; +import org.aksw.iguana.cc.tasks.Task; +import org.aksw.iguana.cc.worker.HttpWorker; +import org.aksw.iguana.cc.worker.ResponseBodyProcessorInstances; +import org.aksw.iguana.cc.worker.impl.SPARQLProtocolWorker; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.concurrent.*; + + +/** + * Stresstest. + * Will stresstest a connection using several Workers (simulated Users) each in one thread. + */ +public class Stresstest implements Task { + + public record Config( + List warmupWorkers, + @JsonProperty(required = true) List workers + ) implements Task.Config {} + + public record Result( + List workerResults, + Calendar startTime, + Calendar endTime + ) {} + + + private static final Logger LOGGER = LoggerFactory.getLogger(Stresstest.class); + + private final List warmupWorkers = new ArrayList<>(); + private final List workers = new ArrayList<>(); + + private final StresstestResultProcessor srp; + + + public Stresstest(String suiteID, long stresstestID, Config config, ResponseBodyProcessorInstances responseBodyProcessorInstances, List storages, List metrics) { + + // initialize workers + long workerId = 0; + if (config.warmupWorkers() != null) { + for (HttpWorker.Config workerConfig : config.warmupWorkers()) { + for (int i = 0; i < workerConfig.number(); i++) { + var responseBodyProcessor = (workerConfig.parseResults()) ? responseBodyProcessorInstances.getProcessor(workerConfig.acceptHeader()) : null; + warmupWorkers.add(new SPARQLProtocolWorker(workerId++, responseBodyProcessor, (SPARQLProtocolWorker.Config) workerConfig)); + } + } + } + + for (HttpWorker.Config workerConfig : config.workers()) { + for (int i = 0; i < workerConfig.number(); i++) { + var responseBodyProcessor = (workerConfig.parseResults()) ? responseBodyProcessorInstances.getProcessor(workerConfig.acceptHeader()) : null; + workers.add(new SPARQLProtocolWorker(workerId++, responseBodyProcessor, (SPARQLProtocolWorker.Config) workerConfig)); + } + } + + // retrieve all query ids + Set queryIDs = new HashSet<>(); + for (HttpWorker.Config wConfig : config.workers) { + if (wConfig instanceof SPARQLProtocolWorker.Config) { + queryIDs.addAll(List.of((wConfig).queries().getAllQueryIds())); + } + } + + srp = new StresstestResultProcessor( + suiteID, + stresstestID, + this.workers, + new ArrayList<>(queryIDs), + metrics, + storages, + responseBodyProcessorInstances.getResults() + ); + } + + public void run() { + var warmupResults = executeWorkers(warmupWorkers); // warmup results will be dismissed + var results = executeWorkers(workers); + + srp.process(results.workerResults); + srp.calculateAndSaveMetrics(results.startTime, results.endTime); + } + + private Result executeWorkers(List workers) { + List results = new ArrayList<>(workers.size()); + Calendar startTime = Calendar.getInstance(); // TODO: Calendar is outdated + var futures = workers.stream().map(HttpWorker::start).toList(); + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); + Calendar endTime = Calendar.getInstance(); // TODO: add start and end time for each worker + for (CompletableFuture future : futures) { + try { + results.add(future.get()); + } catch (ExecutionException e) { + LOGGER.error("Unexpected error during execution of worker.", e); + } catch (InterruptedException ignored) {} + + } + return new Result(results, startTime, endTime); + } + + @Override + public String getTaskName() { + return "stresstest"; + } +} diff --git a/src/main/java/org/aksw/iguana/cc/tasks/impl/StresstestResultProcessor.java b/src/main/java/org/aksw/iguana/cc/tasks/impl/StresstestResultProcessor.java new file mode 100644 index 000000000..26f21a2af --- /dev/null +++ b/src/main/java/org/aksw/iguana/cc/tasks/impl/StresstestResultProcessor.java @@ -0,0 +1,280 @@ +package org.aksw.iguana.cc.tasks.impl; + +import org.aksw.iguana.cc.lang.LanguageProcessor; +import org.aksw.iguana.cc.metrics.*; +import org.aksw.iguana.cc.storage.Storage; +import org.aksw.iguana.cc.worker.HttpWorker; +import org.aksw.iguana.commons.rdf.IGUANA_BASE; +import org.aksw.iguana.commons.rdf.IONT; +import org.aksw.iguana.commons.rdf.IPROP; +import org.aksw.iguana.commons.rdf.IRES; +import org.aksw.iguana.commons.time.TimeUtils; +import org.apache.jena.rdf.model.*; +import org.apache.jena.vocabulary.RDF; +import org.apache.jena.vocabulary.RDFS; + +import java.time.ZonedDateTime; +import java.util.*; +import java.util.function.Supplier; + +public class StresstestResultProcessor { + + private record StartEndTimePair ( + ZonedDateTime startTime, + ZonedDateTime endTime + ) {} + + private final List metrics; + private final List workers; + private final List queryIDs; + private final List storages; + private final Supplier>> lpResults; + + /** + * This array contains each query execution of a worker grouped to its queries. + * The outer array is indexed with the workerID and the inner array with the numeric queryID that the query has + * inside that worker. + */ + private final List[][] workerQueryExecutions; + + /** This map contains each query execution, grouped by each queryID of the task. */ + private final Map> taskQueryExecutions; + + + /** Stores the start and end time for each workerID. */ + private final StartEndTimePair[] workerStartEndTime; + + private final IRES.Factory iresFactory; + + + public StresstestResultProcessor(String suiteID, + long taskID, + List worker, + List queryIDs, + List metrics, + List storages, + Supplier>> lpResults) { + this.workers = worker; + this.queryIDs = queryIDs; + this.storages = storages; + this.metrics = metrics; + this.lpResults = lpResults; + + this.workerQueryExecutions = new ArrayList[workers.size()][]; + for (int i = 0; i < workers.size(); i++) { + this.workerQueryExecutions[i] = new ArrayList[workers.get(i).config().queries().getQueryCount()]; + for (int j = 0; j < workers.get(i).config().queries().getQueryCount(); j++) { + this.workerQueryExecutions[i][j] = new ArrayList<>(); + } + } + + this.taskQueryExecutions = new HashMap<>(); + for (String queryID : queryIDs) { + this.taskQueryExecutions.put(queryID, new ArrayList<>()); + } + + this.iresFactory = new IRES.Factory(suiteID, taskID); + this.workerStartEndTime = new StartEndTimePair[worker.size()]; + } + + /** + * This method stores the given query executions statistics from a worker to their appropriate data location. + * + * @param data the query execution statistics that should be stored + */ + public void process(Collection data) { + for (HttpWorker.Result result : data) { + for (var stat : result.executionStats()) { + workerQueryExecutions[(int) result.workerID()][stat.queryID()].add(stat); + String queryID = workers.get((int) result.workerID()).config().queries().getQueryId(stat.queryID()); + taskQueryExecutions.get(queryID).add(stat); + } + workerStartEndTime[Math.toIntExact(result.workerID())] = new StartEndTimePair(result.startTime(), result.endTime()); // Naively assumes that there won't be more than Integer.MAX workers + } + } + + /** + * This method calculates the metrics and creates the RDF model of the result, which will be sent to the storages. + * It uses the given data that was passed with the 'processQueryExecutions' method. + * + * @param start the start date of the task + * @param end the end date of the task + */ + public void calculateAndSaveMetrics(Calendar start, Calendar end) { + Model m = ModelFactory.createDefaultModel().setNsPrefixes(IGUANA_BASE.PREFIX_MAP); + Resource suiteRes = iresFactory.getSuiteResource(); + Resource taskRes = iresFactory.getTaskResource(); + + m.add(suiteRes, RDF.type, IONT.suite); + m.add(suiteRes, IPROP.task, taskRes); + m.add(taskRes, RDF.type, IONT.task); + m.add(taskRes, RDF.type, IONT.stresstest); + m.add(taskRes, IPROP.noOfWorkers, ResourceFactory.createTypedLiteral(workers.size())); + + for (HttpWorker worker : workers) { + HttpWorker.Config config = worker.config(); + + Resource workerRes = iresFactory.getWorkerResource(worker); + Resource connectionRes = IRES.getResource(config.connection().name()); + if (config.connection().dataset() != null) { + Resource datasetRes = IRES.getResource(config.connection().dataset().name()); + m.add(connectionRes, IPROP.dataset, datasetRes); + m.add(datasetRes, RDFS.label, ResourceFactory.createTypedLiteral(config.connection().dataset().name())); + m.add(datasetRes, RDF.type, IONT.dataset); + } + + m.add(taskRes, IPROP.workerResult, workerRes); + m.add(workerRes, RDF.type, IONT.worker); + m.add(workerRes, IPROP.workerID, ResourceFactory.createTypedLiteral(worker.getWorkerID())); + m.add(workerRes, IPROP.workerType, ResourceFactory.createTypedLiteral(worker.getClass().getSimpleName())); + m.add(workerRes, IPROP.noOfQueries, ResourceFactory.createTypedLiteral(config.queries().getQueryCount())); + m.add(workerRes, IPROP.timeOut, TimeUtils.createTypedDurationLiteral(config.timeout())); + if (config.completionTarget() instanceof HttpWorker.QueryMixes) + m.add(taskRes, IPROP.noOfQueryMixes, ResourceFactory.createTypedLiteral(((HttpWorker.QueryMixes) config.completionTarget()).number())); + if (config.completionTarget() instanceof HttpWorker.TimeLimit) + m.add(taskRes, IPROP.timeLimit, TimeUtils.createTypedDurationLiteral(((HttpWorker.TimeLimit) config.completionTarget()).duration())); + m.add(workerRes, IPROP.connection, connectionRes); + + m.add(connectionRes, RDF.type, IONT.connection); + m.add(connectionRes, RDFS.label, ResourceFactory.createTypedLiteral(config.connection().name())); + if (config.connection().version() != null) { + m.add(connectionRes, IPROP.version, ResourceFactory.createTypedLiteral(config.connection().version())); + } + } + + // Connect task and workers to the Query nodes, that store the triple stats. + for (var worker : workers) { + var config = worker.config(); + var workerQueryIDs = config.queries().getAllQueryIds(); + for (int i = 0; i < config.queries().getQueryCount(); i++) { + Resource workerQueryRes = iresFactory.getWorkerQueryResource(worker, i); + Resource queryRes = IRES.getResource(workerQueryIDs[i]); + m.add(workerQueryRes, IPROP.queryID, queryRes); + } + + var taskQueryIDs = this.queryIDs.toArray(String[]::new); // make elements accessible by index + for (String taskQueryID : taskQueryIDs) { + Resource taskQueryRes = iresFactory.getTaskQueryResource(taskQueryID); + Resource queryRes = IRES.getResource(taskQueryID); + m.add(taskQueryRes, IPROP.queryID, queryRes); + } + } + + for (Metric metric : metrics) { + m.add(this.createMetricModel(metric)); + } + + // Task to queries + for (String queryID : queryIDs) { + m.add(taskRes, IPROP.query, iresFactory.getTaskQueryResource(queryID)); + } + + for (var worker : workers) { + Resource workerRes = iresFactory.getWorkerResource(worker); + + // Worker to queries + for (int i = 0; i < worker.config().queries().getAllQueryIds().length; i++) { + m.add(workerRes, IPROP.query, iresFactory.getWorkerQueryResource(worker, i)); + } + + // start and end times for the workers + final var timePair = workerStartEndTime[Math.toIntExact(worker.getWorkerID())]; + m.add(workerRes, IPROP.startDate, TimeUtils.createTypedZonedDateTimeLiteral(timePair.startTime)); + m.add(workerRes, IPROP.endDate, TimeUtils.createTypedZonedDateTimeLiteral(timePair.endTime)); + } + + m.add(taskRes, IPROP.startDate, ResourceFactory.createTypedLiteral(start)); + m.add(taskRes, IPROP.endDate, ResourceFactory.createTypedLiteral(end)); + + for (var storage : storages) { + storage.storeResult(m); + } + + // Store results of language processors (this shouldn't throw an error if the map is empty) + for (var languageProcessor: lpResults.get().keySet()) { + for (var data : lpResults.get().get(languageProcessor)) { + for (var storage : storages) { + storage.storeData(data); + } + } + } + } + + /** + * For a given metric this method calculates the metric with the stored data and creates the appropriate + * RDF related to that metric. + * + * @param metric the metric that should be calculated + * @return the result model of the metric + */ + private Model createMetricModel(Metric metric) { + Model m = ModelFactory.createDefaultModel(); + Property metricProp = IPROP.createMetricProperty(metric); + Resource metricRes = IRES.getMetricResource(metric); + Resource taskRes = iresFactory.getTaskResource(); + + if (metric instanceof ModelWritingMetric) { + m.add(((ModelWritingMetric) metric).createMetricModel(this.workers, + this.workerQueryExecutions, + this.iresFactory)); + m.add(((ModelWritingMetric) metric).createMetricModel(this.workers, + this.taskQueryExecutions, + this.iresFactory)); + } + + if (metric instanceof TaskMetric) { + Number metricValue = ((TaskMetric) metric).calculateTaskMetric(this.workers, workerQueryExecutions); + if (metricValue != null) { + Literal lit = ResourceFactory.createTypedLiteral(metricValue); + m.add(taskRes, metricProp, lit); + } + m.add(taskRes, IPROP.metric, metricRes); + } + + if (metric instanceof WorkerMetric) { + for (var worker : workers) { + Resource workerRes = iresFactory.getWorkerResource(worker); + Number metricValue = ((WorkerMetric) metric).calculateWorkerMetric( + worker.config(), + workerQueryExecutions[(int) worker.getWorkerID()]); + if (metricValue != null) { + Literal lit = ResourceFactory.createTypedLiteral(metricValue); + m.add(workerRes, metricProp, lit); + } + m.add(workerRes, IPROP.metric, metricRes); + } + } + + if (metric instanceof QueryMetric) { + // queries grouped by worker + for (var worker : workers) { + for (int i = 0; i < worker.config().queries().getQueryCount(); i++) { + Number metricValue = ((QueryMetric) metric).calculateQueryMetric(workerQueryExecutions[(int) worker.getWorkerID()][i]); + if (metricValue != null) { + Literal lit = ResourceFactory.createTypedLiteral(metricValue); + Resource queryRes = iresFactory.getWorkerQueryResource(worker, i); + m.add(queryRes, metricProp, lit); + } + } + } + + // queries grouped by task + for (String queryID : queryIDs) { + Number metricValue = ((QueryMetric) metric).calculateQueryMetric(taskQueryExecutions.get(queryID)); + if (metricValue != null) { + Literal lit = ResourceFactory.createTypedLiteral(metricValue); + Resource queryRes = iresFactory.getTaskQueryResource(queryID); + m.add(queryRes, metricProp, lit); + } + } + } + + m.add(metricRes, RDFS.label, metric.getName()); + m.add(metricRes, RDFS.label, metric.getAbbreviation()); + m.add(metricRes, RDFS.comment, metric.getDescription()); + m.add(metricRes, RDF.type, IONT.getMetricClass(metric)); + m.add(metricRes, RDF.type, IONT.metric); + + return m; + } +} diff --git a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/Stresstest.java b/src/main/java/org/aksw/iguana/cc/tasks/stresstest/Stresstest.java deleted file mode 100644 index ae84cae64..000000000 --- a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/Stresstest.java +++ /dev/null @@ -1,364 +0,0 @@ -package org.aksw.iguana.cc.tasks.stresstest; - -import org.aksw.iguana.cc.config.elements.ConnectionConfig; -import org.aksw.iguana.cc.model.QueryExecutionStats; -import org.aksw.iguana.cc.worker.WorkerMetadata; -import org.aksw.iguana.cc.tasks.AbstractTask; -import org.aksw.iguana.cc.worker.Worker; -import org.aksw.iguana.cc.worker.WorkerFactory; -import org.aksw.iguana.commons.annotation.Shorthand; -import org.apache.jena.rdf.model.Model; -import org.apache.jena.rdf.model.ModelFactory; -import org.apache.jena.riot.RDFDataMgr; -import org.apache.jena.riot.RDFFormat; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.StringWriter; -import java.time.Instant; -import java.util.*; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -import static org.aksw.iguana.commons.time.TimeUtils.durationInMilliseconds; - - -/** - * Stresstest. - * Will stresstest a connection using several Workers (simulated Users) each in one thread. - */ -@Shorthand("Stresstest") -public class Stresstest extends AbstractTask { - private static final Logger LOGGER = LoggerFactory.getLogger(Stresstest.class); - - private final Map warmupConfig; - private final List warmupWorkers = new ArrayList<>(); - private final List> workerConfig; - protected List workers = new LinkedList<>(); - private Double warmupTimeMS; - private Double timeLimit; - private Long noOfQueryMixes; - private Instant startTime; - - private StresstestResultProcessor rp; - - private Calendar startDate; - private Calendar endDate; - - public Stresstest(Integer timeLimit, List> workers) { - this(timeLimit, workers, null); - } - - public Stresstest(Integer timeLimit, List> workers, Map warmup) { - this.timeLimit = timeLimit.doubleValue(); - this.workerConfig = workers; - this.warmupConfig = warmup; - } - - public Stresstest(List> workers, Integer noOfQueryMixes) { - this(workers, null, noOfQueryMixes); - } - - public Stresstest(List> workers, Map warmup, Integer noOfQueryMixes) { - this.noOfQueryMixes = noOfQueryMixes.longValue(); - this.workerConfig = workers; - this.warmupConfig = warmup; - } - - private void initWorkers() { - if (this.warmupConfig != null) { - createWarmupWorkers(); - } - createWorkers(); - } - - private void createWarmupWorkers() { - this.warmupTimeMS = ((Integer) this.warmupConfig.get("timeLimit")).doubleValue(); - - List> warmupWorkerConfig = (List>) this.warmupConfig.get("workers"); - createWorkers(warmupWorkerConfig, this.warmupWorkers, this.warmupTimeMS); - } - - private void createWorkers() { - createWorkers(this.workerConfig, this.workers, this.timeLimit); - } - - private void createWorkers(List> workers, List workersToAddTo, Double timeLimit) { - int workerID = 0; - for (Map workerConfig : workers) { - workerID += createWorker(workerConfig, workersToAddTo, timeLimit, workerID); - } - } - - private int createWorker(Map workerConfig, List workersToAddTo, Double timeLimit, Integer baseID) { - //let TypedFactory create from className and configuration - String className = workerConfig.remove("className").toString(); - //if shorthand classname is used, exchange to full classname - workerConfig.put("connection", this.con); - workerConfig.put("taskID", this.taskID); - - if (timeLimit != null) { - workerConfig.put("timeLimit", timeLimit.intValue()); - } - Integer threads = (Integer) workerConfig.remove("threads"); - for (int i = 0; i < threads; i++) { - workerConfig.put("workerID", baseID + i); - Worker worker = new WorkerFactory().create(className, workerConfig); - if (this.noOfQueryMixes != null) { - worker.endAtNoOfQueryMixes(this.noOfQueryMixes); - } - workersToAddTo.add(worker); - } - return threads; - } - - public void generateTripleStats() { - StringWriter sw = new StringWriter(); - Model tripleStats = ModelFactory.createDefaultModel(); - // TODO: workers might have the same queries, the following code thus adds unnecessary redundancy - for (Worker worker : this.workers) { - tripleStats.add(worker.getQueryHandler().getTripleStats(this.taskID)); - } - RDFDataMgr.write(sw, tripleStats, RDFFormat.NTRIPLES); - } - - /** - * Add extra Meta Data - */ - @Override - public void addMetaData() {} - - - @Override - public void init(String[] ids, String dataset, ConnectionConfig connection, String taskName) { - super.init(ids, dataset, connection, taskName); - - initWorkers(); - addMetaData(); - generateTripleStats(); - this.rp = new StresstestResultProcessor(this.getMetadata()); - } - - /* - * (non-Javadoc) - * - * @see org.aksw.iguana.cc.tasks.Task#start() - */ - @Override - public void execute() { - this.startDate = GregorianCalendar.getInstance(); - warmup(); - LOGGER.info("Task with ID {{}} will be executed now", this.taskID); - // Execute each Worker in ThreadPool - ExecutorService executor = Executors.newFixedThreadPool(this.workers.size()); - this.startTime = Instant.now(); - for (Worker worker : this.workers) { - executor.execute(worker); - } - LOGGER.info("[TaskID: {{}}]All {{}} workers have been started", this.taskID, this.workers.size()); - // wait timeLimit or noOfQueries - executor.shutdown(); - - // if a time limit is set, let the task thread sleep that amount of time, otherwise sleep for 100 ms - // to periodically check if the workers are finished - long sleep = (timeLimit != null) ? timeLimit.longValue() : 100; - while (!isFinished()) { - try { - Thread.sleep(sleep); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - - for (Worker worker : this.workers) { - sendWorkerResult(worker); - } - - LOGGER.debug("Sending stop signal to workers"); - for (Worker worker : this.workers) { - worker.stopSending(); - } - - // Wait 5seconds so the workers can stop themselves, otherwise they will be - // stopped - try { - LOGGER.debug("Will shutdown now..."); - - LOGGER.info("[TaskID: {{}}] Will shutdown and await termination in 5s.", this.taskID); - boolean finished = executor.awaitTermination(5, TimeUnit.SECONDS); - LOGGER.info("[TaskID: {{}}] Task completed. Thread finished status {}", this.taskID, finished); - } catch (InterruptedException e) { - LOGGER.error("[TaskID: {{}}] Could not shutdown Threads/Workers due to ...", this.taskID); - LOGGER.error("... Exception: ", e); - try { - executor.shutdownNow(); - } catch (Exception e1) { - LOGGER.error("Problems shutting down", e1); - } - } - this.endDate = GregorianCalendar.getInstance(); - } - - private void sendWorkerResult(Worker worker) { - Collection props = worker.popQueryResults(); - if (props == null) { - return; - } - - LOGGER.debug("[TaskID: {{}}] Send results", this.taskID); - this.rp.processQueryExecutions(worker.getMetadata(), props); - LOGGER.debug("[TaskID: {{}}] results could be send", this.taskID); - } - - - @Override - public void close() { - rp.calculateAndSaveMetrics(startDate, endDate); - } - - protected long warmup() { - if (this.warmupTimeMS == null || this.warmupTimeMS == 0L) { - return 0; - } - if (this.warmupWorkers.size() == 0) { - return 0; - } - LOGGER.info("[TaskID: {{}}] will start {{}}ms warmup now using {} no of workers in total.", this.taskID, this.warmupTimeMS, this.warmupWorkers.size()); - return executeWarmup(this.warmupWorkers); - } - - - private long executeWarmup(List warmupWorkers) { - ExecutorService exec = Executors.newFixedThreadPool(2); - for (Worker worker : warmupWorkers) { - exec.submit(worker); - } - //wait as long as needed - Instant start = Instant.now(); - exec.shutdown(); - while (durationInMilliseconds(start, Instant.now()) <= this.warmupTimeMS) { - //clean up RAM - for (Worker worker : warmupWorkers) { - worker.popQueryResults(); - } - try { - TimeUnit.MILLISECONDS.sleep(50); - } catch (Exception e) { - LOGGER.error("Could not warmup "); - } - } - for (Worker worker : warmupWorkers) { - worker.stopSending(); - } - try { - exec.awaitTermination(5, TimeUnit.SECONDS); - - } catch (InterruptedException e) { - LOGGER.warn("[TaskID: {{}}] Warmup. Could not await Termination of Workers.", this.taskID); - } - try { - exec.shutdownNow(); - } catch (Exception e1) { - LOGGER.error("Shutdown problems ", e1); - } - //clear up - long queriesExec = 0; - for (Worker w : warmupWorkers) { - queriesExec += w.getExecutedQueries(); - } - warmupWorkers.clear(); - LOGGER.info("[TaskID: {{}}] Warmup finished.", this.taskID); - return queriesExec; - } - - /** - * Checks if restriction (e.g. timelimit or noOfQueryMixes for each Worker) - * occurs - * - * @return true if restriction occurs, false otherwise - */ - protected boolean isFinished() { - if (this.timeLimit != null) { - - Instant current = Instant.now(); - double passed_time = this.timeLimit - durationInMilliseconds(this.startTime, current); - return passed_time <= 0D; - } else if (this.noOfQueryMixes != null) { - - // use noOfQueries of SPARQLWorkers (as soon as a worker hit the noOfQueries, it - // will stop sending results - // UpdateWorker are allowed to execute all their updates - boolean endFlag = true; - for (Worker worker : this.workers) { - LOGGER.debug("No of query Mixes: {} , queriesInMix {}", this.noOfQueryMixes, worker.getExecutedQueries()); - //Check for each worker, if the - if (worker.hasExecutedNoOfQueryMixes(this.noOfQueryMixes)) { - if (!worker.isTerminated()) { - //if the worker was not already terminated, send last results, as tehy will not be sended afterwards - sendWorkerResult(worker); - } - worker.stopSending(); - } else { - endFlag = false; - } - - } - return endFlag; - } - LOGGER.error("Neither time limit nor NoOfQueryMixes is set. executing task now"); - return true; - } - - public long getExecutedQueries() { - long ret = 0; - for (Worker worker : this.workers) { - ret += worker.getExecutedQueries(); - } - return ret; - } - - public StresstestMetadata getMetadata() { - String classname; - if (this.getClass().isAnnotationPresent(Shorthand.class)) { - classname = this.getClass().getAnnotation(Shorthand.class).value(); - } else { - classname = this.getClass().getCanonicalName(); - } - - Set queryIDs = new HashSet<>(); - WorkerMetadata[] workerMetadata = new WorkerMetadata[this.workers.size()]; - for (int i = 0; i < this.workers.size(); i++) { - workerMetadata[i] = this.workers.get(i).getMetadata(); - queryIDs.addAll(Arrays.asList(workerMetadata[i].queryIDs())); - } - - // TODO: workers might have the same queries, the following code thus adds unnecessary redundancy - // TODO: is sw used for anything? - StringWriter sw = new StringWriter(); - Model tripleStats = ModelFactory.createDefaultModel(); - for (Worker worker : this.workers) { - tripleStats.add(worker.getQueryHandler().getTripleStats(this.taskID)); - } - RDFDataMgr.write(sw, tripleStats, RDFFormat.NTRIPLES); - - // TODO: check for correct values - - return new StresstestMetadata( - suiteID, - expID, - taskID, - datasetID, - conID, - Optional.ofNullable(con.getVersion("")), - taskName, - classname, - Optional.ofNullable(this.timeLimit), - Optional.ofNullable(this.noOfQueryMixes), - workerMetadata, - queryIDs, - Optional.ofNullable(sw.toString()), - Optional.of(tripleStats) - ); - } -} diff --git a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/StresstestMetadata.java b/src/main/java/org/aksw/iguana/cc/tasks/stresstest/StresstestMetadata.java deleted file mode 100644 index 66233de82..000000000 --- a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/StresstestMetadata.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.aksw.iguana.cc.tasks.stresstest; - -import org.aksw.iguana.cc.worker.WorkerMetadata; -import org.apache.jena.rdf.model.Model; - -import java.util.Optional; -import java.util.Set; - -public record StresstestMetadata( - String suiteID, - String expID, - String taskID, - String datasetID, - String conID, - Optional conVersion, - String taskname, - String classname, - Optional timelimit, - Optional noOfQueryMixes, - WorkerMetadata[] workers, - Set queryIDs, - Optional simpleTriple, - Optional tripleStats -) {} diff --git a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/StresstestResultProcessor.java b/src/main/java/org/aksw/iguana/cc/tasks/stresstest/StresstestResultProcessor.java deleted file mode 100644 index c0f2dc790..000000000 --- a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/StresstestResultProcessor.java +++ /dev/null @@ -1,230 +0,0 @@ -package org.aksw.iguana.cc.tasks.stresstest; - -import org.aksw.iguana.cc.model.QueryExecutionStats; -import org.aksw.iguana.cc.worker.WorkerMetadata; -import org.aksw.iguana.cc.tasks.stresstest.metrics.*; -import org.aksw.iguana.cc.tasks.stresstest.storage.StorageManager; -import org.aksw.iguana.commons.rdf.IGUANA_BASE; -import org.aksw.iguana.commons.rdf.IONT; -import org.aksw.iguana.commons.rdf.IPROP; -import org.aksw.iguana.commons.rdf.IRES; -import org.apache.jena.rdf.model.*; -import org.apache.jena.vocabulary.RDF; -import org.apache.jena.vocabulary.RDFS; - -import java.util.*; - -public class StresstestResultProcessor { - - private final StresstestMetadata metadata; - private final List metrics; - - /** - * This array contains each query execution, grouped by each worker and each query. - */ - private List[][] workerQueryExecutions; - - /** - * This map contains each query execution, grouped by each query of the task. - */ - private Map> taskQueryExecutions; - - private final Resource taskRes; - - public StresstestResultProcessor(StresstestMetadata metadata) { - this.metadata = metadata; - this.taskRes = IRES.getResource(metadata.taskID()); - this.metrics = MetricManager.getMetrics(); - - WorkerMetadata[] workers = metadata.workers(); - this.workerQueryExecutions = new List[workers.length][]; - for (int i = 0; i < workers.length; i++) { - this.workerQueryExecutions[i] = new List[workers[i].numberOfQueries()]; - for (int j = 0; j < workers[i].numberOfQueries(); j++) { - this.workerQueryExecutions[i][j] = new LinkedList<>(); - } - } - - taskQueryExecutions = new HashMap<>(); - for (String queryID : metadata.queryIDs()) { - taskQueryExecutions.put(queryID, new ArrayList<>()); - } - } - - /** - * This method stores the given query executions statistics from a worker to their appropriate data location. - * - * @param worker the worker that has executed the queries - * @param data a collection of the query execution statistics - */ - public void processQueryExecutions(WorkerMetadata worker, Collection data) { - for(QueryExecutionStats stat : data) { - // The queryIDs returned by the queryHandler are Strings, in the form of ':'. - int queryID = Integer.parseInt(stat.queryID().substring(stat.queryID().indexOf(":") + 1)); - workerQueryExecutions[worker.workerID()][queryID].add(stat); - - taskQueryExecutions.get(stat.queryID()).add(stat); - } - } - - /** - * This method calculates the metrics and creates the RDF model of the result, which will be sent to the storages. - * It uses the given data that was passed with the 'processQueryExecutions' method. - * - * @param start the start date of the task - * @param end the end date of the task - */ - public void calculateAndSaveMetrics(Calendar start, Calendar end) { - Model m = ModelFactory.createDefaultModel(); - Resource suiteRes = IRES.getResource(metadata.suiteID()); - Resource experimentRes = IRES.getResource(metadata.expID()); - Resource datasetRes = IRES.getResource(metadata.datasetID()); - Resource connectionRes = IRES.getResource(metadata.conID()); - - m.add(suiteRes, IPROP.experiment, experimentRes); - m.add(suiteRes, RDF.type, IONT.suite); - m.add(experimentRes, IPROP.dataset, datasetRes); - m.add(experimentRes, RDF.type, IONT.experiment); - m.add(experimentRes, IPROP.task, taskRes); - m.add(datasetRes, RDFS.label, ResourceFactory.createTypedLiteral(metadata.datasetID())); - m.add(datasetRes, RDF.type, IONT.dataset); - m.add(taskRes, IPROP.connection, connectionRes); - if (metadata.noOfQueryMixes().isPresent()) - m.add(taskRes, IPROP.noOfQueryMixes, ResourceFactory.createTypedLiteral(metadata.noOfQueryMixes().get())); - m.add(taskRes, IPROP.noOfWorkers, ResourceFactory.createTypedLiteral(metadata.workers().length)); - if (metadata.timelimit().isPresent()) - m.add(taskRes, IPROP.timeLimit, ResourceFactory.createTypedLiteral(metadata.timelimit().get())); - m.add(taskRes, RDF.type, IONT.task); - - m.add(taskRes, RDF.type, IONT.getClass(metadata.classname())); - if (metadata.conVersion().isPresent()) - m.add(connectionRes, IPROP.version, ResourceFactory.createTypedLiteral(metadata.conVersion().get())); - m.add(connectionRes, RDFS.label, ResourceFactory.createTypedLiteral(metadata.conID())); - m.add(connectionRes, RDF.type, IONT.connection); - - for (WorkerMetadata worker : metadata.workers()) { - Resource workerRes = IRES.getWorkerResource(metadata.taskID(), worker.workerID()); - m.add(taskRes, IPROP.workerResult, workerRes); - m.add(workerRes, IPROP.workerID, ResourceFactory.createTypedLiteral(worker.workerID())); - m.add(workerRes, IPROP.workerType, ResourceFactory.createTypedLiteral(worker.workerType())); - m.add(workerRes, IPROP.noOfQueries, ResourceFactory.createTypedLiteral(worker.queryIDs().length)); - m.add(workerRes, IPROP.timeOut, ResourceFactory.createTypedLiteral(worker.timeout())); - m.add(workerRes, RDF.type, IONT.worker); - } - - if (metadata.tripleStats().isPresent()) { - m.add(metadata.tripleStats().get()); - // Connect task and workers to the Query nodes, that store the triple stats. - for (WorkerMetadata worker : metadata.workers()) { - for (String queryID : worker.queryIDs()) { - int intID = Integer.parseInt(queryID.substring(queryID.indexOf(":") + 1)); - Resource workerQueryRes = IRES.getWorkerQueryResource(metadata.taskID(), worker.workerID(), queryID); - Resource queryRes = IRES.getResource(worker.queryHash() + "/" + intID); - m.add(workerQueryRes, IPROP.queryID, queryRes); - } - - for (String queryID : metadata.queryIDs()) { - int intID = Integer.parseInt(queryID.substring(queryID.indexOf(":") + 1)); - Resource taskQueryRes = IRES.getTaskQueryResource(metadata.taskID(), queryID); - Resource queryRes = IRES.getResource(worker.queryHash() + "/" + intID); - m.add(taskQueryRes, IPROP.queryID, queryRes); - } - } - } - - for (Metric metric : metrics) { - m.add(this.createMetricModel(metric)); - } - - // Task to queries - for (String queryID : metadata.queryIDs()) { - m.add(taskRes, IPROP.query, IRES.getTaskQueryResource(metadata.taskID(), queryID)); - } - - // Worker to queries - for (WorkerMetadata worker : metadata.workers()) { - for (String queryID : worker.queryIDs()) { - Resource workerRes = IRES.getWorkerResource(metadata.taskID(), worker.workerID()); - m.add(workerRes, IPROP.query, IRES.getWorkerQueryResource(metadata.taskID(), worker.workerID(), queryID)); - } - } - - m.add(taskRes, IPROP.startDate, ResourceFactory.createTypedLiteral(start)); - m.add(taskRes, IPROP.endDate, ResourceFactory.createTypedLiteral(end)); - - m.setNsPrefixes(IGUANA_BASE.PREFIX_MAP); - - StorageManager.getInstance().storeResult(m); - } - - /** - * For a given metric this method calculates the metric with the stored data and creates the appropriate - * RDF related to that metric. - * - * @param metric the metric that should be calculated - * @return the result model of the metric - */ - private Model createMetricModel(Metric metric) { - Model m = ModelFactory.createDefaultModel(); - Property metricProp = IPROP.createMetricProperty(metric); - Resource metricRes = IRES.getMetricResource(metric); - - if (metric instanceof ModelWritingMetric) { - m.add(((ModelWritingMetric) metric).createMetricModel(metadata, workerQueryExecutions)); - m.add(((ModelWritingMetric) metric).createMetricModel(metadata, taskQueryExecutions)); - } - - if (metric instanceof TaskMetric) { - Number metricValue = ((TaskMetric) metric).calculateTaskMetric(metadata, workerQueryExecutions); - if (metricValue != null) { - Literal lit = ResourceFactory.createTypedLiteral(metricValue); - m.add(taskRes, metricProp, lit); - } - m.add(taskRes, IPROP.metric, metricRes); - } - - if (metric instanceof WorkerMetric) { - for (WorkerMetadata worker : metadata.workers()) { - Resource workerRes = IRES.getWorkerResource(metadata.taskID(), worker.workerID()); - Number metricValue = ((WorkerMetric) metric).calculateWorkerMetric(worker, workerQueryExecutions[worker.workerID()]); - if (metricValue != null) { - Literal lit = ResourceFactory.createTypedLiteral(metricValue); - m.add(workerRes, metricProp, lit); - } - m.add(workerRes, IPROP.metric, metricRes); - } - } - - if (metric instanceof QueryMetric) { - // queries grouped by worker - for (WorkerMetadata worker : metadata.workers()) { - for (int i = 0; i < worker.numberOfQueries(); i++) { - Number metricValue = ((QueryMetric) metric).calculateQueryMetric(workerQueryExecutions[worker.workerID()][i]); - if (metricValue != null) { - Literal lit = ResourceFactory.createTypedLiteral(metricValue); - Resource queryRes = IRES.getWorkerQueryResource(metadata.taskID(), worker.workerID(), worker.queryIDs()[i]); - m.add(queryRes, metricProp, lit); - } - } - } - - // queries grouped by task - for (String queryID : taskQueryExecutions.keySet()) { - Number metricValue = ((QueryMetric) metric).calculateQueryMetric(taskQueryExecutions.get(queryID)); - if (metricValue != null) { - Literal lit = ResourceFactory.createTypedLiteral(metricValue); - Resource queryRes = IRES.getTaskQueryResource(metadata.taskID(), queryID); - m.add(queryRes, metricProp, lit); - } - } - } - - m.add(metricRes, RDFS.label, metric.getName()); - m.add(metricRes, RDFS.label, metric.getAbbreviation()); - m.add(metricRes, RDFS.comment, metric.getDescription()); - m.add(metricRes, RDF.type, IONT.getMetricClass(metric)); - m.add(metricRes, RDF.type, IONT.metric); - - return m; - } -} diff --git a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/Metric.java b/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/Metric.java deleted file mode 100644 index 2e6a79403..000000000 --- a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/Metric.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.aksw.iguana.cc.tasks.stresstest.metrics; - -public abstract class Metric { - private final String name; - private final String abbreviation; - private final String description; - - public Metric(String name, String abbreviation, String description) { - this.name = name; - this.abbreviation = abbreviation; - this.description = description; - } - - - public String getDescription(){ - return this.description; - } - - public String getName(){ - return this.name; - } - - - public String getAbbreviation(){ - return this.abbreviation; - } -} diff --git a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/MetricManager.java b/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/MetricManager.java deleted file mode 100644 index 5320774fa..000000000 --- a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/MetricManager.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.aksw.iguana.cc.tasks.stresstest.metrics; - -import java.util.List; - -public class MetricManager { - private static List metrics; - - public static void setMetrics(List metrics) { - MetricManager.metrics = metrics; - } - - public static List getMetrics() { - return MetricManager.metrics; - } -} diff --git a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/ModelWritingMetric.java b/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/ModelWritingMetric.java deleted file mode 100644 index bbce4aa49..000000000 --- a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/ModelWritingMetric.java +++ /dev/null @@ -1,20 +0,0 @@ -package org.aksw.iguana.cc.tasks.stresstest.metrics; - -import org.aksw.iguana.cc.model.QueryExecutionStats; -import org.aksw.iguana.cc.tasks.stresstest.StresstestMetadata; -import org.apache.jena.rdf.model.Model; -import org.apache.jena.rdf.model.ModelFactory; - -import javax.annotation.Nonnull; -import java.util.List; -import java.util.Map; - -public interface ModelWritingMetric { - default @Nonnull Model createMetricModel(StresstestMetadata task, List[][] data) { - return ModelFactory.createDefaultModel(); - } - - default @Nonnull Model createMetricModel(StresstestMetadata task, Map> data) { - return ModelFactory.createDefaultModel(); - } -} diff --git a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/QueryMetric.java b/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/QueryMetric.java deleted file mode 100644 index f4a793f1a..000000000 --- a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/QueryMetric.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.aksw.iguana.cc.tasks.stresstest.metrics; - -import org.aksw.iguana.cc.model.QueryExecutionStats; - -import java.util.List; - -public interface QueryMetric { - Number calculateQueryMetric(List data); -} diff --git a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/TaskMetric.java b/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/TaskMetric.java deleted file mode 100644 index 6995f24d1..000000000 --- a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/TaskMetric.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.aksw.iguana.cc.tasks.stresstest.metrics; - -import org.aksw.iguana.cc.model.QueryExecutionStats; -import org.aksw.iguana.cc.tasks.stresstest.StresstestMetadata; - -import java.util.List; - -public interface TaskMetric { - Number calculateTaskMetric(StresstestMetadata task, List[][] data); -} diff --git a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/WorkerMetric.java b/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/WorkerMetric.java deleted file mode 100644 index bc81071e6..000000000 --- a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/WorkerMetric.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.aksw.iguana.cc.tasks.stresstest.metrics; - -import org.aksw.iguana.cc.model.QueryExecutionStats; -import org.aksw.iguana.cc.worker.WorkerMetadata; - -import java.util.List; - -public interface WorkerMetric { - Number calculateWorkerMetric(WorkerMetadata worker, List[] data); -} diff --git a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/AggregatedExecutionStatistics.java b/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/AggregatedExecutionStatistics.java deleted file mode 100644 index a2e332cba..000000000 --- a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/AggregatedExecutionStatistics.java +++ /dev/null @@ -1,101 +0,0 @@ -package org.aksw.iguana.cc.tasks.stresstest.metrics.impl; - -import org.aksw.iguana.cc.model.QueryExecutionStats; -import org.aksw.iguana.cc.tasks.stresstest.StresstestMetadata; -import org.aksw.iguana.cc.worker.WorkerMetadata; -import org.aksw.iguana.cc.tasks.stresstest.metrics.Metric; -import org.aksw.iguana.cc.tasks.stresstest.metrics.ModelWritingMetric; -import org.aksw.iguana.commons.annotation.Shorthand; -import org.aksw.iguana.commons.constants.COMMON; -import org.aksw.iguana.commons.rdf.IONT; -import org.aksw.iguana.commons.rdf.IPROP; -import org.aksw.iguana.commons.rdf.IRES; -import org.apache.jena.rdf.model.Model; -import org.apache.jena.rdf.model.ModelFactory; -import org.apache.jena.rdf.model.Resource; -import org.apache.jena.rdf.model.ResourceFactory; -import org.apache.jena.vocabulary.RDF; - -import javax.annotation.Nonnull; -import java.math.BigInteger; -import java.time.Duration; -import java.util.List; -import java.util.Map; - -import static org.aksw.iguana.commons.time.TimeUtils.toXSDDurationInSeconds; - -@Shorthand("AES") -public class AggregatedExecutionStatistics extends Metric implements ModelWritingMetric { - - public AggregatedExecutionStatistics() { - super("Aggregated Execution Statistics", "AES", "Sums up the statistics of each query execution for each query a worker and task has. The result size only contains the value of the last execution."); - } - - @Override - @Nonnull - public Model createMetricModel(StresstestMetadata task, List[][] data) { - Model m = ModelFactory.createDefaultModel(); - for (WorkerMetadata worker : task.workers()) { - for (int i = 0; i < worker.numberOfQueries(); i++) { - Resource queryRes = IRES.getWorkerQueryResource(task.taskID(), worker.workerID(), worker.queryIDs()[i]); - m.add(createAggregatedModel(data[worker.workerID()][i], queryRes)); - } - } - return m; - } - - @Override - @Nonnull - public Model createMetricModel(StresstestMetadata task, Map> data) { - Model m = ModelFactory.createDefaultModel(); - for (String queryID : data.keySet()) { - Resource queryRes = IRES.getTaskQueryResource(task.taskID(), queryID); - m.add(createAggregatedModel(data.get(queryID), queryRes)); - } - return m; - } - - private static Model createAggregatedModel(List data, Resource queryRes) { - Model m = ModelFactory.createDefaultModel(); - BigInteger succeeded = BigInteger.ZERO; - BigInteger failed = BigInteger.ZERO; - BigInteger resultSize = BigInteger.ZERO; - BigInteger wrongCodes = BigInteger.ZERO; - BigInteger timeOuts = BigInteger.ZERO; - BigInteger unknownExceptions = BigInteger.ZERO; - Duration totalTime = Duration.ZERO; - - for (QueryExecutionStats exec : data) { - // TODO: make response code integer - switch ((int) exec.responseCode()) { - case (int) COMMON.QUERY_SUCCESS -> succeeded = succeeded.add(BigInteger.ONE); - case (int) COMMON.QUERY_SOCKET_TIMEOUT -> { - timeOuts = timeOuts.add(BigInteger.ONE); - failed = failed.add(BigInteger.ONE); - } - case (int) COMMON.QUERY_HTTP_FAILURE -> { - wrongCodes = wrongCodes.add(BigInteger.ONE); - failed = failed.add(BigInteger.ONE); - } - case (int) COMMON.QUERY_UNKNOWN_EXCEPTION -> { - unknownExceptions = unknownExceptions.add(BigInteger.ONE); - failed = failed.add(BigInteger.ONE); - } - } - - totalTime = totalTime.plusNanos((long) (exec.executionTime() * 1000000)); - resultSize = BigInteger.valueOf(exec.resultSize()); - } - - m.add(queryRes, IPROP.succeeded, ResourceFactory.createTypedLiteral(succeeded)); - m.add(queryRes, IPROP.failed, ResourceFactory.createTypedLiteral(failed)); - m.add(queryRes, IPROP.resultSize, ResourceFactory.createTypedLiteral(resultSize)); - m.add(queryRes, IPROP.timeOuts, ResourceFactory.createTypedLiteral(timeOuts)); - m.add(queryRes, IPROP.wrongCodes, ResourceFactory.createTypedLiteral(wrongCodes)); - m.add(queryRes, IPROP.unknownException, ResourceFactory.createTypedLiteral(unknownExceptions)); - m.add(queryRes, IPROP.totalTime, ResourceFactory.createTypedLiteral(toXSDDurationInSeconds(totalTime))); - m.add(queryRes, RDF.type, IONT.executedQuery); - - return m; - } -} diff --git a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/AvgQPS.java b/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/AvgQPS.java deleted file mode 100644 index 5d56e7f9e..000000000 --- a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/AvgQPS.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.aksw.iguana.cc.tasks.stresstest.metrics.impl; - -import org.aksw.iguana.cc.model.QueryExecutionStats; -import org.aksw.iguana.cc.tasks.stresstest.StresstestMetadata; -import org.aksw.iguana.cc.worker.WorkerMetadata; -import org.aksw.iguana.cc.tasks.stresstest.metrics.Metric; -import org.aksw.iguana.cc.tasks.stresstest.metrics.TaskMetric; -import org.aksw.iguana.cc.tasks.stresstest.metrics.WorkerMetric; -import org.aksw.iguana.commons.annotation.Shorthand; - -import java.math.BigDecimal; -import java.math.RoundingMode; -import java.util.List; - -@Shorthand("AvgQPS") -public class AvgQPS extends Metric implements TaskMetric, WorkerMetric { - - public AvgQPS() { - super("Average Queries per Second", "AvgQPS", "This metric calculates the average QPS between all queries."); - } - - @Override - public Number calculateTaskMetric(StresstestMetadata task, List[][] data) { - BigDecimal sum = BigDecimal.ZERO; - for (WorkerMetadata worker : task.workers()) { - sum = sum.add((BigDecimal) this.calculateWorkerMetric(worker, data[worker.workerID()])); - } - - try { - return sum.divide(BigDecimal.valueOf(data.length), 10, RoundingMode.HALF_UP); - } catch (ArithmeticException e) { - return BigDecimal.ZERO; - } - } - - @Override - public Number calculateWorkerMetric(WorkerMetadata worker, List[] data) { - BigDecimal sum = BigDecimal.ZERO; - QPS qpsmetric = new QPS(); - for (List datum : data) { - sum = sum.add((BigDecimal) qpsmetric.calculateQueryMetric(datum)); - } - - try { - return sum.divide(BigDecimal.valueOf(data.length), 10, RoundingMode.HALF_UP); - } catch (ArithmeticException e) { - return BigDecimal.ZERO; - } - } -} diff --git a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/EachExecutionStatistic.java b/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/EachExecutionStatistic.java deleted file mode 100644 index f3771943a..000000000 --- a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/EachExecutionStatistic.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.aksw.iguana.cc.tasks.stresstest.metrics.impl; - -import org.aksw.iguana.cc.model.QueryExecutionStats; -import org.aksw.iguana.cc.tasks.stresstest.StresstestMetadata; -import org.aksw.iguana.cc.worker.WorkerMetadata; -import org.aksw.iguana.cc.tasks.stresstest.metrics.Metric; -import org.aksw.iguana.cc.tasks.stresstest.metrics.ModelWritingMetric; -import org.aksw.iguana.commons.annotation.Shorthand; -import org.aksw.iguana.commons.constants.COMMON; -import org.aksw.iguana.commons.rdf.IPROP; -import org.aksw.iguana.commons.rdf.IRES; -import org.apache.jena.rdf.model.Model; -import org.apache.jena.rdf.model.ModelFactory; -import org.apache.jena.rdf.model.Resource; -import org.apache.jena.rdf.model.ResourceFactory; - -import javax.annotation.Nonnull; -import java.math.BigInteger; -import java.util.List; - -@Shorthand("EachQuery") -public class EachExecutionStatistic extends Metric implements ModelWritingMetric { - - public EachExecutionStatistic() { - super("Each Query Execution Statistic", "EachQuery", "This metric saves the statistics of each query execution."); - } - - @Override - @Nonnull - public Model createMetricModel(StresstestMetadata task, List[][] data) { - Model m = ModelFactory.createDefaultModel(); - for (WorkerMetadata worker : task.workers()) { - for (int i = 0; i < worker.numberOfQueries(); i++) { - Resource queryRes = IRES.getWorkerQueryResource(task.taskID(), worker.workerID(), worker.queryIDs()[i]); - Resource query = IRES.getResource(worker.queryHash() + "/" + worker.queryIDs()[i]); - BigInteger run = BigInteger.ONE; - for (QueryExecutionStats exec : data[worker.workerID()][i]) { - Resource runRes = IRES.getWorkerQueryRunResource(task.taskID(), worker.workerID(), worker.queryIDs()[i], run); - m.add(queryRes, IPROP.queryExecution, runRes); - m.add(runRes, IPROP.time, ResourceFactory.createTypedLiteral(exec.executionTime())); - m.add(runRes, IPROP.success, ResourceFactory.createTypedLiteral(exec.responseCode() == COMMON.QUERY_SUCCESS)); - m.add(runRes, IPROP.run, ResourceFactory.createTypedLiteral(run)); - m.add(runRes, IPROP.code, ResourceFactory.createTypedLiteral(exec.responseCode())); - m.add(runRes, IPROP.resultSize, ResourceFactory.createTypedLiteral(exec.resultSize())); - m.add(runRes, IPROP.queryID, query); - run = run.add(BigInteger.ONE); - } - } - } - return m; - } -} diff --git a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/NoQ.java b/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/NoQ.java deleted file mode 100644 index e1af28be1..000000000 --- a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/NoQ.java +++ /dev/null @@ -1,43 +0,0 @@ -package org.aksw.iguana.cc.tasks.stresstest.metrics.impl; - -import org.aksw.iguana.cc.model.QueryExecutionStats; -import org.aksw.iguana.cc.tasks.stresstest.StresstestMetadata; -import org.aksw.iguana.cc.worker.WorkerMetadata; -import org.aksw.iguana.cc.tasks.stresstest.metrics.Metric; -import org.aksw.iguana.cc.tasks.stresstest.metrics.TaskMetric; -import org.aksw.iguana.cc.tasks.stresstest.metrics.WorkerMetric; -import org.aksw.iguana.commons.annotation.Shorthand; -import org.aksw.iguana.commons.constants.COMMON; - -import java.math.BigInteger; -import java.util.List; - -@Shorthand("NoQ") -public class NoQ extends Metric implements TaskMetric, WorkerMetric { - - public NoQ() { - super("Number of Queries", "NoQ", "This metric calculates the number of successfully executed queries."); - } - - @Override - public Number calculateTaskMetric(StresstestMetadata task, List[][] data) { - BigInteger sum = BigInteger.ZERO; - for (WorkerMetadata worker : task.workers()) { - sum = sum.add((BigInteger) this.calculateWorkerMetric(worker, data[worker.workerID()])); - } - return sum; - } - - @Override - public Number calculateWorkerMetric(WorkerMetadata worker, List[] data) { - BigInteger sum = BigInteger.ZERO; - for (List datum : data) { - for (QueryExecutionStats exec : datum) { - if (exec.responseCode() == COMMON.QUERY_SUCCESS) { - sum = sum.add(BigInteger.ONE); - } - } - } - return sum; - } -} diff --git a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/NoQPH.java b/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/NoQPH.java deleted file mode 100644 index 4015d3df9..000000000 --- a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/NoQPH.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.aksw.iguana.cc.tasks.stresstest.metrics.impl; - -import org.aksw.iguana.cc.model.QueryExecutionStats; -import org.aksw.iguana.cc.tasks.stresstest.StresstestMetadata; -import org.aksw.iguana.cc.worker.WorkerMetadata; -import org.aksw.iguana.cc.tasks.stresstest.metrics.Metric; -import org.aksw.iguana.cc.tasks.stresstest.metrics.TaskMetric; -import org.aksw.iguana.cc.tasks.stresstest.metrics.WorkerMetric; -import org.aksw.iguana.commons.annotation.Shorthand; -import org.aksw.iguana.commons.constants.COMMON; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.math.RoundingMode; -import java.time.Duration; -import java.util.List; - -@Shorthand("NoQPH") -public class NoQPH extends Metric implements TaskMetric, WorkerMetric { - - public NoQPH() { - super("Number of Queries per Hour", "NoQPH", "This metric calculates the number of successfully executed queries per hour."); - } - @Override - public Number calculateTaskMetric(StresstestMetadata task, List[][] data) { - BigDecimal sum = BigDecimal.ZERO; - for (WorkerMetadata worker : task.workers()) { - sum = sum.add((BigDecimal) this.calculateWorkerMetric(worker, data[worker.workerID()])); - } - return sum; - } - - @Override - public Number calculateWorkerMetric(WorkerMetadata worker, List[] data) { - BigDecimal successes = BigDecimal.ZERO; - Duration totalTime = Duration.ZERO; - for (List datum : data) { - for (QueryExecutionStats exec : datum) { - if (exec.responseCode() == COMMON.QUERY_SUCCESS) { - successes = successes.add(BigDecimal.ONE); - totalTime = totalTime.plusNanos((long) exec.executionTime() * 1000000); - } - } - } - BigDecimal tt = (new BigDecimal(BigInteger.valueOf(totalTime.toNanos()), 9)).divide(BigDecimal.valueOf(3600), 20, RoundingMode.HALF_UP); - - try { - return successes.divide(tt, 10, RoundingMode.HALF_UP); - } catch (ArithmeticException e) { - return BigDecimal.ZERO; - } - } -} diff --git a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/PQPS.java b/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/PQPS.java deleted file mode 100644 index bbefdc12b..000000000 --- a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/PQPS.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.aksw.iguana.cc.tasks.stresstest.metrics.impl; - -import org.aksw.iguana.cc.model.QueryExecutionStats; -import org.aksw.iguana.cc.tasks.stresstest.metrics.Metric; -import org.aksw.iguana.cc.tasks.stresstest.metrics.QueryMetric; -import org.aksw.iguana.commons.annotation.Shorthand; -import org.aksw.iguana.commons.constants.COMMON; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.math.RoundingMode; -import java.time.Duration; -import java.util.List; - -@Shorthand("PQPS") -public class PQPS extends Metric implements QueryMetric { - - private final int penalty; - - public PQPS(Integer penalty) { - super("Penalized Queries per Second", "PQPS", "This metric calculates for each query the amount of executions per second. Failed executions receive a time penalty."); - this.penalty = penalty; - } - - @Override - public Number calculateQueryMetric(List data) { - BigDecimal successes = BigDecimal.ZERO; - Duration totalTime = Duration.ZERO; - for (QueryExecutionStats exec : data) { - successes = successes.add(BigDecimal.ONE); - if (exec.responseCode() == COMMON.QUERY_SUCCESS) { - totalTime = totalTime.plusNanos((long) exec.executionTime() * 1000000); - } else { - totalTime = totalTime.plusMillis(penalty); - } - } - BigDecimal tt = (new BigDecimal(BigInteger.valueOf(totalTime.toNanos()), 9)); - - try { - return successes.divide(tt, 10, RoundingMode.HALF_UP); - } catch (ArithmeticException e) { - return BigDecimal.ZERO; - } - } -} diff --git a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/QMPH.java b/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/QMPH.java deleted file mode 100644 index c3a3e01cf..000000000 --- a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/metrics/impl/QMPH.java +++ /dev/null @@ -1,54 +0,0 @@ -package org.aksw.iguana.cc.tasks.stresstest.metrics.impl; - -import org.aksw.iguana.cc.model.QueryExecutionStats; -import org.aksw.iguana.cc.tasks.stresstest.StresstestMetadata; -import org.aksw.iguana.cc.worker.WorkerMetadata; -import org.aksw.iguana.cc.tasks.stresstest.metrics.Metric; -import org.aksw.iguana.cc.tasks.stresstest.metrics.TaskMetric; -import org.aksw.iguana.cc.tasks.stresstest.metrics.WorkerMetric; -import org.aksw.iguana.commons.annotation.Shorthand; -import org.aksw.iguana.commons.constants.COMMON; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.math.RoundingMode; -import java.time.Duration; -import java.util.List; - -@Shorthand("QMPH") -public class QMPH extends Metric implements TaskMetric, WorkerMetric { - - public QMPH() { - super("Query Mixes per Hour", "QMPH", "This metric calculates the amount of query mixes (a given set of queries) that are executed per hour."); - } - @Override - public Number calculateTaskMetric(StresstestMetadata task, List[][] data) { - BigDecimal sum = BigDecimal.ZERO; - for (WorkerMetadata worker : task.workers()) { - sum = sum.add((BigDecimal) this.calculateWorkerMetric(worker, data[worker.workerID()])); - } - return sum; - } - - @Override - public Number calculateWorkerMetric(WorkerMetadata worker, List[] data) { - BigDecimal successes = BigDecimal.ZERO; - BigDecimal noq = BigDecimal.valueOf(worker.numberOfQueries()); - Duration totalTime = Duration.ZERO; - for (List datum : data) { - for (QueryExecutionStats exec : datum) { - if (exec.responseCode() == COMMON.QUERY_SUCCESS) { - successes = successes.add(BigDecimal.ONE); - totalTime = totalTime.plusNanos((long) exec.executionTime() * 1000000); - } - } - } - BigDecimal tt = (new BigDecimal(BigInteger.valueOf(totalTime.toNanos()), 9)).divide(BigDecimal.valueOf(3600), 20, RoundingMode.HALF_UP); - - try { - return successes.divide(tt, 10, RoundingMode.HALF_UP).divide(noq, 10, RoundingMode.HALF_UP); - } catch (ArithmeticException e) { - return BigDecimal.ZERO; - } - } -} diff --git a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/storage/Storage.java b/src/main/java/org/aksw/iguana/cc/tasks/stresstest/storage/Storage.java deleted file mode 100644 index d1b045651..000000000 --- a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/storage/Storage.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.aksw.iguana.cc.tasks.stresstest.storage; - -import org.apache.jena.rdf.model.Model; - -/** - * Interface for the Result Storages - * - * @author f.conrads - * - */ -public interface Storage { - - /** - * Stores the task result into the storage. This method will be executed after a task has finished. - * - * @param data the given result model - */ - void storeResult(Model data); -} diff --git a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/storage/StorageManager.java b/src/main/java/org/aksw/iguana/cc/tasks/stresstest/storage/StorageManager.java deleted file mode 100644 index 5de7fd4e0..000000000 --- a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/storage/StorageManager.java +++ /dev/null @@ -1,73 +0,0 @@ -package org.aksw.iguana.cc.tasks.stresstest.storage; - -import org.apache.jena.rdf.model.Model; - -import java.util.*; - - -/** - * Manager for Storages - * - * @author f.conrads - * - */ -public class StorageManager { - - private Set storages = new HashSet<>(); - - private static StorageManager instance; - - public static synchronized StorageManager getInstance() { - if (instance == null) { - instance = new StorageManager(); - } - return instance; - } - - /** - * Will add the Storage - * - * @param storage - */ - public void addStorage(Storage storage){ - if(storage==null){ - return; - } - storages.add(storage); - } - - /** - * Will return each Storage - * - * @return - */ - public Set getStorages(){ - return storages; - } - - /** - * Simply adds a Model - * @param m - */ - public void storeResult(Model m){ - for(Storage s : storages){ - s.storeResult(m); - } - } - - @Override - public String toString(){ - StringBuilder ret = new StringBuilder(); - Iterator it = storages.iterator(); - for(int i=0;i storages) { - this.storages.addAll(storages); - } -} diff --git a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/storage/TripleBasedStorage.java b/src/main/java/org/aksw/iguana/cc/tasks/stresstest/storage/TripleBasedStorage.java deleted file mode 100644 index 599113307..000000000 --- a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/storage/TripleBasedStorage.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * - */ -package org.aksw.iguana.cc.tasks.stresstest.storage; - -import org.aksw.iguana.commons.constants.COMMON; -import org.apache.jena.rdf.model.Model; -import org.apache.jena.rdf.model.ModelFactory; - -/** - * This Storage will save all the metric results as triples - * - * @author f.conrads - * - */ -public abstract class TripleBasedStorage implements Storage { - - protected String baseUri = COMMON.BASE_URI; - protected Model metricResults = ModelFactory.createDefaultModel(); - - @Override - public String toString() { - return this.getClass().getSimpleName(); - } - - public void storeResult(Model data){ - metricResults.add(data); - } -} diff --git a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/storage/impl/CSVStorage.java b/src/main/java/org/aksw/iguana/cc/tasks/stresstest/storage/impl/CSVStorage.java deleted file mode 100644 index f382c8e45..000000000 --- a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/storage/impl/CSVStorage.java +++ /dev/null @@ -1,282 +0,0 @@ -package org.aksw.iguana.cc.tasks.stresstest.storage.impl; - -import com.opencsv.CSVReader; -import com.opencsv.CSVWriter; -import com.opencsv.CSVWriterBuilder; -import com.opencsv.exceptions.CsvValidationException; -import org.aksw.iguana.cc.config.IguanaConfig; -import org.aksw.iguana.cc.tasks.stresstest.metrics.*; -import org.aksw.iguana.cc.tasks.stresstest.metrics.impl.AggregatedExecutionStatistics; -import org.aksw.iguana.cc.tasks.stresstest.storage.Storage; -import org.aksw.iguana.commons.annotation.Shorthand; -import org.aksw.iguana.commons.rdf.IONT; -import org.aksw.iguana.commons.rdf.IPROP; -import org.apache.jena.arq.querybuilder.SelectBuilder; -import org.apache.jena.query.*; -import org.apache.jena.rdf.model.*; -import org.apache.jena.sparql.lang.sparql_11.ParseException; -import org.apache.jena.vocabulary.RDF; -import org.apache.jena.vocabulary.RDFS; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.*; -import java.nio.file.Files; -import java.nio.file.InvalidPathException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.LinkedList; -import java.util.List; -import java.util.NoSuchElementException; -import java.util.function.Predicate; - -@Shorthand("CSVStorage") -public class CSVStorage implements Storage { - - private static final Logger LOGGER = LoggerFactory.getLogger(CSVStorage.class); - - private final Path folder; - private final Path taskFile; - - private List workerResources; - private Resource taskRes; - private String connection; - private String connectionVersion; - private String dataset; - - public CSVStorage(String folderPath) { - Path parentFolder; - try { - parentFolder = Paths.get(folderPath); - } catch (InvalidPathException e) { - LOGGER.error("Can't store csv files, the given path is invalid.", e); - this.folder = null; - this.taskFile = null; - return; - } - - this.folder = parentFolder.resolve(IguanaConfig.getSuiteID()); - this.taskFile = this.folder.resolve("tasks-overview.csv"); - - if (Files.notExists(parentFolder)) { - try { - Files.createDirectory(parentFolder); - } catch (IOException e) { - LOGGER.error("Can't store csv files, directory couldn't be created.", e); - return; - } - } - - if (Files.notExists(folder)) { - try { - Files.createDirectory(folder); - } catch (IOException e) { - LOGGER.error("Can't store csv files, directory couldn't be created.", e); - return; - } - } - - try { - Files.createFile(taskFile); - } catch (IOException e) { - LOGGER.error("Couldn't create the file: " + taskFile.toAbsolutePath(), e); - return; - } - - // write headers for the tasks.csv file - // This only works because the metrics are initialized sooner - try (CSVWriter csvWriter = getCSVWriter(taskFile)) { - Metric[] taskMetrics = MetricManager.getMetrics().stream().filter(x -> TaskMetric.class.isAssignableFrom(x.getClass())).toArray(Metric[]::new); - List headerList = new LinkedList<>(); - headerList.addAll(List.of("connection", "dataset", "startDate", "endDate", "noOfWorkers")); - headerList.addAll(Arrays.stream(taskMetrics).map(Metric::getAbbreviation).toList()); - String[] header = headerList.toArray(String[]::new); - csvWriter.writeNext(header, true); - } catch (IOException e) { - LOGGER.error("Error while writing to file: " + taskFile.toAbsolutePath(), e); - } - } - - /** - * Stores the task result into the storage. This method will be executed after a task has finished. - * - * @param data the given result model - */ - @Override - public void storeResult(Model data) { - try { - setObjectAttributes(data); - } catch (NoSuchElementException e) { - LOGGER.error("Error while querying the result model. The given model is probably incorrect.", e); - return; - } - - try { - storeTaskResults(data); - } catch (IOException e) { - LOGGER.error("Error while storing the task result in a csv file.", e); - } catch (NoSuchElementException | ParseException e) { - LOGGER.error("Error while storing the task result in a csv file. The given model is probably incorrect.", e); - } - - try { - Path temp = createCSVFile(dataset, connection, connectionVersion, "worker"); - storeWorkerResults(this.taskRes, temp, data); - for (Resource workerRes : workerResources) { - String workerID = data.listObjectsOfProperty(workerRes, IPROP.workerID).next().asLiteral().getLexicalForm(); - try { - Path file = createCSVFile(dataset, connection, connectionVersion, "worker", "query", workerID); - storeQueryResults(workerRes, file, data); - } catch (IOException e) { - LOGGER.error("Error while storing the query results of a worker in a csv file.", e); - } catch (NoSuchElementException e) { - LOGGER.error("Error while storing the query results of a worker in a csv file. The given model is probably incorrect.", e); - } - } - } catch (IOException e) { - LOGGER.error("Error while storing the worker results in a csv file.", e); - } catch (NoSuchElementException e) { - LOGGER.error("Error while storing the worker results in a csv file. The given model is probably incorrect.", e); - } - - try { - Path file = createCSVFile(dataset, connection, connectionVersion, "query"); - storeQueryResults(taskRes, file, data); - } catch (IOException e) { - LOGGER.error("Error while storing the query results of a task result in a csv file.", e); - } catch (NoSuchElementException e) { - LOGGER.error("Error while storing the query results of a task result in a csv file. The given model is probably incorrect.", e); - } - } - - /** - * This method sets the objects attributes by querying the given model. - * - * @param data the result model - * @throws NoSuchElementException might be thrown if the model is incorrect - */ - private void setObjectAttributes(Model data) throws NoSuchElementException { - ResIterator resIterator = data.listSubjectsWithProperty(RDF.type, IONT.dataset); - Resource datasetRes = resIterator.nextResource(); - NodeIterator nodeIterator = data.listObjectsOfProperty(datasetRes, RDFS.label); - this.dataset = nodeIterator.next().asLiteral().getLexicalForm(); - - resIterator = data.listSubjectsWithProperty(RDF.type, IONT.connection); - Resource connectionRes = resIterator.nextResource(); - nodeIterator = data.listObjectsOfProperty(connectionRes, RDFS.label); - this.connection = nodeIterator.next().asLiteral().getLexicalForm(); - this.connectionVersion = ""; - nodeIterator = data.listObjectsOfProperty(connectionRes, IPROP.version); - if (nodeIterator.hasNext()) { - this.connectionVersion = nodeIterator.next().toString(); - } - - resIterator = data.listSubjectsWithProperty(RDF.type, IONT.task); - this.taskRes = resIterator.nextResource(); - - nodeIterator = data.listObjectsOfProperty(this.taskRes, IPROP.workerResult); - this.workerResources = nodeIterator.toList().stream().map(RDFNode::asResource).toList(); - } - - /** - * Creates a CSV file with the given name values that will be located inside the parent folder. The name value are - * joined together with the character '-'. Empty values will be ignored. - * - * @param nameValues strings that build up the name of the file - * @throws IOException if an I/O error occurs - * @return path object to the created CSV file - */ - private Path createCSVFile(String... nameValues) throws IOException { - // remove empty string values - nameValues = Arrays.stream(nameValues).filter(Predicate.not(String::isEmpty)).toArray(String[]::new); - String filename = String.join("-", nameValues) + ".csv"; - Path file = this.folder.resolve(filename); - Files.createFile(file); - return file; - } - - private static void storeQueryResults(Resource parentRes, Path file, Model data) throws IOException, NoSuchElementException { - boolean containsAggrStats = !MetricManager.getMetrics().stream().filter(AggregatedExecutionStatistics.class::isInstance).toList().isEmpty(); - Metric[] queryMetrics = MetricManager.getMetrics().stream().filter(x -> QueryMetric.class.isAssignableFrom(x.getClass())).toArray(Metric[]::new); - - SelectBuilder sb = new SelectBuilder(); - sb.addWhere(parentRes, IPROP.query, "?eQ"); - queryProperties(sb, "?eQ", IPROP.queryID); - if (containsAggrStats) { - queryProperties(sb, "?eQ", IPROP.succeeded, IPROP.failed, IPROP.totalTime, IPROP.resultSize, IPROP.wrongCodes, IPROP.timeOuts, IPROP.unknownException); - } - queryMetrics(sb, "?eQ", queryMetrics); - - executeAndStoreQuery(sb, file, data); - } - - private void storeTaskResults(Model data) throws IOException, NoSuchElementException, ParseException { - Metric[] taskMetrics = MetricManager.getMetrics().stream().filter(x -> TaskMetric.class.isAssignableFrom(x.getClass())).toArray(Metric[]::new); - - SelectBuilder sb = new SelectBuilder(); - sb.addVar("connection") - .addWhere("?taskRes", IPROP.connection, "?connRes") - .addWhere("?connRes", RDFS.label, "?connection") - .addVar("dataset") - .addWhere("?expRes", IPROP.dataset, "?datasetRes") - .addWhere("?datasetRes", RDFS.label, "?dataset"); - queryProperties(sb, String.format("<%s>", this.taskRes.toString()), IPROP.startDate, IPROP.endDate, IPROP.noOfWorkers); - queryMetrics(sb, String.format("<%s>", this.taskRes.toString()), taskMetrics); - - try(QueryExecution exec = QueryExecutionFactory.create(sb.build(), data); - CSVWriter csvWriter = getCSVWriter(taskFile); - ByteArrayOutputStream baos = new ByteArrayOutputStream()) { - ResultSet results = exec.execSelect(); - ResultSetFormatter.outputAsCSV(baos, results); - - // workaround to remove the created header from the ResultSetFormatter - CSVReader reader = new CSVReader(new StringReader(baos.toString())); - try { - reader.readNext(); - csvWriter.writeNext(reader.readNext(), true); - } catch (CsvValidationException ignored) { - // shouldn't happen - } - } - } - - private static void storeWorkerResults(Resource taskRes, Path file, Model data) throws IOException, NoSuchElementException { - Metric[] workerMetrics = MetricManager.getMetrics().stream().filter(x -> WorkerMetric.class.isAssignableFrom(x.getClass())).toArray(Metric[]::new); - - SelectBuilder sb = new SelectBuilder(); - sb.addWhere(taskRes, IPROP.workerResult, "?worker"); - queryProperties(sb, "?worker", IPROP.workerID, IPROP.workerType, IPROP.noOfQueries, IPROP.timeOut); - queryMetrics(sb, "?worker", workerMetrics); - - executeAndStoreQuery(sb, file, data); - } - - private static CSVWriter getCSVWriter(Path file) throws IOException { - return (CSVWriter) new CSVWriterBuilder(new FileWriter(file.toAbsolutePath().toString(), true)) - .withQuoteChar('\"') - .withSeparator(',') - .withLineEnd("\n") - .build(); - } - - private static void queryProperties(SelectBuilder sb, String variable, Property... properties) { - for (Property prop : properties) { - sb.addVar(prop.getLocalName()).addWhere(variable, prop, "?" + prop.getLocalName()); - } - } - - private static void queryMetrics(SelectBuilder sb, String variable, Metric[] metrics) { - for (Metric m : metrics) { - sb.addVar(m.getAbbreviation()).addWhere(variable, IPROP.createMetricProperty(m), "?" + m.getAbbreviation()); - } - } - - private static void executeAndStoreQuery(SelectBuilder sb, Path file, Model data) throws IOException { - try(QueryExecution exec = QueryExecutionFactory.create(sb.build(), data); - FileOutputStream fos = new FileOutputStream(file.toFile())) { - ResultSet results = exec.execSelect(); - ResultSetFormatter.outputAsCSV(fos, results); - } - } -} diff --git a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/storage/impl/NTFileStorage.java b/src/main/java/org/aksw/iguana/cc/tasks/stresstest/storage/impl/NTFileStorage.java deleted file mode 100644 index cd94e3e15..000000000 --- a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/storage/impl/NTFileStorage.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * - */ -package org.aksw.iguana.cc.tasks.stresstest.storage.impl; - -import org.aksw.iguana.cc.tasks.stresstest.storage.TripleBasedStorage; -import org.aksw.iguana.commons.annotation.Shorthand; -import org.apache.jena.rdf.model.Model; -import org.apache.jena.riot.RDFDataMgr; -import org.apache.jena.riot.RDFFormat; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.util.Calendar; - -/** - * - * Will save results as NTriple File either using the provided name or the a generated one. - * - * @author f.conrads - * - */ -@Shorthand("NTFileStorage") -public class NTFileStorage extends TripleBasedStorage { - - private static final Logger LOGGER = LoggerFactory.getLogger(NTFileStorage.class); - - private final StringBuilder file; - - /** - * Uses a generated file called results_{DD}-{MM}-{YYYY}_{HH}-{mm}.nt - */ - public NTFileStorage() { - Calendar now = Calendar.getInstance(); - - this.file = new StringBuilder(); - file.append("results_") - .append( - String.format("%d-%02d-%02d_%02d-%02d.%03d", - now.get(Calendar.YEAR), - now.get(Calendar.MONTH) + 1, - now.get(Calendar.DAY_OF_MONTH), - now.get(Calendar.HOUR_OF_DAY), - now.get(Calendar.MINUTE), - now.get(Calendar.MILLISECOND) - ) - ) - .append(".nt"); - } - - /** - * Uses the provided filename - * - * @param fileName - */ - public NTFileStorage(String fileName) { - this.file = new StringBuilder(fileName); - } - - @Override - public void storeResult(Model data) { - super.storeResult(data); - try (OutputStream os = new FileOutputStream(file.toString(), true)) { - RDFDataMgr.write(os, metricResults, RDFFormat.NTRIPLES); - metricResults.removeAll(); - } catch (IOException e) { - LOGGER.error("Could not commit to NTFileStorage.", e); - } - } - - @Override - public String toString() { - return this.getClass().getSimpleName(); - } - - public String getFileName() { - return this.file.toString(); - } -} - diff --git a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/storage/impl/RDFFileStorage.java b/src/main/java/org/aksw/iguana/cc/tasks/stresstest/storage/impl/RDFFileStorage.java deleted file mode 100644 index 10ee67817..000000000 --- a/src/main/java/org/aksw/iguana/cc/tasks/stresstest/storage/impl/RDFFileStorage.java +++ /dev/null @@ -1,75 +0,0 @@ -package org.aksw.iguana.cc.tasks.stresstest.storage.impl; - -import org.aksw.iguana.cc.tasks.stresstest.storage.TripleBasedStorage; -import org.aksw.iguana.commons.annotation.Shorthand; -import org.apache.jena.rdf.model.Model; -import org.apache.jena.riot.Lang; -import org.apache.jena.riot.RDFDataMgr; -import org.apache.jena.riot.RDFLanguages; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.util.Calendar; - -@Shorthand("RDFFileStorage") -public class RDFFileStorage extends TripleBasedStorage { - - private static final Logger LOGGER = LoggerFactory.getLogger(RDFFileStorage.class.getName()); - - private Lang lang = Lang.TTL; - private final StringBuilder file; - - /** - * Uses a generated file called results_{DD}-{MM}-{YYYY}_{HH}-{mm}.ttl - */ - public RDFFileStorage() { - Calendar now = Calendar.getInstance(); - - this.file = new StringBuilder(); - file.append("results_") - .append( - String.format("%d-%02d-%02d_%02d-%02d.%03d", - now.get(Calendar.YEAR), - now.get(Calendar.MONTH) + 1, - now.get(Calendar.DAY_OF_MONTH), - now.get(Calendar.HOUR_OF_DAY), - now.get(Calendar.MINUTE), - now.get(Calendar.MILLISECOND) - ) - ) - .append(".ttl"); - } - - /** - * Uses the provided filename - * @param fileName - */ - public RDFFileStorage(String fileName){ - this.file = new StringBuilder(fileName); - this.lang= RDFLanguages.filenameToLang(fileName, Lang.TTL); - } - - @Override - public void storeResult(Model data){ - super.storeResult(data); - try (OutputStream os = new FileOutputStream(file.toString(), true)) { - RDFDataMgr.write(os, metricResults, this.lang); - metricResults.removeAll(); - } catch (IOException e) { - LOGGER.error("Could not commit to RDFFileStorage using lang: "+lang, e); - } - } - - - @Override - public String toString(){ - return this.getClass().getSimpleName(); - } - - public String getFileName(){ - return this.file.toString(); - } -} diff --git a/src/main/java/org/aksw/iguana/cc/utils/CLIProcessManager.java b/src/main/java/org/aksw/iguana/cc/utils/CLIProcessManager.java deleted file mode 100644 index 42a8bdd61..000000000 --- a/src/main/java/org/aksw/iguana/cc/utils/CLIProcessManager.java +++ /dev/null @@ -1,137 +0,0 @@ -package org.aksw.iguana.cc.utils; - -import org.apache.commons.lang.SystemUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.*; -import java.util.ArrayList; -import java.util.List; - -/** - * CLI Utils class - */ -public class CLIProcessManager { - - private static final Logger LOGGER = LoggerFactory.getLogger(CLIProcessManager.class); - - /** - * Creates a process - * @param command - * @return - */ - public static Process createProcess(String command) { - ProcessBuilder processBuilder = new ProcessBuilder(); - processBuilder.redirectErrorStream(true); - - Process process = null; - try { - if (SystemUtils.IS_OS_LINUX) { - - processBuilder.command("bash", "-c", command); - - } else if (SystemUtils.IS_OS_WINDOWS) { - processBuilder.command("cmd.exe", "-c", command); - } - process = processBuilder.start(); - - } catch (IOException e) { - LOGGER.error("New process could not be created: {}", e); - } - - return process; - } - - /** - * Destroys a process forcibly - * @param process - */ - public static void destroyProcess(Process process) { - process.destroyForcibly(); - } - - /** - * Short handler for destroyProcess and createProcess - * @param process - * @param command - * @return - */ - public static Process destroyAndCreateNewProcess(Process process, String command) { - destroyProcess(process); - return createProcess(command); - } - - /** - * Create n processes of the same command - * @param n the amount of processes created - * @param command the command to create the process with - * @return - */ - public static List createProcesses(int n, String command) { - List processList = new ArrayList<>(5); - for (int i = 0; i < n; i++) { - processList.add(createProcess(command)); - } - - return processList; - } - - /** - * Count and returns the no. of lines of one process until a certain string appears, - * @param process - * @param successString the string of the process after the no of line should be returned - * @param errorString the error string, will throw an IOException if this appeared. - * @return - * @throws IOException - */ - public static long countLinesUntilStringOccurs(Process process, String successString, String errorString) throws IOException { - String line; - LOGGER.debug("Will look for: {} or as error: {}",successString, errorString); - StringBuilder output = new StringBuilder(); - - long size = -1; - BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); - - try { - while ((line = reader.readLine()) != null) { - if (line.contains(errorString)) { - LOGGER.debug("Found error"); - LOGGER.debug("Query finished with {}", errorString); - - throw new IOException(line); - } else if (line.contains(successString)) { - LOGGER.debug("Query finished with {}", successString); - break; - } - - // Only save first 1000 lines of the output - if (size < 1000) { - output.append(line).append("\n"); - } - size++; - } - - } catch (IOException e) { - LOGGER.debug("Exception in reading the output of the process. ", e); - throw e; - } - - return size; - } - - public static void executeCommand(Process process, String command) throws IOException { - BufferedWriter output = new BufferedWriter(new OutputStreamWriter(process.getOutputStream())); - output.write(command + "\n"); - output.flush(); - } - - /** - * Checks if the process input stream is ready to be read. - * @param process - * @return - * @throws IOException - */ - public static boolean isReaderReady(Process process) throws IOException { - return new BufferedReader(new InputStreamReader(process.getInputStream())).ready(); - } -} diff --git a/src/main/java/org/aksw/iguana/cc/utils/FileUtils.java b/src/main/java/org/aksw/iguana/cc/utils/FileUtils.java index 0cdaec81c..745150f12 100644 --- a/src/main/java/org/aksw/iguana/cc/utils/FileUtils.java +++ b/src/main/java/org/aksw/iguana/cc/utils/FileUtils.java @@ -3,7 +3,7 @@ import java.io.*; import java.nio.charset.StandardCharsets; import java.nio.file.Files; -import java.nio.file.Paths; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; @@ -15,7 +15,7 @@ */ public class FileUtils { - public static int getHashcodeFromFileContent(String filepath) { + public static int getHashcodeFromFileContent(Path filepath) { int hashcode; try { String fileContents = readFile(filepath); @@ -26,9 +26,8 @@ public static int getHashcodeFromFileContent(String filepath) { return hashcode; } - public static String readFile(String path) throws IOException { - byte[] encoded = Files.readAllBytes(Paths.get(path)); - return new String(encoded, StandardCharsets.UTF_8); + public static String readFile(Path path) throws IOException { + return Files.readString(path, StandardCharsets.UTF_8); } /** @@ -46,10 +45,8 @@ public static String readFile(String path) throws IOException { * @return the line ending used in the given file * @throws IOException */ - public static String getLineEnding(String filepath) throws IOException { - try(FileInputStream fis = new FileInputStream(filepath); - InputStreamReader isr = new InputStreamReader(fis, StandardCharsets.UTF_8); - BufferedReader br = new BufferedReader(isr)) { + public static String getLineEnding(Path filepath) throws IOException { + try(BufferedReader br = Files.newBufferedReader(filepath)) { char c; while ((c = (char) br.read()) != (char) -1) { if (c == '\n') diff --git a/src/main/java/org/aksw/iguana/cc/utils/IndexedQueryReader.java b/src/main/java/org/aksw/iguana/cc/utils/IndexedQueryReader.java index 38691061e..b89ee7ae6 100644 --- a/src/main/java/org/aksw/iguana/cc/utils/IndexedQueryReader.java +++ b/src/main/java/org/aksw/iguana/cc/utils/IndexedQueryReader.java @@ -1,7 +1,16 @@ package org.aksw.iguana.cc.utils; +import org.apache.commons.io.input.AutoCloseInputStream; +import org.apache.commons.io.input.BoundedInputStream; + import java.io.*; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.*; import java.util.stream.Collectors; @@ -18,22 +27,24 @@ public class IndexedQueryReader { /** * This list stores the start position and the length of each indexed content. */ - private List indices; + private final List indices; - /** The file whose content should be indexed. */ - private final File file; + /** + * The file whose content should be indexed. + */ + private final Path path; /** * Indexes each content in between two of the given separators (including the beginning and end of the file). The * given separator isn't allowed to be empty. * - * @param filepath path to the file + * @param filepath path to the file * @param separator the separator line that is used in the file (isn't allowed to be empty) * @return reader to access the indexed content * @throws IllegalArgumentException the given separator was empty * @throws IOException */ - public static IndexedQueryReader makeWithStringSeparator(String filepath, String separator) throws IOException { + public static IndexedQueryReader makeWithStringSeparator(Path filepath, String separator) throws IOException { if (separator.isEmpty()) throw new IllegalArgumentException("Separator for makeWithStringSeparator can not be empty."); return new IndexedQueryReader(filepath, separator); @@ -48,7 +59,7 @@ public static IndexedQueryReader makeWithStringSeparator(String filepath, String * @return reader to access the indexed content * @throws IOException */ - public static IndexedQueryReader makeWithEmptyLines(String filepath) throws IOException { + public static IndexedQueryReader makeWithEmptyLines(Path filepath) throws IOException { String lineEnding = FileUtils.getLineEnding(filepath); return new IndexedQueryReader(filepath, lineEnding + lineEnding); } @@ -60,7 +71,7 @@ public static IndexedQueryReader makeWithEmptyLines(String filepath) throws IOEx * @return reader to access the indexed lines * @throws IOException */ - public static IndexedQueryReader make(String filepath) throws IOException { + public static IndexedQueryReader make(Path filepath) throws IOException { return new IndexedQueryReader(filepath, FileUtils.getLineEnding(filepath)); } @@ -68,13 +79,13 @@ public static IndexedQueryReader make(String filepath) throws IOException { * Creates an object that indexes each content in between two of the given separators (including the beginning and * end of the given file).
* - * @param filepath path to the file + * @param filepath path to the file * @param separator the separator for each query * @throws IOException */ - private IndexedQueryReader(String filepath, String separator) throws IOException { - this.file = new File(filepath); - this.indexFile(separator); + private IndexedQueryReader(Path filepath, String separator) throws IOException { + path = filepath; + indices = indexFile(path, separator); } /** @@ -86,14 +97,22 @@ private IndexedQueryReader(String filepath, String separator) throws IOException */ public String readQuery(int index) throws IOException { // Indexed queries can't be larger than ~2GB - byte[] data = new byte[Math.toIntExact(this.indices.get(index)[1])]; - String output; - try (RandomAccessFile raf = new RandomAccessFile(this.file, "r")) { - raf.seek(this.indices.get(index)[0]); - raf.read(data); - output = new String(data, StandardCharsets.UTF_8); + try (FileChannel channel = FileChannel.open(path, StandardOpenOption.READ)) { + final ByteBuffer buffer = ByteBuffer.allocate((int) indices.get(index)[1]); + final var read = channel.read(buffer, indices.get(index)[0]); + assert read == indices.get(index)[1]; + return new String(buffer.array(), StandardCharsets.UTF_8); } - return output; + } + + public InputStream streamQuery(int index) throws IOException { + return new AutoCloseInputStream( + new BufferedInputStream( + new BoundedInputStream( + Channels.newInputStream( + FileChannel.open(path, StandardOpenOption.READ) + .position(this.indices.get(index)[0] /* offset */)), + this.indices.get(index)[1] /* length */))); } /** @@ -124,12 +143,13 @@ public int size() { * separators too. * * @param separator the custom separator + * @return the Indexes * @throws IOException */ - private void indexFile(String separator) throws IOException { - try (FileInputStream fi = new FileInputStream(file); + private static List indexFile(Path filepath, String separator) throws IOException { + try (InputStream fi = Files.newInputStream(filepath, StandardOpenOption.READ); BufferedInputStream bis = new BufferedInputStream(fi)) { - this.indices = FileUtils.indexStream(separator,bis) + return FileUtils.indexStream(separator, bis) .stream().filter((long[] e) -> e[1] > 0 /* Only elements with length > 0 */).collect(Collectors.toList()); } } diff --git a/src/main/java/org/aksw/iguana/cc/utils/ResultSizeRetriever.java b/src/main/java/org/aksw/iguana/cc/utils/ResultSizeRetriever.java deleted file mode 100644 index 097843606..000000000 --- a/src/main/java/org/aksw/iguana/cc/utils/ResultSizeRetriever.java +++ /dev/null @@ -1,49 +0,0 @@ -package org.aksw.iguana.cc.utils; - -import org.apache.jena.query.QueryExecution; -import org.apache.jena.query.QueryExecutionFactory; -import org.apache.jena.query.ResultSetFormatter; - -import java.io.BufferedReader; -import java.io.FileReader; -import java.io.PrintWriter; - -/** - * Util class to retrieve the resultsize of a queryfile and an sparql endpoint. - */ -public class ResultSizeRetriever { - - public static void main(String[] args) { - if(args.length!=3) { - System.out.println("resretriever.sh http://endpoint queryfile.sparql outputfile.tsv"); - return; - } - int i=0; - try(BufferedReader reader = new BufferedReader(new FileReader(args[1]));PrintWriter pw = new PrintWriter(args[2])){ - String line; - while((line=reader.readLine())!=null) { - if(line.isEmpty()) { - continue; - } - try { - pw.println(i+"\t"+retrieveSize(args[0], line)); - }catch(Exception e) { - pw.println(i+"\t?"); - e.printStackTrace(); - } - System.out.println(i+" done"); - i++; - } - - }catch(Exception e) { - e.printStackTrace(); - } - } - - - public static int retrieveSize(String endpoint, String query) { - QueryExecution exec = QueryExecutionFactory.sparqlService(endpoint, query); - return ResultSetFormatter.consume(exec.execSelect()); - } - -} diff --git a/src/main/java/org/aksw/iguana/cc/utils/SPARQLQueryStatistics.java b/src/main/java/org/aksw/iguana/cc/utils/SPARQLQueryStatistics.java deleted file mode 100644 index 14b58f8af..000000000 --- a/src/main/java/org/aksw/iguana/cc/utils/SPARQLQueryStatistics.java +++ /dev/null @@ -1,49 +0,0 @@ -package org.aksw.iguana.cc.utils; - -import org.apache.jena.query.Query; -import org.apache.jena.sparql.syntax.ElementWalker; - -/** - * Simple SPARQL Query statistics - */ -public class SPARQLQueryStatistics { - - public int aggr=0; - public int filter=0; - public int optional=0; - public int union=0; - public int having=0; - public int groupBy=0; - public int offset=0; - public double size=0.0; - public int orderBy=0; - public int triples=0; - - - /** - * Will add the stats of the provided query to this statistics count. - * @param q - */ - public void getStatistics(Query q) { - if(q.isSelectType()) { - - size++; - offset+=q.hasOffset()?1:0; - aggr+=q.hasAggregators()?1:0; - groupBy+=q.hasGroupBy()?1:0; - having+=q.hasHaving()?1:0; - orderBy+=q.hasOrderBy()?1:0; - - StatisticsVisitor visitor = new StatisticsVisitor(); - visitor.setElementWhere(q.getQueryPattern()); - ElementWalker.walk(q.getQueryPattern(), visitor); - - union+=visitor.union?1:0; - optional+=visitor.optional?1:0; - filter+=visitor.filter?1:0; - triples += visitor.bgps; - - } - } - -} diff --git a/src/main/java/org/aksw/iguana/cc/utils/StatisticsVisitor.java b/src/main/java/org/aksw/iguana/cc/utils/StatisticsVisitor.java deleted file mode 100644 index c1d033b0b..000000000 --- a/src/main/java/org/aksw/iguana/cc/utils/StatisticsVisitor.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.aksw.iguana.cc.utils; - -import org.apache.jena.sparql.syntax.*; - - -/** - * Simple visitor to check if simple statistics of a SPARQL Query appeared. - */ -public class StatisticsVisitor extends RecursiveElementVisitor{ - - public boolean filter; - public boolean regexFilter=false; - public boolean cmpFilter=false; - public boolean union; - public boolean optional; - private boolean started; - private Element where; - public int bgps; - - public StatisticsVisitor() { - super(new ElementVisitorBase()); - } - - public void startElement(ElementGroup el) { - if (!started && el.equals(where)) { - // root element found - started = true; - - } - } - - public void setElementWhere(Element el) { - this.where = el; - } - - public void endElement(ElementPathBlock el) { - - if (started) { - bgps+=el.getPattern().getList().size(); - } - - } - - public void startElement(ElementFilter el) {this.filter=true;el.getExpr();} - public void startElement(ElementUnion el) {this.union=true;} - public void startElement(ElementOptional el) {this.optional=true;} - -} diff --git a/src/main/java/org/aksw/iguana/cc/worker/AbstractWorker.java b/src/main/java/org/aksw/iguana/cc/worker/AbstractWorker.java deleted file mode 100644 index 9ffed99c7..000000000 --- a/src/main/java/org/aksw/iguana/cc/worker/AbstractWorker.java +++ /dev/null @@ -1,280 +0,0 @@ -package org.aksw.iguana.cc.worker; - -import org.aksw.iguana.cc.config.elements.ConnectionConfig; -import org.aksw.iguana.cc.model.QueryExecutionStats; -import org.aksw.iguana.cc.query.handler.QueryHandler; -import org.aksw.iguana.commons.annotation.Nullable; -import org.aksw.iguana.commons.annotation.Shorthand; -import org.apache.http.HttpHost; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.AuthCache; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.client.protocol.HttpClientContext; -import org.apache.http.impl.auth.BasicScheme; -import org.apache.http.impl.client.BasicAuthCache; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.apache.http.protocol.HttpContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.time.Instant; -import java.util.*; - - -/** - * The Abstract Worker which will implement the runnable, the main loop, the - * time to wait before a query and will send the results to the ResultProcessor - * module
- * so the Implemented Workers only need to implement which query to test next - * and how to test this query. - * - * @author f.conrads - */ -public abstract class AbstractWorker implements Worker { - protected static final Logger LOGGER = LoggerFactory.getLogger(AbstractWorker.class); - - protected String taskID; - - /** - * The unique ID of the worker, should be from 0 to n - */ - protected Integer workerID; - - /** - * The worker Type. f.e. SPARQL or UPDATE or SQL or whatever - * Determined by the Shorthand of the class, if no Shorthand is provided the class name is used. - * The workerType is only used in logging messages. - */ - protected String workerType; - protected ConnectionConfig connection; - protected Map queries; - - protected Double timeLimit; - protected Double timeOut = 180000D; - protected Integer fixedLatency = 0; - protected Integer gaussianLatency = 0; - - protected boolean endSignal = false; - protected long executedQueries; - protected Instant startTime; - protected ConnectionConfig con; - protected int queryHash; - protected QueryHandler queryHandler; - protected Collection results = new LinkedList<>(); - private Random latencyRandomizer; - private Long endAtNOQM = null; - - public AbstractWorker(String taskID, Integer workerID, ConnectionConfig connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency) { - this.taskID = taskID; - this.workerID = workerID; - this.con = connection; - - handleTimeParams(timeLimit, timeOut, fixedLatency, gaussianLatency); - setWorkerType(); - - this.queryHandler = new QueryHandler(queries, this.workerID); - - LOGGER.debug("Initialized new Worker[{{}} : {{}}] for taskID {{}}", this.workerType, workerID, taskID); - } - - - @Override - public void waitTimeMs() { - double wait = this.fixedLatency.doubleValue(); - double gaussian = this.latencyRandomizer.nextDouble(); - wait += (gaussian * 2) * this.gaussianLatency; - LOGGER.debug("Worker[{} : {}]: Time to wait for next Query {}", this.workerType, this.workerID, wait); - try { - if (wait > 0) - Thread.sleep((int) wait); - } catch (InterruptedException e) { - LOGGER.error("Worker[{{}} : {}]: Could not wait time before next query due to: {}", this.workerType, this.workerID, e); - } - } - - /** - * This will start the worker. It will get the next query, wait as long as it - * should wait before executing the next query, then it will test the query and - * send it if not aborted yet to the ResultProcessor Module - */ - public void startWorker() { - // For Update and Logging purpose get startTime of Worker - this.startTime = Instant.now(); - - this.queryHash = this.queryHandler.hashCode(); - - LOGGER.info("Starting Worker[{{}} : {{}}].", this.workerType, this.workerID); - // Execute Queries as long as the Stresstest will need. - while (!this.endSignal && !hasExecutedNoOfQueryMixes(this.endAtNOQM)) { - // Get next query - StringBuilder query = new StringBuilder(); - StringBuilder queryID = new StringBuilder(); - try { - getNextQuery(query, queryID); - // check if endsignal was triggered - if (this.endSignal) { - break; - } - } catch (IOException e) { - LOGGER.error( - "Worker[{{}} : {{}}] : Something went terrible wrong in getting the next query. Worker will be shut down.", - this.workerType, this.workerID); - LOGGER.error("Error which occured:_", e); - break; - } - // Simulate Network Delay (or whatever should be simulated) - waitTimeMs(); - - // benchmark query - try { - executeQuery(query.toString(), queryID.toString()); - } catch (Exception e) { - LOGGER.error("Worker[{{}} : {{}}] : ERROR with query: {{}}", this.workerType, this.workerID, query); - } - //this.executedQueries++; - } - LOGGER.info("Stopping Worker[{{}} : {{}}].", this.workerType, this.workerID); - } - - @Override - public void getNextQuery(StringBuilder query, StringBuilder queryID) throws IOException { - this.queryHandler.getNextQuery(query, queryID); - } - - protected HttpContext getAuthContext(String endpoint) { - HttpClientContext context = HttpClientContext.create(); - - if (this.con.getPassword() != null && this.con.getUser() != null && !this.con.getPassword().isEmpty() && !this.con.getUser().isEmpty()) { - CredentialsProvider provider = new BasicCredentialsProvider(); - - provider.setCredentials(new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT), - new UsernamePasswordCredentials(this.con.getUser(), this.con.getPassword())); - - //create target host - String targetHost = endpoint; - try { - URI uri = new URI(endpoint); - targetHost = uri.getScheme() + "://" + uri.getHost() + ":" + uri.getPort(); - } catch (URISyntaxException e) { - e.printStackTrace(); - } - //set Auth cache - AuthCache authCache = new BasicAuthCache(); - BasicScheme basicAuth = new BasicScheme(); - authCache.put(HttpHost.create(targetHost), basicAuth); - - context.setCredentialsProvider(provider); - context.setAuthCache(authCache); - - } - return context; - } - - public synchronized void addResults(QueryExecutionStats results) { - // TODO: check if statement for bugs, if the if line exists in the UpdateWorker, the UpdateWorker fails its tests - if (!this.endSignal && !hasExecutedNoOfQueryMixes(this.endAtNOQM)) { - this.results.add(results); - this.executedQueries++; - - // - if (getNoOfQueries() > 0 && getExecutedQueries() % getNoOfQueries() == 0) { - LOGGER.info("Worker executed {} queryMixes", getExecutedQueries() * 1.0 / getNoOfQueries()); - } - } - } - - @Override - public synchronized Collection popQueryResults() { - if (this.results.isEmpty()) { - return null; - } - Collection ret = this.results; - this.results = new LinkedList<>(); - return ret; - } - - @Override - public long getExecutedQueries() { - return this.executedQueries; - } - - @Override - public void stopSending() { - this.endSignal = true; - LOGGER.debug("Worker[{{}} : {{}}] got stop signal.", this.workerType, this.workerID); - } - - @Override - public boolean isTerminated() { - return this.endSignal; - } - - - @Override - public void run() { - startWorker(); - } - - @Override - public long getNoOfQueries() { - return this.queryHandler.getQueryCount(); - } - - @Override - public boolean hasExecutedNoOfQueryMixes(Long noOfQueryMixes) { - if (noOfQueryMixes == null) { - return false; - } - return getExecutedQueries() / (getNoOfQueries() * 1.0) >= noOfQueryMixes; - } - - @Override - public void endAtNoOfQueryMixes(Long noOfQueryMixes) { - this.endAtNOQM = noOfQueryMixes; - } - - @Override - public QueryHandler getQueryHandler() { - return this.queryHandler; - } - - private void handleTimeParams(Integer timeLimit, Integer timeOut, Integer fixedLatency, Integer gaussianLatency) { - if (timeLimit != null) { - this.timeLimit = timeLimit.doubleValue(); - } - if (timeOut != null) { - this.timeOut = timeOut.doubleValue(); - } - if (fixedLatency != null) { - this.fixedLatency = fixedLatency; - } - if (gaussianLatency != null) { - this.gaussianLatency = gaussianLatency; - } - this.latencyRandomizer = new Random(this.workerID); - } - - private void setWorkerType() { - if (this.getClass().getAnnotation(Shorthand.class) != null) { - this.workerType = this.getClass().getAnnotation(Shorthand.class).value(); - } else { - this.workerType = this.getClass().getName(); - } - } - - @Override - public WorkerMetadata getMetadata() { - return new WorkerMetadata( - this.workerID, - this.workerType, - this.timeOut, - this.queryHandler.getQueryCount(), - this.queryHandler.hashCode(), - this.queryHandler.getAllQueryIds() - ); - } -} diff --git a/src/main/java/org/aksw/iguana/cc/worker/HttpWorker.java b/src/main/java/org/aksw/iguana/cc/worker/HttpWorker.java new file mode 100644 index 000000000..6e0242d9a --- /dev/null +++ b/src/main/java/org/aksw/iguana/cc/worker/HttpWorker.java @@ -0,0 +1,161 @@ +package org.aksw.iguana.cc.worker; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.aksw.iguana.cc.config.elements.ConnectionConfig; +import org.aksw.iguana.cc.query.handler.QueryHandler; +import org.aksw.iguana.cc.query.selector.QuerySelector; +import org.aksw.iguana.cc.tasks.impl.Stresstest; +import org.aksw.iguana.cc.worker.impl.SPARQLProtocolWorker; + +import java.net.http.HttpTimeoutException; +import java.time.Duration; +import java.time.Instant; +import java.time.ZonedDateTime; +import java.util.Base64; +import java.util.List; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; + +/** + * Interface for the Worker Thread used in the {@link Stresstest} + */ +public abstract class HttpWorker { + + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type") + @JsonSubTypes({ + @JsonSubTypes.Type(value = SPARQLProtocolWorker.Config.class, name = "SPARQLProtocolWorker"), + }) + public interface Config { + CompletionTarget completionTarget(); + + String acceptHeader(); + + /** + * Returns the number of workers with this configuration that will be started. + * + * @return the number of workers + */ + Integer number(); + + /** + * Determines whether the results should be parsed based on the acceptHeader. + * + * @return true if the results should be parsed, false otherwise + */ + Boolean parseResults(); + + QueryHandler queries(); + + ConnectionConfig connection(); + + Duration timeout(); + } + + public record ExecutionStats( + int queryID, + Instant startTime, + Duration duration, // should always exist + Optional httpStatusCode, + OptionalLong contentLength, + OptionalLong responseBodyHash, + Optional error + ) { + public enum END_STATE { + SUCCESS(0), + TIMEOUT(110), // ETIMEDOUT - Connection timed out + HTTP_ERROR(111), // ECONNREFUSED - Connection refused + MISCELLANEOUS_EXCEPTION(1); + + public final int value; + END_STATE(int value) { + this.value = value; + } + } + + public END_STATE endState() { + if (successful()) { + return END_STATE.SUCCESS; + } else if (timeout()) { + return END_STATE.TIMEOUT; + } else if (httpError()) { + return END_STATE.HTTP_ERROR; + } else { + return END_STATE.MISCELLANEOUS_EXCEPTION; + } + } + + public boolean completed() { + return httpStatusCode().isPresent(); + } + + public boolean successful() { + return error.isEmpty() && httpStatusCode.orElse(0) / 100 == 2; + } + + public boolean timeout() { + boolean timeout = false; + if (!successful() && error().isPresent()) { + timeout |= error().get() instanceof java.util.concurrent.TimeoutException; + if (error().get() instanceof ExecutionException exec) { + timeout = exec.getCause() instanceof HttpTimeoutException; + } + } + return timeout; + } + + public boolean httpError() { + if (httpStatusCode.isEmpty()) + return false; + return httpStatusCode().orElse(0) / 100 != 2; + } + + public boolean miscellaneousException() { + return error().isPresent() && !timeout() && !httpError(); + } + } + + public record Result(long workerID, List executionStats, ZonedDateTime startTime, ZonedDateTime endTime) {} + + @JsonTypeInfo(use = JsonTypeInfo.Id.DEDUCTION) + @JsonSubTypes({ + @JsonSubTypes.Type(value = TimeLimit.class), + @JsonSubTypes.Type(value = QueryMixes.class) + }) + sealed public interface CompletionTarget permits TimeLimit, QueryMixes {} + + public record TimeLimit(@JsonProperty(required = true) Duration duration) implements CompletionTarget {} + + public record QueryMixes(@JsonProperty(required = true) int number) implements CompletionTarget {} + + final protected long workerID; + final protected Config config; + final protected ResponseBodyProcessor responseBodyProcessor; + final protected QuerySelector querySelector; + + public HttpWorker(long workerID, ResponseBodyProcessor responseBodyProcessor, Config config) { + this.workerID = workerID; + this.responseBodyProcessor = responseBodyProcessor; + this.config = config; + this.querySelector = this.config.queries().getQuerySelectorInstance(); + } + + public static String basicAuth(String username, String password) { + return "Basic " + Base64.getEncoder().encodeToString((username + ":" + password).getBytes()); + } + + public abstract CompletableFuture start(); + + public Config config() { + return this.config; + } + + public long getWorkerID() { + return this.workerID; + } +} diff --git a/src/main/java/org/aksw/iguana/cc/worker/LatencyStrategy.java b/src/main/java/org/aksw/iguana/cc/worker/LatencyStrategy.java deleted file mode 100644 index dfcca2a14..000000000 --- a/src/main/java/org/aksw/iguana/cc/worker/LatencyStrategy.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.aksw.iguana.cc.worker; - -/** - * The Strategy Names to simulate different network latency behaviors - * - * @author f.conrads - * - */ -public enum LatencyStrategy { - /** - * No Latency should be simulated - */ - NONE, - - /** - * A fixed time/ms should be waited between queries (time is the latency base value) - */ - FIXED, - - /** - * The time/ms should be calculated randomly each time - * out of a gaussian intervall based on the latency base value as follows - * - * [0;2*latencyBaseValue] - */ - VARIABLE -} diff --git a/src/main/java/org/aksw/iguana/cc/worker/ResponseBodyProcessor.java b/src/main/java/org/aksw/iguana/cc/worker/ResponseBodyProcessor.java new file mode 100644 index 000000000..3cd6e56a2 --- /dev/null +++ b/src/main/java/org/aksw/iguana/cc/worker/ResponseBodyProcessor.java @@ -0,0 +1,82 @@ +package org.aksw.iguana.cc.worker; + +import org.aksw.iguana.cc.lang.LanguageProcessor; +import org.aksw.iguana.commons.io.BigByteArrayInputStream; +import org.aksw.iguana.commons.io.BigByteArrayOutputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.text.MessageFormat; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.*; + +public class ResponseBodyProcessor { + public record Config(String contentType, Integer threads) { + public Config(String contentType, Integer threads) { + this.contentType = contentType; + this.threads = threads == null ? 1 : threads; + } + } + + public record Key(long contentLength, long xxh64) {} + + public ResponseBodyProcessor(Config config) { + this.executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(config.threads == null ? 1 : config.threads); + this.languageProcessor = LanguageProcessor.getInstance(config.contentType); + } + + public ResponseBodyProcessor(String contentType) { + this(new Config(contentType, null)); + } + + private static final Logger LOGGER = LoggerFactory.getLogger(ResponseBodyProcessor.class); + + private final ConcurrentHashMap.KeySetView seenResponseBodies = ConcurrentHashMap.newKeySet(); + + private final List responseDataMetrics = Collections.synchronizedList(new ArrayList<>()); + private final LanguageProcessor languageProcessor; + + private final ThreadPoolExecutor executor; + + public boolean add(long contentLength, long xxh64, BigByteArrayOutputStream bbaos) { + final var key = new Key(contentLength, xxh64); + if (seenResponseBodies.add(key)) { + submit(key, bbaos); + return true; + } + return false; + } + + private void submit(Key key, BigByteArrayOutputStream bigByteArrayOutputStream) { + executor.execute(() -> { + var processingResult = languageProcessor.process(new BigByteArrayInputStream(bigByteArrayOutputStream), key.xxh64); + responseDataMetrics.add(processingResult); + }); + } + + public List getResponseDataMetrics() { + if (executor.isTerminated()) { + return responseDataMetrics; + } + + final var timeout = Duration.ofMinutes(10); + LOGGER.info(MessageFormat.format("Shutting down ResponseBodyProcessor with {0}min timeout to finish processing. {1} tasks remaining.", timeout.toMinutes(), executor.getQueue().size())); + boolean noTimeout; + try { + executor.shutdown(); + noTimeout = executor.awaitTermination(10, TimeUnit.MINUTES); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + if (noTimeout) LOGGER.info("ResponseBodyProcessor completed."); + else LOGGER.warn("ResponseBodyProcessor timed out."); + return responseDataMetrics; + } + + public LanguageProcessor getLanguageProcessor() { + return this.languageProcessor; + } +} diff --git a/src/main/java/org/aksw/iguana/cc/worker/ResponseBodyProcessorInstances.java b/src/main/java/org/aksw/iguana/cc/worker/ResponseBodyProcessorInstances.java new file mode 100644 index 000000000..95be1f0e9 --- /dev/null +++ b/src/main/java/org/aksw/iguana/cc/worker/ResponseBodyProcessorInstances.java @@ -0,0 +1,44 @@ +package org.aksw.iguana.cc.worker; + +import org.aksw.iguana.cc.lang.LanguageProcessor; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +public class ResponseBodyProcessorInstances { + final private Map processors = new HashMap<>(); + + public ResponseBodyProcessorInstances() {} + + public ResponseBodyProcessorInstances(List configs) { + if (configs == null) return; + for (var config : configs) { + processors.put(config.contentType(), new ResponseBodyProcessor(config)); + } + } + + public ResponseBodyProcessor getProcessor(String contentType) { + if (!processors.containsKey(contentType)) { + processors.put(contentType, new ResponseBodyProcessor(contentType)); + } + return processors.get(contentType); + } + + /** + * Returns a Supplier that returns the results of all ResponseBodyProcessors. A supplier is used for data + * abstraction. + * + * @return supplier for all results + */ + public Supplier>> getResults() { + return () -> { // TODO: consider removing the languageProcessor as the key, it's only used right now for creating strings for naming + Map> out = new HashMap<>(); + for (var processor : processors.values()) { + out.put(processor.getLanguageProcessor(), processor.getResponseDataMetrics()); + } + return out; + }; + } +} diff --git a/src/main/java/org/aksw/iguana/cc/worker/Worker.java b/src/main/java/org/aksw/iguana/cc/worker/Worker.java deleted file mode 100644 index fb7076895..000000000 --- a/src/main/java/org/aksw/iguana/cc/worker/Worker.java +++ /dev/null @@ -1,99 +0,0 @@ -package org.aksw.iguana.cc.worker; - -import org.aksw.iguana.cc.model.QueryExecutionStats; -import org.aksw.iguana.cc.query.handler.QueryHandler; -import org.aksw.iguana.cc.tasks.stresstest.Stresstest; - -import java.io.IOException; -import java.util.Collection; - -/** - * Interface for the Worker Thread used in the {@link Stresstest} - * - * @author f.conrads - * - */ -public interface Worker extends Runnable{ - - - /** - * This method executes a query and adds the results to the Result Processor for proper result and metric calculations. - * Note: Some of the Worker implementations employ background threads to process the result of the query. - * Due to this, this method does not return anything and each implementation of this method must also add the - * results to Result Processor within this method. This can be done by calling AbstractWorker.addResults(QueryExecutionStats) - * - * @param query The query which should be executed - * @param queryID the ID of the query which should be executed - */ - void executeQuery(String query, String queryID); - - /** - * This method saves the next query in the queryStr StringBuilder and - * the query id in the queryID. - * - * @param queryStr The query should be stored in here! - * @param queryID The queryID should be stored in here! - * @throws IOException - */ - void getNextQuery(StringBuilder queryStr, StringBuilder queryID) throws IOException; - - /** - * This method will return the query handler which is used by this worker - * - * @return QueryHandler which is used by this worker - */ - QueryHandler getQueryHandler(); - - - /** - * This should stop the next sending process. - * If an execution started before this method was called, but answered after, it should not be counted! - */ - void stopSending(); - - /** - * This will simulate the Time in ms to wait before testing the next query. - * It can be used to simulate network delay. - */ - void waitTimeMs(); - - - /** - * This will return the amount of executed queries so far - * - * @return no. of executed queries - */ - long getExecutedQueries(); - - /** - * Get and remove all internal stored results of finished queries - * - * @return list of Properties to send to RabbitMQ - */ - Collection popQueryResults(); - - boolean isTerminated(); - - /** - * Returns the no of queries in the queryset of the worker - * @return no of queries in the queryset - */ - long getNoOfQueries(); - - /** - * Returns if the no of query mixes were already executed - * @param noOfQueryMixes no of query mixes - * @return true if the no of query mixes were already executed - */ - boolean hasExecutedNoOfQueryMixes(Long noOfQueryMixes); - - - /** - * Sets the end restriction - * - * @param noOfQueryMixes after which the worker should stop - */ - void endAtNoOfQueryMixes(Long noOfQueryMixes); - - WorkerMetadata getMetadata(); -} diff --git a/src/main/java/org/aksw/iguana/cc/worker/WorkerFactory.java b/src/main/java/org/aksw/iguana/cc/worker/WorkerFactory.java deleted file mode 100644 index 401cf5fdb..000000000 --- a/src/main/java/org/aksw/iguana/cc/worker/WorkerFactory.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.aksw.iguana.cc.worker; - -import org.aksw.iguana.commons.factory.TypedFactory; - -/** - * Factory to create a {@link Worker} - * - * @author f.conrads - * - */ -public class WorkerFactory extends TypedFactory{ - - -} diff --git a/src/main/java/org/aksw/iguana/cc/worker/WorkerMetadata.java b/src/main/java/org/aksw/iguana/cc/worker/WorkerMetadata.java deleted file mode 100644 index 678306aba..000000000 --- a/src/main/java/org/aksw/iguana/cc/worker/WorkerMetadata.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.aksw.iguana.cc.worker; - -public record WorkerMetadata( - int workerID, - String workerType, - double timeout, - int numberOfQueries, - int queryHash, - String[] queryIDs -) {} diff --git a/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputFileWorker.java b/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputFileWorker.java deleted file mode 100644 index 5660a73d6..000000000 --- a/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputFileWorker.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.aksw.iguana.cc.worker.impl; - -import org.aksw.iguana.cc.config.elements.ConnectionConfig; -import org.aksw.iguana.commons.annotation.Nullable; -import org.aksw.iguana.commons.annotation.Shorthand; - -import java.io.File; -import java.io.IOException; -import java.io.PrintWriter; -import java.util.Map; - -/** - * Worker to execute a query against a CLI process, the connection.service will be the command to execute the query against. - *

- * Assumes that the CLI process won't stop but will just accepts queries one after another and returns the results in the CLI output. - * also assumes that the query has to be read from a file instead of plain input - *

- * This worker can be set to be created multiple times in the background if one process will throw an error, a backup process was already created and can be used. - * This is handy if the process won't just prints an error message, but simply exits. - */ -@Shorthand("CLIInputFileWorker") -public class CLIInputFileWorker extends MultipleCLIInputWorker { - private final String dir; - - public CLIInputFileWorker(String taskID, Integer workerID, ConnectionConfig connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, String initFinished, String queryFinished, String queryError, @Nullable Integer numberOfProcesses, String directory) { - super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency, initFinished, queryFinished, queryError, numberOfProcesses); - this.dir = directory; - } - - @Override - protected String writableQuery(String query) { - File f; - - try { - new File(this.dir).mkdirs(); - f = new File(this.dir + File.separator + "tmpquery.sparql"); - f.createNewFile(); - f.deleteOnExit(); - try (PrintWriter pw = new PrintWriter(f)) { - pw.print(query); - } - return f.getName(); - } catch (IOException e) { - e.printStackTrace(); - } - - return query; - } - -} diff --git a/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputPrefixWorker.java b/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputPrefixWorker.java deleted file mode 100644 index e21120219..000000000 --- a/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputPrefixWorker.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.aksw.iguana.cc.worker.impl; - - -import org.aksw.iguana.cc.config.elements.ConnectionConfig; -import org.aksw.iguana.commons.annotation.Nullable; -import org.aksw.iguana.commons.annotation.Shorthand; - -import java.util.Map; - -/** - * Worker to execute a query against a CLI process, the connection.service will be the command to execute the query against. - *

- * Assumes that the CLI process won't stop but will just accepts queries one after another and returns the results in the CLI output. - * also assumes that the query has to be prefixed and suffixed. - * For example: SPARQL SELECT * {?s ?p ?o} ; whereas 'SPARQL' is the prefix and ';' is the suffix. - *

- * This worker can be set to be created multiple times in the background if one process will throw an error, a backup process was already created and can be used. - * This is handy if the process won't just prints an error message, but simply exits. - */ -@Shorthand("CLIInputPrefixWorker") -public class CLIInputPrefixWorker extends MultipleCLIInputWorker { - - private final String prefix; - private final String suffix; - - public CLIInputPrefixWorker(String taskID, Integer workerID, ConnectionConfig connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, String initFinished, String queryFinished, String queryError, @Nullable Integer numberOfProcesses, String queryPrefix, String querySuffix) { - super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency, initFinished, queryFinished, queryError, numberOfProcesses); - this.prefix = queryPrefix; - this.suffix = querySuffix; - } - - @Override - protected String writableQuery(String query) { - return this.prefix + " " + query + " " + this.suffix; - } - -} diff --git a/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputWorker.java b/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputWorker.java deleted file mode 100644 index 70b30acbb..000000000 --- a/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputWorker.java +++ /dev/null @@ -1,124 +0,0 @@ -package org.aksw.iguana.cc.worker.impl; - -import org.aksw.iguana.cc.config.elements.ConnectionConfig; -import org.aksw.iguana.cc.model.QueryExecutionStats; -import org.aksw.iguana.cc.utils.CLIProcessManager; -import org.aksw.iguana.cc.worker.AbstractWorker; -import org.aksw.iguana.commons.annotation.Nullable; -import org.aksw.iguana.commons.annotation.Shorthand; -import org.aksw.iguana.commons.constants.COMMON; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.time.Instant; -import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; - -import static org.aksw.iguana.commons.time.TimeUtils.durationInMilliseconds; - -/** - * Worker to execute a query against a CLI process, the connection.service will be the command to execute the query against. - *

- * Assumes that the CLI process won't stop but will just accepts queries one after another and returns the results in the CLI output. - */ -@Shorthand("CLIInputWorker") -public class CLIInputWorker extends AbstractWorker { - - private final Logger LOGGER = LoggerFactory.getLogger(getClass()); - private final String initFinished; - private final String queryFinished; - private final String error; - private Process process; - - public CLIInputWorker(String taskID, Integer workerID, ConnectionConfig connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, String initFinished, String queryFinished, String queryError) { - super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency); - this.initFinished = initFinished; - this.queryFinished = queryFinished; - this.error = queryError; - this.setWorkerProperties(); - } - - private void setWorkerProperties() { - // Create a CLI process, initialize it - this.process = CLIProcessManager.createProcess(this.con.getEndpoint()); - try { - CLIProcessManager.countLinesUntilStringOccurs(process, this.initFinished, this.error); //Init - } catch (IOException e) { - LOGGER.error("Exception while trying to wait for init of CLI Process", e); - } - } - - @Override - public void executeQuery(String query, String queryID) { - Instant start = Instant.now(); - - try { - // Create background thread that will watch the output of the process and prepare results - AtomicLong size = new AtomicLong(-1); - AtomicBoolean failed = new AtomicBoolean(false); - ExecutorService executor = Executors.newSingleThreadExecutor(); - executor.execute(() -> { - try { - LOGGER.debug("Process Alive: {}", this.process.isAlive()); - LOGGER.debug("Reader ready: {}", CLIProcessManager.isReaderReady(this.process)); - size.set(CLIProcessManager.countLinesUntilStringOccurs(this.process, this.queryFinished, this.error)); - } catch (IOException e) { - failed.set(true); - } - }); - - // Execute the query on the process - try { - if (this.process.isAlive()) { - CLIProcessManager.executeCommand(this.process, writableQuery(query)); - } else if (this.endSignal) { - super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, durationInMilliseconds(start, Instant.now()))); - return; - } else { - super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, durationInMilliseconds(start, Instant.now()))); - return; - } - } finally { - executor.shutdown(); - executor.awaitTermination((long) (double) this.timeOut, TimeUnit.MILLISECONDS); - } - - // At this point, query is executed and background thread has processed the results. - // Next, calculate time for benchmark. - double duration = durationInMilliseconds(start, Instant.now()); - - if (duration >= this.timeOut) { - super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_SOCKET_TIMEOUT, duration)); - return; - } else if (failed.get()) { - super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, duration)); - return; - } - - // SUCCESS - LOGGER.debug("Query successfully executed size: {}", size.get()); - super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_SUCCESS, duration, size.get())); - } catch (IOException | InterruptedException e) { - LOGGER.warn("Exception while executing query ", e); - // ERROR - super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, durationInMilliseconds(start, Instant.now()))); - } - } - - - protected String writableQuery(String query) { - return query; - } - - - @Override - public void stopSending() { - super.stopSending(); - this.process.destroyForcibly(); - } -} diff --git a/src/main/java/org/aksw/iguana/cc/worker/impl/CLIWorker.java b/src/main/java/org/aksw/iguana/cc/worker/impl/CLIWorker.java deleted file mode 100644 index fb5b0fd1a..000000000 --- a/src/main/java/org/aksw/iguana/cc/worker/impl/CLIWorker.java +++ /dev/null @@ -1,98 +0,0 @@ -package org.aksw.iguana.cc.worker.impl; - -import org.aksw.iguana.cc.config.elements.ConnectionConfig; -import org.aksw.iguana.cc.model.QueryExecutionStats; -import org.aksw.iguana.cc.worker.AbstractWorker; -import org.aksw.iguana.commons.annotation.Nullable; -import org.aksw.iguana.commons.annotation.Shorthand; -import org.aksw.iguana.commons.constants.COMMON; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.time.Instant; -import java.util.Map; - -import static org.aksw.iguana.commons.time.TimeUtils.durationInMilliseconds; - -/** - * Worker to execute a query again a CLI process, the connection.service will be the command to execute the query against. - *

- * command may look like the following

- * cliprocess.sh $QUERY$ $USER$ $PASSWORD$ - *
- * whereas $QUERY$ will be exchanged with the actual query as well as user and password. - * Further on it is possible to encode the query using $ENCODEDQUERY$ instead of $QUERY$ - */ -@Shorthand("CLIWorker") -public class CLIWorker extends AbstractWorker { - - private final Logger LOGGER = LoggerFactory.getLogger(getClass()); - - public CLIWorker(String taskID, Integer workerID, ConnectionConfig connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency) { - super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency); - } - - @Override - public void executeQuery(String query, String queryID) { - Instant start = Instant.now(); - // use cli as service - String encodedQuery = URLEncoder.encode(query, StandardCharsets.UTF_8); - String queryCLI = getReplacedQuery(query, encodedQuery); - // execute queryCLI and read response - ProcessBuilder processBuilder = new ProcessBuilder().redirectErrorStream(true); - processBuilder.command("bash", "-c", queryCLI); - try { - - Process process = processBuilder.start(); - - long size = -1; - - try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { - - // -1 as the first line should be the header - while (reader.readLine() != null) { - size++; - } - } catch (Exception e) { - e.printStackTrace(); - } - int exitVal = process.waitFor(); - if (exitVal == 0) { - LOGGER.debug("Query successfully executed size: {}", size); - } else { - LOGGER.warn("Exit Value of Process was not 0, was {} ", exitVal); - super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, durationInMilliseconds(start, Instant.now()))); - return; - } - super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_SUCCESS, durationInMilliseconds(start, Instant.now()), size)); - return; - } catch (Exception e) { - LOGGER.warn("Unknown Exception while executing query", e); - } - // ERROR - super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, durationInMilliseconds(start, Instant.now()))); - } - - private String getReplacedQuery(String query, String encodedQuery) { - String queryCLI = this.con.getEndpoint().replace("$QUERY$", query); - queryCLI = queryCLI.replace("$ENCODEDQUERY$", encodedQuery); - - if (this.con.getUser() != null) { - queryCLI = queryCLI.replace("$USER$", this.con.getUser()); - } else { - queryCLI = queryCLI.replace("$USER$", ""); - - } - if (this.con.getPassword() != null) { - queryCLI = queryCLI.replace("$PASSWORD$", this.con.getPassword()); - } else { - queryCLI = queryCLI.replace("$PASSWORD$", ""); - - } - return queryCLI; - } -} diff --git a/src/main/java/org/aksw/iguana/cc/worker/impl/HttpGetWorker.java b/src/main/java/org/aksw/iguana/cc/worker/impl/HttpGetWorker.java deleted file mode 100644 index 857d0395c..000000000 --- a/src/main/java/org/aksw/iguana/cc/worker/impl/HttpGetWorker.java +++ /dev/null @@ -1,57 +0,0 @@ -package org.aksw.iguana.cc.worker.impl; - -import org.aksw.iguana.cc.config.elements.ConnectionConfig; -import org.aksw.iguana.commons.annotation.Nullable; -import org.aksw.iguana.commons.annotation.Shorthand; -import org.apache.http.HttpHeaders; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.HttpGet; - -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.util.Map; - - -/** - * HTTP Get Worker. - * Uses HTTP Get to execute a Query.

- * if the parameter type was not set it will use 'query' as the parameter as default, otherwise it will use the provided parameter - */ -@Shorthand("HttpGetWorker") -public class HttpGetWorker extends HttpWorker { - - protected String parameter = "query"; - - protected String responseType = null; - - public HttpGetWorker(String taskID, Integer workerID, ConnectionConfig connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, @Nullable String parameterName, @Nullable String responseType) { - super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency); - - if (parameterName != null) { - this.parameter = parameterName; - } - if (responseType != null) { - this.responseType = responseType; - } - } - - void buildRequest(String query, String queryID) { - String qEncoded = URLEncoder.encode(query, StandardCharsets.UTF_8); - String addChar = "?"; - if (this.con.getEndpoint().contains("?")) { - addChar = "&"; - } - String url = this.con.getEndpoint() + addChar + this.parameter + "=" + qEncoded; - this.request = new HttpGet(url); - RequestConfig requestConfig = - RequestConfig.custom() - .setSocketTimeout(this.timeOut.intValue()) - .setConnectTimeout(this.timeOut.intValue()) - .build(); - - if (this.responseType != null) - this.request.setHeader(HttpHeaders.ACCEPT, this.responseType); - - this.request.setConfig(requestConfig); - } -} diff --git a/src/main/java/org/aksw/iguana/cc/worker/impl/HttpPostWorker.java b/src/main/java/org/aksw/iguana/cc/worker/impl/HttpPostWorker.java deleted file mode 100644 index c6f884dac..000000000 --- a/src/main/java/org/aksw/iguana/cc/worker/impl/HttpPostWorker.java +++ /dev/null @@ -1,61 +0,0 @@ -package org.aksw.iguana.cc.worker.impl; - -import org.aksw.iguana.cc.config.elements.ConnectionConfig; -import org.aksw.iguana.commons.annotation.Nullable; -import org.aksw.iguana.commons.annotation.Shorthand; -import org.apache.http.HttpHeaders; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.StringEntity; - -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.util.Map; - -/** - * HTTP Post worker. - * Uses HTTP posts to execute a query. - *

- * Sends the query in plain as POST data if parameter type was not set, otherwise uses json as follows:
- * {PARAMETER: QUERY} - */ -@Shorthand("HttpPostWorker") -public class HttpPostWorker extends HttpGetWorker { - - private String contentType = "text/plain"; - - public HttpPostWorker(String taskID, Integer workerID, ConnectionConfig connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, @Nullable String parameterName, @Nullable String responseType, @Nullable String contentType) { - super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency, parameterName, responseType); - - if (parameterName == null) { - this.parameter = null; - } - if (contentType != null) { - this.contentType = contentType; - } - } - - void buildRequest(String query, String queryID) { - StringBuilder data = new StringBuilder(); - if (parameter != null) { - String qEncoded = URLEncoder.encode(query, StandardCharsets.UTF_8); - data.append("{ \"").append(parameter).append("\": \"").append(qEncoded).append("\"}"); - } else { - data.append(query); - } - StringEntity entity = new StringEntity(data.toString(), StandardCharsets.UTF_8); - request = new HttpPost(con.getUpdateEndpoint()); - ((HttpPost) request).setEntity(entity); - request.setHeader("Content-Type", contentType); - RequestConfig requestConfig = RequestConfig.custom() - .setSocketTimeout(timeOut.intValue()) - .setConnectTimeout(timeOut.intValue()) - .build(); - - if (this.responseType != null) - request.setHeader(HttpHeaders.ACCEPT, this.responseType); - - request.setConfig(requestConfig); - } - -} diff --git a/src/main/java/org/aksw/iguana/cc/worker/impl/HttpWorker.java b/src/main/java/org/aksw/iguana/cc/worker/impl/HttpWorker.java deleted file mode 100644 index d9151232b..000000000 --- a/src/main/java/org/aksw/iguana/cc/worker/impl/HttpWorker.java +++ /dev/null @@ -1,296 +0,0 @@ -package org.aksw.iguana.cc.worker.impl; - -import org.aksw.iguana.cc.config.elements.ConnectionConfig; -import org.aksw.iguana.cc.model.QueryExecutionStats; -import org.aksw.iguana.cc.model.QueryResultHashKey; -import org.aksw.iguana.cc.worker.AbstractWorker; -import org.aksw.iguana.commons.annotation.Nullable; -import org.aksw.iguana.commons.constants.COMMON; -import org.apache.http.Header; -import org.apache.http.HttpEntity; -import org.apache.http.client.ClientProtocolException; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpRequestBase; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.impl.conn.BasicHttpClientConnectionManager; -import org.apache.http.message.BasicHeader; -import org.json.simple.parser.ParseException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.xml.sax.SAXException; - -import javax.xml.parsers.ParserConfigurationException; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.UnsupportedEncodingException; -import java.time.Instant; -import java.util.Map; -import java.util.concurrent.*; - -import static org.aksw.iguana.commons.time.TimeUtils.durationInMilliseconds; - -/** - * Abstract HTTP worker - */ -public abstract class HttpWorker extends AbstractWorker { - - - protected final ExecutorService resultProcessorService = Executors.newFixedThreadPool(5); - protected ScheduledThreadPoolExecutor timeoutExecutorPool = new ScheduledThreadPoolExecutor(1); - protected ConcurrentMap processedResults = new ConcurrentHashMap<>(); - protected CloseableHttpClient client; - protected HttpRequestBase request; - protected ScheduledFuture abortCurrentRequestFuture; - protected CloseableHttpResponse response; - protected boolean resultsSaved = false; - protected boolean requestTimedOut = false; - protected String queryId; - protected Instant requestStartTime; - protected long tmpExecutedQueries = 0; - - public HttpWorker(String taskID, Integer workerID, ConnectionConfig connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency) { - super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency); - this.timeoutExecutorPool.setRemoveOnCancelPolicy(true); - } - - public ConcurrentMap getProcessedResults() { - return this.processedResults; - } - - protected void setTimeout(int timeOut) { - assert (this.request != null); - this.abortCurrentRequestFuture = this.timeoutExecutorPool.schedule( - () -> { - synchronized (this) { - this.request.abort(); - this.requestTimedOut = true; - } - }, - timeOut, TimeUnit.MILLISECONDS); - } - - protected void abortTimeout() { - if (!this.abortCurrentRequestFuture.isDone()) { - this.abortCurrentRequestFuture.cancel(false); - } - } - - - @Override - public void stopSending() { - super.stopSending(); - abortTimeout(); - try { - if (this.request != null && !this.request.isAborted()) { - this.request.abort(); - } - } catch (Exception ignored) { - } - closeClient(); - shutdownResultProcessor(); - } - - - public void shutdownResultProcessor() { - this.resultProcessorService.shutdown(); - try { - boolean finished = this.resultProcessorService.awaitTermination(3000, TimeUnit.MILLISECONDS); - if (!finished) { - LOGGER.error("Result Processor could be shutdown orderly. Terminating."); - this.resultProcessorService.shutdownNow(); - } - } catch (InterruptedException e) { - LOGGER.error("Could not shut down http result processor: " + e.getLocalizedMessage()); - } - - try { - boolean finished = this.timeoutExecutorPool.awaitTermination(3000, TimeUnit.MILLISECONDS); - if (!finished) { - LOGGER.error("Timeout Executor could be shutdown orderly. Terminating."); - this.timeoutExecutorPool.shutdownNow(); - } - } catch (InterruptedException e) { - LOGGER.error("Could not shut down http timout executor: " + e.getLocalizedMessage()); - } - } - - synchronized protected void addResultsOnce(QueryExecutionStats queryExecutionStats) { - if (!this.resultsSaved) { - addResults(queryExecutionStats); - this.resultsSaved = true; - } - } - - @Override - public void executeQuery(String query, String queryID) { - this.queryId = queryID; - this.resultsSaved = false; - this.requestTimedOut = false; - - if (this.client == null) - initClient(); - - try { - buildRequest(query, this.queryId); - - setTimeout(this.timeOut.intValue()); - - this.requestStartTime = Instant.now(); - this.response = this.client.execute(this.request, getAuthContext(this.con.getEndpoint())); - // method to process the result in background - processHttpResponse(); - - abortTimeout(); - - } catch (ClientProtocolException e) { - handleException(query, COMMON.QUERY_HTTP_FAILURE, e); - } catch (IOException e) { - if (this.requestTimedOut) { - LOGGER.warn("Worker[{} : {}]: Reached timeout on query (ID {})\n{}", - this.workerType, this.workerID, this.queryId, query); - addResultsOnce(new QueryExecutionStats(this.queryId, COMMON.QUERY_SOCKET_TIMEOUT, this.timeOut)); - } else { - handleException(query, COMMON.QUERY_UNKNOWN_EXCEPTION, e); - } - } catch (Exception e) { - handleException(query, COMMON.QUERY_UNKNOWN_EXCEPTION, e); - } finally { - abortTimeout(); - closeResponse(); - } - } - - private void handleException(String query, Long cause, Exception e) { - double duration = durationInMilliseconds(this.requestStartTime, Instant.now()); - addResultsOnce(new QueryExecutionStats(this.queryId, cause, duration)); - LOGGER.warn("Worker[{} : {}]: {} on query (ID {})\n{}", - this.workerType, this.workerID, e.getMessage(), this.queryId, query); - closeClient(); - initClient(); - } - - protected void processHttpResponse() { - int responseCode = this.response.getStatusLine().getStatusCode(); - boolean responseCodeSuccess = responseCode >= 200 && responseCode < 300; - boolean responseCodeOK = responseCode == 200; - - if (responseCodeOK) { // response status is OK (200) - // get content type header - HttpEntity httpResponse = this.response.getEntity(); - Header contentTypeHeader = new BasicHeader(httpResponse.getContentType().getName(), httpResponse.getContentType().getValue()); - // get content stream - try (InputStream contentStream = httpResponse.getContent()) { - // read content stream with resultProcessor, return length, set string in StringBuilder. - ByteArrayOutputStream responseBody = new ByteArrayOutputStream(); - long length = this.queryHandler.getLanguageProcessor().readResponse(contentStream, responseBody); - this.tmpExecutedQueries++; - // check if such a result was already parsed and is cached - double duration = durationInMilliseconds(this.requestStartTime, Instant.now()); - synchronized (this) { - QueryResultHashKey resultCacheKey = new QueryResultHashKey(queryId, length); - if (this.processedResults.containsKey(resultCacheKey)) { - LOGGER.debug("found result cache key {} ", resultCacheKey); - Long preCalculatedResultSize = this.processedResults.get(resultCacheKey); - addResultsOnce(new QueryExecutionStats(this.queryId, COMMON.QUERY_SUCCESS, duration, preCalculatedResultSize)); - } else { - // otherwise: parse it. The parsing result is cached for the next time. - if (!this.endSignal) { - this.resultProcessorService.submit(new HttpResultProcessor(this, this.queryId, duration, contentTypeHeader, responseBody, length)); - this.resultsSaved = true; - } - } - } - - } catch (IOException | TimeoutException e) { - double duration = durationInMilliseconds(this.requestStartTime, Instant.now()); - addResultsOnce(new QueryExecutionStats(this.queryId, COMMON.QUERY_HTTP_FAILURE, duration)); - } - } else if (responseCodeSuccess) { // response status is succeeded (2xx) but not OK (200) - double duration = durationInMilliseconds(this.requestStartTime, Instant.now()); - addResultsOnce(new QueryExecutionStats(this.queryId, COMMON.QUERY_SUCCESS, duration, 0)); - } else { // response status indicates that the query did not succeed (!= 2xx) - double duration = durationInMilliseconds(this.requestStartTime, Instant.now()); - addResultsOnce(new QueryExecutionStats(this.queryId, COMMON.QUERY_HTTP_FAILURE, duration)); - } - } - - abstract void buildRequest(String query, String queryID) throws UnsupportedEncodingException; - - protected void initClient() { - this.client = HttpClients.custom().setConnectionManager(new BasicHttpClientConnectionManager()).build(); - } - - protected void closeClient() { - closeResponse(); - try { - if (this.client != null) - this.client.close(); - } catch (IOException e) { - LOGGER.error("Could not close http response ", e); - } - this.client = null; - } - - protected void closeResponse() { - try { - if (this.response != null) - this.response.close(); - } catch (IOException e) { - LOGGER.error("Could not close Client ", e); - } - this.response = null; - } - - /** - * Http Result Processor, analyzes the http response in the background, if it was cached already, what is the result size, - * did the response was a success or failure. - */ - static class HttpResultProcessor implements Runnable { - - private final Logger LOGGER = LoggerFactory.getLogger(getClass()); - - private final HttpWorker httpWorker; - private final String queryId; - private final double duration; - private final Header contentTypeHeader; - private final long contentLength; - private ByteArrayOutputStream contentStream; - - public HttpResultProcessor(HttpWorker httpWorker, String queryId, double duration, Header contentTypeHeader, ByteArrayOutputStream contentStream, long contentLength) { - this.httpWorker = httpWorker; - this.queryId = queryId; - this.duration = duration; - this.contentTypeHeader = contentTypeHeader; - this.contentStream = contentStream; - this.contentLength = contentLength; - } - - @Override - public void run() { - // Result size is not saved before. Process the http response. - - ConcurrentMap processedResults = this.httpWorker.getProcessedResults(); - QueryResultHashKey resultCacheKey = new QueryResultHashKey(this.queryId, this.contentLength); - try { - //String content = contentStream.toString(StandardCharsets.UTF_8.name()); - //contentStream = null; // might be hugh, dereference immediately after consumed - Long resultSize = this.httpWorker.queryHandler.getLanguageProcessor().getResultSize(this.contentTypeHeader, this.contentStream, this.contentLength); - this.contentStream = null; - // Save the result size to be re-used - processedResults.put(resultCacheKey, resultSize); - LOGGER.debug("added Result Cache Key {}", resultCacheKey); - - this.httpWorker.addResults(new QueryExecutionStats(this.queryId, COMMON.QUERY_SUCCESS, this.duration, resultSize)); - - } catch (IOException | ParseException | ParserConfigurationException | SAXException e) { - LOGGER.error("Query results could not be parsed. ", e); - this.httpWorker.addResults(new QueryExecutionStats(this.queryId, COMMON.QUERY_UNKNOWN_EXCEPTION, this.duration)); - } catch (Exception e) { - e.printStackTrace(); - } - } - } -} - diff --git a/src/main/java/org/aksw/iguana/cc/worker/impl/MultipleCLIInputWorker.java b/src/main/java/org/aksw/iguana/cc/worker/impl/MultipleCLIInputWorker.java deleted file mode 100644 index 355aa3767..000000000 --- a/src/main/java/org/aksw/iguana/cc/worker/impl/MultipleCLIInputWorker.java +++ /dev/null @@ -1,180 +0,0 @@ -package org.aksw.iguana.cc.worker.impl; - -import org.aksw.iguana.cc.config.elements.ConnectionConfig; -import org.aksw.iguana.cc.model.QueryExecutionStats; -import org.aksw.iguana.cc.utils.CLIProcessManager; -import org.aksw.iguana.cc.worker.AbstractWorker; -import org.aksw.iguana.commons.annotation.Nullable; -import org.aksw.iguana.commons.annotation.Shorthand; -import org.aksw.iguana.commons.constants.COMMON; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.time.Instant; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; - -import static org.aksw.iguana.commons.time.TimeUtils.durationInMilliseconds; - -/** - * Worker to execute a query against a CLI process, the connection.service will be the command to execute the query against. - *

- * Assumes that the CLI process won't stop but will just accepts queries one after another and returns the results in the CLI output. - *

- * This worker can be set to be created multiple times in the background if one process will throw an error, a backup process was already created and can be used. - * This is handy if the process won't just prints an error message, but simply exits. - */ -@Shorthand("MultipleCLIInputWorker") -public class MultipleCLIInputWorker extends AbstractWorker { - - private final Logger LOGGER = LoggerFactory.getLogger(getClass()); - private final String initFinished; - private final String queryFinished; - private final String error; - protected List processList; - protected int currentProcessId = 0; - protected int numberOfProcesses = 5; - private Process currentProcess; - - public MultipleCLIInputWorker(String taskID, Integer workerID, ConnectionConfig connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, String initFinished, String queryFinished, String queryError, @Nullable Integer numberOfProcesses) { - super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency); - this.initFinished = initFinished; - this.queryFinished = queryFinished; - this.error = queryError; - if (numberOfProcesses != null) { - this.numberOfProcesses = numberOfProcesses; - } - this.setWorkerProperties(); - } - - private void setWorkerProperties() { - // start cli input - - // Create processes, set first process as current process - this.processList = CLIProcessManager.createProcesses(this.numberOfProcesses, this.con.getEndpoint()); - this.currentProcess = this.processList.get(0); - - // Make sure that initialization is complete - for (Process value : this.processList) { - try { - CLIProcessManager.countLinesUntilStringOccurs(value, initFinished, error); - } catch (IOException e) { - LOGGER.error("Exception while trying to wait for init of CLI Process", e); - } - } - } - - - @Override - public void executeQuery(String query, String queryID) { - Instant start = Instant.now(); - // execute queryCLI and read response - try { - // Create background thread that will watch the output of the process and prepare results - AtomicLong size = new AtomicLong(-1); - AtomicBoolean failed = new AtomicBoolean(false); - ExecutorService executor = Executors.newSingleThreadExecutor(); - executor.execute(() -> { - try { - LOGGER.debug("Process Alive: {}", this.currentProcess.isAlive()); - LOGGER.debug("Reader ready: {}", CLIProcessManager.isReaderReady(this.currentProcess)); - size.set(CLIProcessManager.countLinesUntilStringOccurs(this.currentProcess, this.queryFinished, this.error)); - } catch (IOException e) { - failed.set(true); - } - }); - - // Execute the query on the process - try { - if (this.currentProcess.isAlive()) { - CLIProcessManager.executeCommand(this.currentProcess, writableQuery(query)); - } else if (this.endSignal) { - super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, durationInMilliseconds(start, Instant.now()))); - return; - } else { - setNextProcess(); - super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, durationInMilliseconds(start, Instant.now()))); - return; - } - } finally { - executor.shutdown(); - executor.awaitTermination((long) (double) this.timeOut, TimeUnit.MILLISECONDS); - } - - // At this point, query is executed and background thread has processed the results. - // Next, calculate time for benchmark. - double duration = durationInMilliseconds(start, Instant.now()); - - if (duration >= this.timeOut) { - setNextProcess(); - super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_SOCKET_TIMEOUT, duration)); - return; - } else if (failed.get()) { - if (!this.currentProcess.isAlive()) { - setNextProcess(); - } - super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, duration)); - return; - } - - // SUCCESS - LOGGER.debug("Query successfully executed size: {}", size.get()); - super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_SUCCESS, duration, size.get())); - } catch (IOException | InterruptedException e) { - LOGGER.warn("Exception while executing query ", e); - // ERROR - super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, durationInMilliseconds(start, Instant.now()))); - } - } - - private void setNextProcess() { - int oldProcessId = this.currentProcessId; - this.currentProcessId = this.currentProcessId == this.processList.size() - 1 ? 0 : this.currentProcessId + 1; - - // destroy old process - CLIProcessManager.destroyProcess(this.currentProcess); - if (oldProcessId == this.currentProcessId) { - try { - this.currentProcess.waitFor(); - } catch (InterruptedException e) { - LOGGER.error("Process was Interrupted", e); - } - } - - // create and initialize new process to replace previously destroyed process - Process replacementProcess = CLIProcessManager.createProcess(this.con.getEndpoint()); - try { - CLIProcessManager.countLinesUntilStringOccurs(replacementProcess, this.initFinished, this.error); // Init - this.processList.set(oldProcessId, replacementProcess); - } catch (IOException e) { - LOGGER.error("Process replacement didn't work", e); - } - - // finally, update current process - this.currentProcess = this.processList.get(this.currentProcessId); - } - - protected String writableQuery(String query) { - return query; - } - - - @Override - public void stopSending() { - super.stopSending(); - for (Process pr : this.processList) { - pr.destroyForcibly(); - try { - pr.waitFor(); - } catch (InterruptedException e) { - LOGGER.error("Process waitFor was Interrupted", e); - } - } - } -} diff --git a/src/main/java/org/aksw/iguana/cc/worker/impl/SPARQLProtocolWorker.java b/src/main/java/org/aksw/iguana/cc/worker/impl/SPARQLProtocolWorker.java new file mode 100644 index 000000000..4d43350b2 --- /dev/null +++ b/src/main/java/org/aksw/iguana/cc/worker/impl/SPARQLProtocolWorker.java @@ -0,0 +1,445 @@ +package org.aksw.iguana.cc.worker.impl; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonValue; +import net.jpountz.xxhash.XXHashFactory; +import org.aksw.iguana.cc.config.elements.ConnectionConfig; +import org.aksw.iguana.cc.query.handler.QueryHandler; +import org.aksw.iguana.cc.worker.ResponseBodyProcessor; +import org.aksw.iguana.cc.worker.HttpWorker; +import org.aksw.iguana.commons.io.BigByteArrayOutputStream; +import org.apache.http.client.utils.URIBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.helpers.MessageFormatter; + +import java.io.IOException; +import java.io.InputStream; +import java.net.*; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.time.ZonedDateTime; +import java.util.*; +import java.util.concurrent.*; +import java.util.function.BiFunction; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +public class SPARQLProtocolWorker extends HttpWorker { + + public final static class RequestFactory { + public enum RequestType { + GET_QUERY("get query"), + POST_URL_ENC_QUERY("post url-enc query"), + POST_QUERY("post query"), + POST_URL_ENC_UPDATE("post url-enc update"), + POST_UPDATE("post update"); + + private final String value; + + @JsonCreator + RequestType(String value) { + this.value = Objects.requireNonNullElse(value, "get query"); + } + + @JsonValue + public String value() { + return value; + } + } + + private final RequestType requestType; + + public RequestFactory(RequestType requestType) { + this.requestType = requestType; + } + + private static String urlEncode(List parameters) { + return parameters.stream() + .map(e -> e[0] + "=" + URLEncoder.encode(e[1], StandardCharsets.UTF_8)) + .collect(Collectors.joining("&")); + } + + public HttpRequest buildHttpRequest(InputStream queryStream, + Duration timeout, + ConnectionConfig connection, + String requestHeader) throws URISyntaxException, IOException { + HttpRequest.Builder request = HttpRequest.newBuilder().timeout(timeout); + + class CustomStreamSupplier { + boolean used = false; // assume, that the stream will only be used again, if the first request failed, because of the client + public Supplier getStreamSupplier() { + if (!used) { + used = true; + return () -> queryStream; + } + else + return () -> null; + } + } + + if (requestHeader != null) + request.header("Accept", requestHeader); + if (connection.authentication() != null && connection.authentication().user() != null) + request.header("Authorization", + HttpWorker.basicAuth(connection.authentication().user(), + Optional.ofNullable(connection.authentication().password()).orElse(""))); + switch (this.requestType) { + case GET_QUERY -> { + request.uri(new URIBuilder(connection.endpoint()) + .setParameter("query", + new String(queryStream.readAllBytes(), StandardCharsets.UTF_8)) + .build()) + .GET(); + } + case POST_URL_ENC_QUERY -> { + request.uri(connection.endpoint()) + .header("Content-Type", "application/x-www-form-urlencoded") + .POST(HttpRequest.BodyPublishers.ofString( + urlEncode(Collections.singletonList( + new String[]{"query" /* query is already URL encoded */, + new String(queryStream.readAllBytes(), StandardCharsets.UTF_8)})))); + } + case POST_QUERY -> { + request.uri(connection.endpoint()) + .header("Content-Type", "application/sparql-query") + .POST(HttpRequest.BodyPublishers.ofInputStream(new CustomStreamSupplier().getStreamSupplier())); + } + case POST_URL_ENC_UPDATE -> { + request.uri(connection.endpoint()) + .header("Content-Type", "application/x-www-form-urlencoded") + .POST(HttpRequest.BodyPublishers.ofString( + urlEncode(Collections.singletonList( + new String[]{"update" /* query is already URL encoded */, + new String(queryStream.readAllBytes(), StandardCharsets.UTF_8)})))); + } + case POST_UPDATE -> { + request.uri(connection.endpoint()) + .header("Content-Type", "application/sparql-update") + .POST(HttpRequest.BodyPublishers.ofInputStream(new CustomStreamSupplier().getStreamSupplier())); + } + } + return request.build(); + } + } + + + public record Config( + Integer number, + QueryHandler queries, + CompletionTarget completionTarget, + ConnectionConfig connection, + Duration timeout, + String acceptHeader /* e.g. application/sparql-results+json */, + RequestFactory.RequestType requestType, + Boolean parseResults + ) implements HttpWorker.Config { + public Config(Integer number, + @JsonProperty(required = true) QueryHandler queries, + @JsonProperty(required = true) CompletionTarget completionTarget, + @JsonProperty(required = true) ConnectionConfig connection, + @JsonProperty(required = true) Duration timeout, + String acceptHeader, + RequestFactory.RequestType requestType, + Boolean parseResults) { + this.number = number == null ? 1 : number; + this.queries = queries; + this.completionTarget = completionTarget; + this.connection = connection; + this.timeout = timeout; + this.acceptHeader = acceptHeader; + this.requestType = requestType == null ? RequestFactory.RequestType.GET_QUERY : requestType; + this.parseResults = parseResults == null || parseResults; + } + } + + record HttpExecutionResult( + int queryID, + Optional> response, + Instant requestStart, + Duration duration, + Optional outputStream, + OptionalLong actualContentLength, + OptionalLong hash, + Optional exception + ) { + public boolean completed() { + return response.isPresent(); + } + + public boolean successful() { + if (response.isPresent() && exception.isEmpty()) + return (response.get().statusCode() / 100) == 2; + return false; + } + } + + + private HttpClient httpClient; + private final ThreadPoolExecutor executor; + + private final XXHashFactory hasherFactory = XXHashFactory.fastestJavaInstance(); + private final RequestFactory requestFactory; + + private final ResponseBodyProcessor responseBodyProcessor; + + // declared here, so it can be reused across multiple method calls + private BigByteArrayOutputStream responseBodybbaos = new BigByteArrayOutputStream(); + + // used to read the http response body + private final byte[] buffer = new byte[4096]; + + private final static Logger LOGGER = LoggerFactory.getLogger(SPARQLProtocolWorker.class); + + @Override + public Config config() { + return (SPARQLProtocolWorker.Config) config; + } + + + public SPARQLProtocolWorker(long workerId, ResponseBodyProcessor responseBodyProcessor, Config config) { + super(workerId, responseBodyProcessor, config); + this.responseBodyProcessor = responseBodyProcessor; + this.executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(1); + this.requestFactory = new RequestFactory(config().requestType()); + this.httpClient = buildHttpClient(); + } + + /** + * Starts the worker and returns a CompletableFuture, which will be completed, when the worker has finished the + * completion target. The CompletableFuture will contain a Result object, which contains the execution stats of the + * worker. The execution stats contain the execution time, the http status code, the content length and the hash of + * the response body. If the worker failed to execute a query, the execution stats will contain an exception. + * If the worker failed to execute a query, because of a set time limit in the worker configuration, the result + * of that execution will be discarded. + * + * @return the CompletableFuture the contains the results of the worker. + */ + public CompletableFuture start() { + return CompletableFuture.supplyAsync(() -> { + ZonedDateTime startTime = ZonedDateTime.now(); + List executionStats = new ArrayList<>(); + if (config().completionTarget() instanceof QueryMixes queryMixes) { + for (int i = 0; i < queryMixes.number(); i++) { + for (int j = 0; j < config().queries().getQueryCount(); j++) { + ExecutionStats execution = executeQuery(config().timeout(), false); + if (execution == null) throw new RuntimeException("Execution returned null at a place, where it should have never been null."); + logExecution(execution); + executionStats.add(execution); + } + LOGGER.info("{}\t:: Completed {} out of {} querymixes", this, i + 1, queryMixes.number()); + } + } else if (config().completionTarget() instanceof TimeLimit timeLimit) { + final Instant endTime = Instant.now().plus(timeLimit.duration()); + Instant now; + while ((now = Instant.now()).isBefore(endTime)) { + final Duration timeToEnd = Duration.between(now, endTime); + final boolean reducedTimeout = config().timeout().compareTo(timeToEnd) > 0; + final Duration thisQueryTimeOut = (reducedTimeout) ? timeToEnd : config().timeout(); + ExecutionStats execution = executeQuery(thisQueryTimeOut, reducedTimeout); + if (execution != null){ // If timeout is reduced, the execution result might be discarded if it failed and executeQuery returns null. + logExecution(execution); + executionStats.add(execution); + } + } + LOGGER.info("{}\t:: Reached time limit of {}.", this, timeLimit.duration()); + } + ZonedDateTime endTime = ZonedDateTime.now(); + return new Result(this.workerID, executionStats, startTime, endTime); + }, executor); + } + + /** + * Executes the next query given by the query selector from the query handler. If the execution fails and + * discardOnFailure is true, the execution will be discarded and null will be returned. If the execution fails and + * discardOnFailure is false, the execution statistic with the failed results will be returned. + * + * @param timeout the timeout for the execution + * @param discardOnFailure if true, this method will return null, if the execution fails + * @return the execution statistic of the execution + */ + private ExecutionStats executeQuery(Duration timeout, boolean discardOnFailure) { + HttpExecutionResult result = executeHttpRequest(timeout); + Optional statuscode = Optional.empty(); + if (result.response().isPresent()) + statuscode = Optional.of(result.response().get().statusCode()); + + if (result.successful() && this.config.parseResults()) { // 2xx + if (result.actualContentLength.isEmpty() || result.hash.isEmpty() || result.outputStream.isEmpty()) { + throw new RuntimeException("Response body is null, but execution was successful."); // This should never happen + } + + // process result + if (!responseBodyProcessor.add(result.actualContentLength().orElse(-1), result.hash().orElse(-1), result.outputStream().orElse(new BigByteArrayOutputStream()))) { + this.responseBodybbaos = result.outputStream().orElse(new BigByteArrayOutputStream()); + } else { + this.responseBodybbaos = new BigByteArrayOutputStream(); + } + } + + try { + this.responseBodybbaos.reset(); + } catch (IOException e) { + this.responseBodybbaos = new BigByteArrayOutputStream(); + } + + // This is not explicitly checking for a timeout, instead it just checks if the execution was successful or not. + // TODO: This might cause problems if the query actually fails before the timeout and discardOnFailure is true. + if (!result.successful() && discardOnFailure) { + LOGGER.debug("{}\t:: Discarded execution, because the time limit has been reached: [queryID={}]", this, result.queryID); + return null; + } + + return new ExecutionStats( + result.queryID(), + result.requestStart(), + result.duration(), + statuscode, + result.actualContentLength(), + result.hash, + result.exception() + ); + } + + + private HttpExecutionResult executeHttpRequest(Duration timeout) { + final QueryHandler.QueryStreamWrapper queryHandle; + try { + queryHandle = config().queries().getNextQueryStream(this.querySelector); + } catch (IOException e) { + return new HttpExecutionResult( + this.querySelector.getCurrentIndex(), + Optional.empty(), + Instant.now(), + Duration.ZERO, + Optional.empty(), + OptionalLong.empty(), + OptionalLong.empty(), + Optional.of(e) + ); + } + + final HttpRequest request; + + try { + request = requestFactory.buildHttpRequest( + queryHandle.queryInputStream(), + timeout, + config().connection(), + config().acceptHeader() + ); + } catch (IOException | URISyntaxException e) { + return new HttpExecutionResult( + queryHandle.index(), + Optional.empty(), + Instant.now(), + Duration.ZERO, + Optional.empty(), + OptionalLong.empty(), + OptionalLong.empty(), + Optional.of(e) + ); + } + + // check if the last execution task is stuck + if (this.httpClient.executor().isPresent() && ((ThreadPoolExecutor) this.httpClient.executor().get()).getActiveCount() != 0) { + // This might never cancel the task if the client that's connected to is broken. There also seems to be a + // bug where the httpClient never properly handles the interrupt from the shutdownNow method. + // See: https://bugs.openjdk.org/browse/JDK-8294047 + ((ThreadPoolExecutor) this.httpClient.executor().get()).shutdownNow(); + final var waitStart = Instant.now(); + try { + while (!((ThreadPoolExecutor) this.httpClient.executor().get()).awaitTermination(1, TimeUnit.SECONDS)) { + LOGGER.warn("{}\t:: [Thread-ID: {}]\t:: Waiting for the http client to shutdown. Elapsed time: {}", this, Thread.currentThread().getId(), Duration.between(waitStart, Instant.now())); + } + } catch (InterruptedException ignored) { + LOGGER.warn("{}\t:: Http client never shutdown. Continuing with the creation of a new http client.", this); + } + this.httpClient = buildHttpClient(); + } + + final Instant timeStamp = Instant.now(); + final var requestStart = System.nanoTime(); + BiFunction, Exception, HttpExecutionResult> createFailedResult = (response, e) -> new HttpExecutionResult( + queryHandle.index(), + Optional.ofNullable(response), + timeStamp, + Duration.ofNanos(System.nanoTime() - requestStart), + Optional.empty(), + OptionalLong.empty(), + OptionalLong.empty(), + Optional.ofNullable(e) + ); + + try { + return httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream()) + .thenApply(httpResponse -> { + try (final var bodyStream = httpResponse.body()) { + if (httpResponse.statusCode() / 100 == 2) { // Request was successful + OptionalLong contentLength = httpResponse.headers().firstValueAsLong("Content-Length"); + try (var hasher = hasherFactory.newStreamingHash64(0)) { + int readBytes; + while ((readBytes = bodyStream.readNBytes(this.buffer, 0, this.buffer.length)) != 0) { + if (Duration.between(Instant.now(), timeStamp.plus(timeout)).isNegative()) { + return createFailedResult.apply(httpResponse, new TimeoutException()); + } + hasher.update(this.buffer, 0, readBytes); + this.responseBodybbaos.write(this.buffer, 0, readBytes); + } + + if (contentLength.isPresent() && + (this.responseBodybbaos.size() < contentLength.getAsLong() || + this.responseBodybbaos.size() > contentLength.getAsLong())) { + return createFailedResult.apply(httpResponse, new ProtocolException("Content-Length header value doesn't match actual content length.")); + } + + return new HttpExecutionResult( + queryHandle.index(), + Optional.of(httpResponse), + timeStamp, + Duration.ofNanos(System.nanoTime() - requestStart), + Optional.of(this.responseBodybbaos), + OptionalLong.of(this.responseBodybbaos.size()), + OptionalLong.of(hasher.getValue()), + Optional.empty() + ); + } + } else { + return createFailedResult.apply(httpResponse, null); + } + } catch (IOException ex) { + return createFailedResult.apply(httpResponse, ex); + } + }).get(timeout.toNanos(), TimeUnit.NANOSECONDS); + } catch (CompletionException | InterruptedException | ExecutionException | TimeoutException e) { + return createFailedResult.apply(null, e); + } + } + + private HttpClient buildHttpClient() { + return HttpClient.newBuilder() + .executor(Executors.newFixedThreadPool(1)) + .followRedirects(HttpClient.Redirect.ALWAYS) + .connectTimeout(config().timeout()) + .build(); + } + + private void logExecution(ExecutionStats execution) { + switch (execution.endState()) { + case SUCCESS -> LOGGER.debug("{}\t:: Successfully executed query: [queryID={}].", this, execution.queryID()); + case TIMEOUT -> LOGGER.warn("{}\t:: Timeout during query execution: [queryID={}, duration={}].", this, execution.queryID(), execution.duration()); // TODO: look for a possibility to add the query string for better logging + case HTTP_ERROR -> LOGGER.warn("{}\t:: HTTP Error occurred during query execution: [queryID={}, httpError={}].", this, execution.queryID(), execution.httpStatusCode().orElse(-1)); + case MISCELLANEOUS_EXCEPTION -> LOGGER.warn("{}\t:: Miscellaneous exception occurred during query execution: [queryID={}, exception={}].", this, execution.queryID(), execution.error().orElse(null)); + } + } + + @Override + public String toString() { + return MessageFormatter.format("[{}-{}]", SPARQLProtocolWorker.class.getSimpleName(), this.workerID).getMessage(); + } +} diff --git a/src/main/java/org/aksw/iguana/cc/worker/impl/UPDATEWorker.java b/src/main/java/org/aksw/iguana/cc/worker/impl/UPDATEWorker.java deleted file mode 100644 index ece958b67..000000000 --- a/src/main/java/org/aksw/iguana/cc/worker/impl/UPDATEWorker.java +++ /dev/null @@ -1,110 +0,0 @@ -package org.aksw.iguana.cc.worker.impl; - -import org.aksw.iguana.cc.config.elements.ConnectionConfig; -import org.aksw.iguana.cc.model.QueryExecutionStats; -import org.aksw.iguana.cc.worker.impl.update.UpdateTimer; -import org.aksw.iguana.commons.annotation.Nullable; -import org.aksw.iguana.commons.annotation.Shorthand; - -import java.io.IOException; -import java.time.Instant; -import java.util.Map; - -import static org.aksw.iguana.commons.time.TimeUtils.durationInMilliseconds; - -/** - * A Worker using SPARQL Updates to create service request. - * - * @author f.conrads - */ -@Shorthand("UPDATEWorker") -public class UPDATEWorker extends HttpPostWorker { - private final String timerStrategy; - private UpdateTimer updateTimer = new UpdateTimer(); - - private int queryCount; - - public UPDATEWorker(String taskID, Integer workerID, ConnectionConfig connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, @Nullable String timerStrategy) { - super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency, null, null, "application/sparql-update"); - this.timerStrategy = timerStrategy; - } - - @Override - public void startWorker() { - setUpdateTimer(this.timerStrategy); - super.startWorker(); - } - - @Override - public void waitTimeMs() { - double wait = this.updateTimer.calculateTime(durationInMilliseconds(this.startTime, Instant.now()), this.tmpExecutedQueries); - LOGGER.debug("Worker[{{}} : {{}}]: Time to wait for next Query {{}}", this.workerType, this.workerID, wait); - try { - Thread.sleep((long) wait); - } catch (InterruptedException e) { - LOGGER.error("Worker[{{}} : {{}}]: Could not wait time before next query due to: {{}}", this.workerType, this.workerID, e); - LOGGER.error("", e); - } - super.waitTimeMs(); - } - - @Override - public synchronized void addResults(QueryExecutionStats result) { - this.results.add(result); - this.executedQueries++; - } - - @Override - public void getNextQuery(StringBuilder queryStr, StringBuilder queryID) throws IOException { - // If there is no more update send end signal, as there is nothing to do anymore - if (this.queryCount >= this.queryHandler.getQueryCount()) { - stopSending(); - return; - } - - this.queryHandler.getNextQuery(queryStr, queryID); - this.queryCount++; - } - - /** - * Sets Update Timer according to strategy - * - * @param strategyStr The String representation of a UpdateTimer.Strategy - */ - private void setUpdateTimer(String strategyStr) { - if (strategyStr == null) return; - UpdateTimer.Strategy strategy = UpdateTimer.Strategy.valueOf(strategyStr.toUpperCase()); - switch (strategy) { - case FIXED: - if (this.timeLimit != null) { - this.updateTimer = new UpdateTimer(this.timeLimit / this.queryHandler.getQueryCount()); - } else { - LOGGER.warn("Worker[{{}} : {{}}]: FIXED Updates can only be used with timeLimit!", this.workerType, this.workerID); - } - break; - case DISTRIBUTED: - if (this.timeLimit != null) { - this.updateTimer = new UpdateTimer(this.queryHandler.getQueryCount(), this.timeLimit); - } else { - LOGGER.warn("Worker[{{}} : {{}}]: DISTRIBUTED Updates can only be used with timeLimit!", this.workerType, this.workerID); - } - break; - default: - break; - } - LOGGER.debug("Worker[{{}} : {{}}]: UpdateTimer was set to UpdateTimer:{{}}", this.workerType, this.workerID, this.updateTimer); - } - - - /** - * Checks if one queryMix was already executed, as it does not matter how many mixes should be executed - * - * @param noOfQueryMixes - * @return - */ - @Override - public boolean hasExecutedNoOfQueryMixes(Long noOfQueryMixes) { - return getExecutedQueries() / (getNoOfQueries() * 1.0) >= 1; - } - -} diff --git a/src/main/java/org/aksw/iguana/cc/worker/impl/update/UpdateTimer.java b/src/main/java/org/aksw/iguana/cc/worker/impl/update/UpdateTimer.java deleted file mode 100644 index f1660baa1..000000000 --- a/src/main/java/org/aksw/iguana/cc/worker/impl/update/UpdateTimer.java +++ /dev/null @@ -1,100 +0,0 @@ -package org.aksw.iguana.cc.worker.impl.update; - -/** - * - * Class to calculate time between two update queries. - * - * @author f.conrads - * - */ -public class UpdateTimer { - - private Strategy strategy; - private double baseValue; - private Double timeLimit; - - - /** - * - * The possible strategies - *

    - *
  • NONE: updates will be executed immediately after another
  • - *
  • FIXED: a fixed value in ms will be waited before the next update query
  • - *
  • DISTRIBUTED: the updates will be equally distributed over the time limit of the task
  • - *
- * - * @author f.conrads - * - */ - public enum Strategy { - /** - * updates will be executed immediately after another - */ - NONE, - - /** - * a fixed value in ms will be waited before the next update query - */ - FIXED, - - /** - * the updates will be equally distributed over the time limit of the task - */ - DISTRIBUTED - } - - /** - * Creates the default UpdateTimer - * All update queries will be executed immediately after another - */ - public UpdateTimer() { - this.strategy= Strategy.NONE; - } - - /** - * Creates a FixedUpdateTimer - * - * @param fixedValue the fixed time to wait between queries - */ - public UpdateTimer(double fixedValue) { - this.strategy= Strategy.FIXED; - this.baseValue=fixedValue; - } - - /** - * Creates a distributed UpdateTimer - * - * @param noOfUpdates the number of update queries - * @param timeLimit the timeLimit of the task - */ - public UpdateTimer(int noOfUpdates, Double timeLimit) { - this.strategy= Strategy.DISTRIBUTED; - this.baseValue=noOfUpdates; - this.timeLimit = timeLimit; - } - - - /** - * calculates the time the UPDATEWorker has to wait until the next update query - * - * @param timeExceeded The time it took from start of the task to now - * @param executedQueries currently number of executed Update Queries - * @return The time to wait - */ - public double calculateTime(double timeExceeded, long executedQueries) { - switch(strategy) { - case FIXED: - return baseValue; - case DISTRIBUTED: - return (timeLimit-timeExceeded)/(baseValue-executedQueries); - default: - return 0; - } - } - - - @Override - public String toString() { - return "[strategy: "+this.strategy.name()+"]"; - } -} \ No newline at end of file diff --git a/src/main/java/org/aksw/iguana/commons/annotation/Nullable.java b/src/main/java/org/aksw/iguana/commons/annotation/Nullable.java deleted file mode 100644 index 3d11e9dd7..000000000 --- a/src/main/java/org/aksw/iguana/commons/annotation/Nullable.java +++ /dev/null @@ -1,12 +0,0 @@ -package org.aksw.iguana.commons.annotation; - -import java.lang.annotation.*; - -/** - * Lets the TypeFactory know that the Parameter can be null and thus be ignored. - */ -@Documented -@Retention(RetentionPolicy.RUNTIME) -@Target(ElementType.PARAMETER) -public @interface Nullable { -} diff --git a/src/main/java/org/aksw/iguana/commons/annotation/ParameterNames.java b/src/main/java/org/aksw/iguana/commons/annotation/ParameterNames.java deleted file mode 100644 index ceae9f810..000000000 --- a/src/main/java/org/aksw/iguana/commons/annotation/ParameterNames.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.aksw.iguana.commons.annotation; - -import java.lang.annotation.*; - -/** - * Uses provided names in the order of the constructor parameters, instead of the constructor parameter names for the TypeFactory - */ -@Retention(RetentionPolicy.RUNTIME) -@Target(ElementType.CONSTRUCTOR) -@Inherited -public @interface ParameterNames { - - String[] names() default ""; -} diff --git a/src/main/java/org/aksw/iguana/commons/annotation/Shorthand.java b/src/main/java/org/aksw/iguana/commons/annotation/Shorthand.java deleted file mode 100644 index ee19817ca..000000000 --- a/src/main/java/org/aksw/iguana/commons/annotation/Shorthand.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.aksw.iguana.commons.annotation; - -import java.lang.annotation.*; - -/** - * Sets a short name to be used in the TypedFactory instead of the whole class name - */ -@Documented -@Retention(RetentionPolicy.RUNTIME) -@Target(ElementType.TYPE) -public @interface Shorthand { - - String value(); -} diff --git a/src/main/java/org/aksw/iguana/commons/constants/COMMON.java b/src/main/java/org/aksw/iguana/commons/constants/COMMON.java deleted file mode 100644 index 1a63bc9bf..000000000 --- a/src/main/java/org/aksw/iguana/commons/constants/COMMON.java +++ /dev/null @@ -1,113 +0,0 @@ -package org.aksw.iguana.commons.constants; - -/** - * Constants several modules need - * - * @author f.conrads - * - */ -public class COMMON { - - /* - * COMMON CONSTANTS - */ - - - /** - * The key for the experiment task ID in the properties received from the core - */ - public static final String EXPERIMENT_TASK_ID_KEY = "taskID"; - - /** - * The key for the experiment ID in the properties received from the core - */ - public static final String EXPERIMENT_ID_KEY = "expID"; - - /** - * The key for suite ID in the properties received from the core - */ - public static final String SUITE_ID_KEY = "suiteID"; - - - /** - * The key for starting an experiment task. Must be in the receiving properties - */ - public static final String RECEIVE_DATA_START_KEY = "startExperimentTask"; - - /** - * The key for ending an experiment task. Must be in the receiving properties - */ - public static final String RECEIVE_DATA_END_KEY = "endExperimentTask"; - - - /** - * Key in the properties receiving from the core to start an experiment - * as well as internal rp metrics key - */ - public static final String METRICS_PROPERTIES_KEY = "metrics"; - - - - /** - * TP2RP query time key - */ - public static final String RECEIVE_DATA_TIME = "resultTime"; - - /** - * TP2RP (Controller2RP) query success key - */ - public static final String RECEIVE_DATA_SUCCESS = "resultSuccess"; - - /** - * The number of Queries in the particular experiment - * will be used in the meta data. - */ - public static final String NO_OF_QUERIES = "noOfQueries"; - - - - public static final String QUERY_ID_KEY = "queryID"; - - public static final String CONNECTION_ID_KEY = "connID"; - - public static final String DATASET_ID_KEY = "datasetID"; - - public static final String EXTRA_META_KEY = "extraMeta"; - - public static final String EXTRA_IS_RESOURCE_KEY = "setIsResource"; - - public static final String QUERY_STRING = "queryString"; - - public static final String DOUBLE_RAW_RESULTS = "doubleRawResults"; - - public static final String SIMPLE_TRIPLE_KEY = "cleanTripleText"; - - public static final String QUERY_STATS = "queryStats"; - - public static final Object RECEIVE_DATA_SIZE = "resultSize"; - - public static final String QUERY_HASH = "queryHash"; - - public static final String WORKER_ID = "workerID"; - - /* Various status codes to denote the status of query execution and to prepare QueryExecutionStats object */ - public static final long QUERY_UNKNOWN_EXCEPTION = 0L; - - public static final long QUERY_SUCCESS = 1L; - - public static final long QUERY_SOCKET_TIMEOUT = -1L; - - public static final long QUERY_HTTP_FAILURE = -2L; - - public static final String EXPERIMENT_TASK_CLASS_ID_KEY = "actualTaskClass" ; - - public static final String BASE_URI = "http://iguana-benchmark.eu"; - - - public static final String RES_BASE_URI = BASE_URI+"/resource/"; - public static final String PROP_BASE_URI = BASE_URI+"/properties/"; - public static final String CLASS_BASE_URI = BASE_URI+"/class/"; - public static final String PENALTY = "penalty"; - public static final String CONNECTION_VERSION_KEY = "connectionVersion"; - public static final String EXPERIMENT_TASK_NAME_KEY = "taskName"; -} diff --git a/src/main/java/org/aksw/iguana/commons/factory/TypedFactory.java b/src/main/java/org/aksw/iguana/commons/factory/TypedFactory.java deleted file mode 100644 index 4385f8a52..000000000 --- a/src/main/java/org/aksw/iguana/commons/factory/TypedFactory.java +++ /dev/null @@ -1,293 +0,0 @@ -/** - * - */ -package org.aksw.iguana.commons.factory; - -import org.aksw.iguana.commons.annotation.Nullable; -import org.aksw.iguana.commons.annotation.ParameterNames; -import org.aksw.iguana.commons.reflect.ShorthandMapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Parameter; -import java.util.*; - - -/** - * Factory for a Type. - * Creates an Object from Constructor and Constructor Arguments - * - * @author f.conrads - * @param The Type which should be created - * - */ -public class TypedFactory { - - private static final Logger LOGGER = LoggerFactory - .getLogger(TypedFactory.class); - - private String getClassName(String className){ - Map map = ShorthandMapper.getInstance().getShortMap(); - if(map.containsKey(className)){ - return map.get(className); - } - return className; - } - - - /** - * Will create a T Object from a Constructor Object created by the - * class name and the constructor arguments, be aware that all arguments - * must be Strings in the constructor. - * - * @param className - * The Class Name of the Implemented T Object - * @param constructorArgs - * constructor arguments (must be Strings), can be safely null - * @return The T Object created by the Constructor using the - * constructor args - */ - @SuppressWarnings("unchecked") - public T create(String className, Object[] constructorArgs){ - Object[] constructorArgs2 = constructorArgs; - if (constructorArgs2 == null) { - constructorArgs2 = new Object[0]; - } - Class[] stringClass = new Class[constructorArgs2.length]; - Arrays.fill(stringClass, String.class); - return create(className, constructorArgs2, stringClass); - } - - /** - * Will create a T Object from a Constructor Object created by the - * class name and the constructor arguments, and an Array which states each - * Constructor Object Class - * - * @param className - * The Class Name of the Implemented T Object - * @param constructorArgs - * constructor arguments (must be Strings), can be safely null - * @param constructorClasses The class of each constructor argument - * @return The T Object created by the Constructor using the - * constructor args - */ - @SuppressWarnings("unchecked") - public T create(String className, Object[] constructorArgs, Class[] constructorClasses) { - - Object[] constructorArgs2 = constructorArgs; - - if (className == null) { - return null; - } - Class clazz; - try { - clazz = (Class) ClassLoader - .getSystemClassLoader().loadClass(className); - } catch (ClassNotFoundException e1) { - LOGGER.error("Could not load Object (name: " + className - + ")", e1); - return null; - } - - - if (constructorArgs2 == null) { - constructorArgs2 = new Object[0]; - } - if(constructorClasses==null){ - constructorClasses = new Class[constructorArgs2.length]; - Arrays.fill(constructorClasses, String.class); - } - - try { - Constructor constructor = clazz - .getConstructor(constructorClasses); - return constructor.newInstance(constructorArgs2); - } catch (InstantiationException | IllegalAccessException - | IllegalArgumentException | InvocationTargetException - | NoSuchMethodException | SecurityException e) { - LOGGER.error("Could not initialize class " + clazz.getName() - + " with constructor.", e); - return null; - } - } - - - /** - * Uses the parameter Names and types of a constructor to find the best fitting constructor - * - * Only works with jvm -paramaters, otherwise use createAnnotated and annotate the constructors with ParameterNames and set names to the paramater names - * like - * . @ParameterNames(names={"a", "b"}) - * public Constructor(String a, Object b){...} - * - * @param className The Class Name of the Implemented T Object - * @param map key-value pair, whereas key represents the parameter name, where as value will be the value of the instantiation - * @return The instantiated object or null no constructor was found - */ - public T create(String className, Map map) { - Class clazz; - if (className == null) { - return null; - } - try { - clazz = (Class) ClassLoader.getSystemClassLoader().loadClass(getClassName(className)); - } catch (ClassNotFoundException e1) { - return null; - } - Constructor[] constructors = clazz.getConstructors(); - find: - for (Constructor constructor : constructors) { - //ParameterNames would be a backup - //ParameterNames paramNames = (ParameterNames) constructor.getAnnotation(ParameterNames.class); - //if(paramNames==null){ - // continue ; - //} - Parameter[] params = constructor.getParameters(); - - List names = new ArrayList<>(); - List> types = new ArrayList<>(); - Set canBeNull = new HashSet<>(); - for (Parameter p : params) { - names.add(p.getName()); - types.add(p.getType()); - if (p.isAnnotationPresent(Nullable.class)) { - canBeNull.add(p.getName()); - } - } - List instanceNames = new ArrayList<>(map.keySet()); - Object[] constructorArgs = new Object[names.size()]; - if (!checkIfFits(map, names, canBeNull)) { - continue; - } - for (String key : instanceNames) { - Object value = map.get(key); - //Check if constructor can map keys to param Names - int indexKey = names.indexOf(key); - Class clazz2 = types.get(indexKey); - if (!clazz2.isInstance(value)) { - continue find; - } - constructorArgs[indexKey] = value; - } - try { - return (T) constructor.newInstance(constructorArgs); - } catch (InstantiationException | IllegalAccessException - | IllegalArgumentException | InvocationTargetException - | SecurityException e) { - //As we check that the COnstructor fits this shouldn't be thrown at all. Something very bad happend - LOGGER.error("Could not initialize class " + clazz.getName() - + " with constructor.", e); - return null; - } - } - LOGGER.error("Could not initialize class " + clazz.getName() - + " with constructor. Maybe Config file has wrong names?."); - return null; - } - - /** - * Checks if the giving parameter key-value mapping fits the constructor parameter names (key vs names) and takes into account that the parameter is allowed to be null and thus - * can be disregarded - * - * @param map paramater - Object Map - * @param names parameter names of the actual constructor - * @param canBeNull all paramaters who can be null - * @return true if constructor fits, otherwise false - */ - private boolean checkIfFits(Map map, List names, Set canBeNull) { - //check if all provided parameter names are in the constructor - for (String key : map.keySet()) { - if (!names.contains(key)) { - return false; - } - } - //check if all notNull objects are provided - Set keySet = map.keySet(); - for (String name : names) { - //we can safely assume that Object is string - if (!keySet.contains(name)) { - //check if parameter is Nullable - if (!canBeNull.contains(name)) { - return false; - } - } - } - return true; - } - - /** - * Uses the parameter Names and types of a constructor to find the best fitting constructor - * - * Uses the ParameterNames annotation of a constructor to get the parameter names - * - * like - * . @ParameterNames(names={"a", "b"}) - * public Constructor(String a, Object b){...} - * - * @param className The Class Name of the Implemented T Object - * @param map Parameter name - value mapping - * @return The instantiated object or null no constructor was found - */ - public T createAnnotated(String className, Map map) { - Class clazz; - try { - clazz = (Class) ClassLoader.getSystemClassLoader().loadClass(getClassName(className)); - } catch (ClassNotFoundException e1) { - return null; - } - Constructor[] constructors = clazz.getConstructors(); - find: - for (Constructor constructor : constructors) { - ParameterNames paramNames = constructor.getAnnotation(ParameterNames.class); - if (paramNames == null) { - continue; - } - Parameter[] params = constructor.getParameters(); - - List names = new ArrayList<>(); - List> types = new ArrayList<>(); - Set canBeNull = new HashSet<>(); - for (int i = 0; i < params.length; i++) { - Parameter p = params[i]; - names.add(paramNames.names()[i]); - types.add(p.getType()); - if (p.isAnnotationPresent(Nullable.class)) { - canBeNull.add(p.getName()); - } - } - List instanceNames = new ArrayList<>(map.keySet()); - Object[] constructorArgs = new Object[names.size()]; - if (!checkIfFits(map, names, canBeNull)) { - continue; - } - for (String key : instanceNames) { - Object value = map.get(key); - //Check if constructor can map keys to param Names - int indexKey = names.indexOf(key); - Class clazz2 = types.get(indexKey); - if (!clazz2.isInstance(value)) { - continue find; - } - constructorArgs[indexKey] = value; - } - try { - return (T) constructor.newInstance(constructorArgs); - } catch (InstantiationException | IllegalAccessException - | IllegalArgumentException | InvocationTargetException - | SecurityException e) { - //As we check that the Constructor fits this shouldn't be thrown at all. Something very bad happend - LOGGER.error("Could not initialize class " + clazz.getName() - + " with constructor.", e); - return null; - } - } - LOGGER.error("Could not initialize class " + clazz.getName() - + " with constructor. Maybe Config file has wrong names?."); - return null; - } - - -} - diff --git a/src/main/java/org/aksw/iguana/commons/io/BigByteArrayInputStream.java b/src/main/java/org/aksw/iguana/commons/io/BigByteArrayInputStream.java index 5c41dfedd..1559fd7f1 100644 --- a/src/main/java/org/aksw/iguana/commons/io/BigByteArrayInputStream.java +++ b/src/main/java/org/aksw/iguana/commons/io/BigByteArrayInputStream.java @@ -2,53 +2,162 @@ import java.io.IOException; import java.io.InputStream; +import java.util.Objects; + +import static java.lang.Math.min; public class BigByteArrayInputStream extends InputStream { - private BigByteArrayOutputStream bbaos; + final private BigByteArrayOutputStream bbaos; + + private byte[] currentBuffer; + private int currentBufferSize = -1; + private int posInCurrentBuffer = 0; - private byte[] curArray; - private int curPos=0; - private int curPosInArray=0; + private boolean ended = true; public BigByteArrayInputStream(byte[] bytes) throws IOException { bbaos = new BigByteArrayOutputStream(); bbaos.write(bytes); - setNextArray(); + activateNextBuffer(); } - public BigByteArrayInputStream(BigByteArrayOutputStream bbaos){ + /** + * The given bbaos will be closed, when read from it. + * + * @param bbaos + */ + public BigByteArrayInputStream(BigByteArrayOutputStream bbaos) { this.bbaos = bbaos; - setNextArray(); + activateNextBuffer(); } - private void setNextArray(){ - curArray=bbaos.getBaos().get(curPos++).toByteArray(); - } @Override public int read() throws IOException { - if(eos()){ - return -1; - } - int ret; + this.bbaos.close(); - if(curPosInArray==2147483639){ - ret = curArray[curPosInArray]; - curPosInArray=0; - setNextArray(); - } - else{ - ret=curArray[curPosInArray++]; + if (ended) return -1; + final var ret = currentBuffer[posInCurrentBuffer++]; + if (availableBytes() == 0) + activateNextBuffer(); + return ret & 0xFF; // convert byte (-128...127) to (0...255) + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + this.bbaos.close(); + Objects.checkFromIndexSize(off, len, b.length); + + if (ended) return -1; + + final var copyLength1 = min(availableBytes(), len); + System.arraycopy(currentBuffer, posInCurrentBuffer, b, off, copyLength1); + posInCurrentBuffer += copyLength1; + off += copyLength1; + if (availableBytes() == 0) + activateNextBuffer(); + + // check if b is already filled up or if there is nothing left to read + if (copyLength1 == len || ended) return copyLength1; + + // there might be the rare case, where reading one additional baos might not be enough to fill the buffer, + // because there are different array size limitations across different JVMs + final var copyLength2 = min(availableBytes(), len - copyLength1); + System.arraycopy(currentBuffer, posInCurrentBuffer, b, off, copyLength2); + posInCurrentBuffer += copyLength2; + + if (availableBytes() == 0) + activateNextBuffer(); + + return copyLength1 + copyLength2; + } + + @Override + public int readNBytes(byte[] b, int off, int len) throws IOException { + this.bbaos.close(); + Objects.checkFromIndexSize(off, len, b.length); + + if (ended) return 0; + + final var copyLength1 = min(availableBytes(), len); + System.arraycopy(currentBuffer, posInCurrentBuffer, b, off, copyLength1); + posInCurrentBuffer += copyLength1; + off += copyLength1; + if (availableBytes() == 0) + activateNextBuffer(); + + // check if b is already filled up or if there is nothing left to read + if (copyLength1 == len || ended) return copyLength1; + + // there might be the rare case, where reading one additional baos might not be enough to fill the buffer, + // because there are different array size limitations across different JVMs + final var copyLength2 = min(availableBytes(), len - copyLength1); + System.arraycopy(currentBuffer, posInCurrentBuffer, b, off, copyLength2); + posInCurrentBuffer += copyLength2; + + if (availableBytes() == 0) + activateNextBuffer(); + + return copyLength1 + copyLength2; + } + + @Override + public byte[] readAllBytes() throws IOException { + throw new IOException("Reading all bytes from a BigByteArrayInputStream is prohibited because it might exceed the array capacity"); + } + + @Override + public long skip(long n) throws IOException { + if (n <= 0) return 0; + long skipped = 0; + while (skipped < n) { + long thisSkip = min(availableBytes(), n - skipped); + skipped += thisSkip; + posInCurrentBuffer += (int) thisSkip; // conversion to int is lossless, because skipped is at maximum INT_MAX big + if (availableBytes() == 0) + if (!activateNextBuffer()) + return skipped; } - return ret ; + return skipped; } - private boolean eos() { - //if the current Position is equal the length of the array, this is the last array in bbaos and the last element was already read - if(curArray.length==curPosInArray){ - return true; + /** + * Activate the next buffer the underlying BigByteArrayOutputStream. + * + * @return true if the next buffer was activated, false if there are no more buffers available + */ + private boolean activateNextBuffer() { + // check if another buffer is available + if (bbaos.getBaos().isEmpty()) { + currentBuffer = null; // release memory + currentBufferSize = 0; + posInCurrentBuffer = 0; + ended = true; + return false; } - return false; + + // activate next buffer + currentBuffer = bbaos.getBaos().get(0).getBuffer(); + currentBufferSize = bbaos.getBaos().get(0).size(); + posInCurrentBuffer = 0; + + // remove the current buffer from the list to save memory + bbaos.getBaos().remove(0); + + // check if the new buffer contains anything + if (currentBuffer.length == 0) + return ended = activateNextBuffer(); + ended = false; + return true; + } + + /** + * Returns the number of available bytes in the current buffer. + * + * @return the number of available bytes in the current buffer + */ + private int availableBytes() { + return currentBufferSize - posInCurrentBuffer; } } diff --git a/src/main/java/org/aksw/iguana/commons/io/BigByteArrayOutputStream.java b/src/main/java/org/aksw/iguana/commons/io/BigByteArrayOutputStream.java index 605131977..2085b4158 100644 --- a/src/main/java/org/aksw/iguana/commons/io/BigByteArrayOutputStream.java +++ b/src/main/java/org/aksw/iguana/commons/io/BigByteArrayOutputStream.java @@ -1,108 +1,204 @@ package org.aksw.iguana.commons.io; -import java.io.ByteArrayOutputStream; +import org.apache.hadoop.hbase.io.ByteArrayOutputStream; + import java.io.IOException; import java.io.OutputStream; -import java.io.UnsupportedEncodingException; -import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; - +import java.util.Objects; +import java.util.stream.IntStream; + +/** + * This class represents a ByteArrayOutputStream that can hold a large amount of byte data. + * It is designed to overcome the limitations of the standard ByteArrayOutputStream, which + * has a fixed internal byte array and can run into out of memory errors when trying to write + * a large amount of data. + *

+ * The BigByteArrayOutputStream works by using an ArrayList of ByteArrayOutputStreams to store + * the byte data. When the current ByteArrayOutputStream fills up, a new one is created with the + * maximum array size (Integer.MAX_VALUE - 8) as its initial capacity and added to the list. + * Writing data to the stream involves writing to the current active ByteArrayOutputStream. When + * the stream is cleared, all the internal ByteArrayOutputStreams are cleared and a new one is + * added to the list. + */ public class BigByteArrayOutputStream extends OutputStream { - private List baos = new ArrayList(); + /** + * The maximum size limit for an array. This is no limit to the amount of bytes {@code BigByteArrayOutputStream} can consume. + */ + public final static int ARRAY_SIZE_LIMIT = Integer.MAX_VALUE - 8; + + /** + * Holds a list of ByteArrayOutputStream objects. + */ + private final List baosList; + + /** + * The index of a ByteArrayOutputStream in the List baosList. + */ + private int baosListIndex; + /** + * Represents the current ByteArrayOutputStream used for writing data. + */ + private ByteArrayOutputStream currentBaos; + + private boolean closed = false; + + /** + * Initializes a new instance of the BigByteArrayOutputStream class with default buffer size. + */ public BigByteArrayOutputStream() { - baos.add(new ByteArrayOutputStream()); + baosList = new ArrayList<>(); + baosList.add(new ByteArrayOutputStream()); + try { + reset(); + } catch (IOException ignored) {} + } + + /** + * Initializes a new instance of the BigByteArrayOutputStream class with buffer size. + * + * @param bufferSize initial guaranteed buffer size + */ + public BigByteArrayOutputStream(int bufferSize) { + if (bufferSize < 0) + throw new IllegalArgumentException("Negative initial size: " + bufferSize); + baosList = new ArrayList<>(1); + baosList.add(new ByteArrayOutputStream(bufferSize)); + try { + reset(); + } catch (IOException ignored) {} + } + + /** + * Initializes a new instance of the BigByteArrayOutputStream class with buffer size. + * + * @param bufferSize initial guaranteed buffer size + */ + public BigByteArrayOutputStream(long bufferSize) { + if (bufferSize < 0) + throw new IllegalArgumentException("Negative initial size: " + bufferSize); + if (bufferSize <= ARRAY_SIZE_LIMIT) { + baosList = new ArrayList<>(1); + baosList.add(new ByteArrayOutputStream((int) bufferSize)); + } else { + final var requiredBaoss = (int) ((bufferSize - 1) / ARRAY_SIZE_LIMIT) + 1; // -1 to prevent creating a fully sized, but empty baos at the end if the buffer size is a multiple of ARRAY_SIZE_LIMIT + baosList = new ArrayList<>(requiredBaoss); + IntStream.range(0, requiredBaoss).forEachOrdered(i -> baosList.add(new ByteArrayOutputStream(ARRAY_SIZE_LIMIT))); + } + try { + reset(); + } catch (IOException ignored) {} } public List getBaos() { - return baos; + return baosList; } public void write(BigByteArrayOutputStream bbaos) throws IOException { - for (byte[] bao : bbaos.toByteArray()) { - for (Byte b : bao) { - write(b); - } - } - + write(bbaos.toByteArray()); } public long size() { - long ret = 0; - for (ByteArrayOutputStream ba : baos) { - ret += ba.size(); - } - return ret; + return baosList.stream().mapToLong(ByteArrayOutputStream::size).sum(); } - public synchronized byte[][] toByteArray() { - byte[][] ret = new byte[baos.size()][]; - for (int i = 0; i < baos.size(); i++) { - ret[i] = baos.get(i).toByteArray(); + public byte[][] toByteArray() { + byte[][] ret = new byte[baosList.size()][]; + for (int i = 0; i < baosList.size(); i++) { + ret[i] = baosList.get(i).toByteArray(); } return ret; } - - public void write(byte[] i) throws IOException { - for (byte b : i) { - write(b); + @Override + public void write(byte[] b, int off, int len) throws IOException { + if (closed) throw new IOException("Tried to write to a closed stream"); + + Objects.checkFromIndexSize(off, len, b.length); + final var space = ensureSpace(); + final var writeLength = Math.min(len, space); + this.currentBaos.write(b, off, writeLength); + final var remainingBytes = len - writeLength; + if (remainingBytes > 0) { + ensureSpace(); + this.currentBaos.write(b, off + writeLength, remainingBytes); } } - public void write(byte[][] i) throws IOException { - for (byte[] arr : i) { - for (byte b : arr) { - write(b); - } + public void write(byte[][] byteArray) throws IOException { + for (byte[] arr : byteArray) { + write(arr); } } - public void write(byte i) throws IOException { - ByteArrayOutputStream current = baos.get(baos.size() - 1); - current = ensureSpace(current); - current.write(i); + public void write(byte b) throws IOException { + if (closed) throw new IOException("Tried to write to a closed stream"); + + ensureSpace(); + this.currentBaos.write(b); } @Override public void write(int i) throws IOException { - ByteArrayOutputStream current = baos.get(baos.size() - 1); - current = ensureSpace(current); - current.write(i); + if (closed) throw new IOException("Tried to write to a closed stream"); + + ensureSpace(); + this.currentBaos.write(i); } - private ByteArrayOutputStream ensureSpace(ByteArrayOutputStream current) { - if (current.size() == 2147483639) { - baos.add(new ByteArrayOutputStream()); + + private int ensureSpace() { + var space = ARRAY_SIZE_LIMIT - currentBaos.size(); + if (space == 0) { + space = ARRAY_SIZE_LIMIT; + if (baosListIndex == baosList.size() - 1) { + baosListIndex++; + currentBaos = new ByteArrayOutputStream(ARRAY_SIZE_LIMIT); + baosList.add(currentBaos); + } else { + baosListIndex++; + currentBaos = baosList.get(baosListIndex); + currentBaos.reset(); + } } - return baos.get(baos.size() - 1); + return space; } - public String toString(String charset) throws UnsupportedEncodingException { - StringBuilder builder = new StringBuilder(); - for(ByteArrayOutputStream baos : this.baos){ - builder.append(baos.toString(charset)); + /** + * Resets the state of the object by setting the baosListIndex to zero + * and assigning the first ByteArrayOutputStream in the baosList to the + * currentBaos variable. No {@link ByteArrayOutputStream}s are actually removed. + */ + public void reset() throws IOException { + if (closed) throw new IOException("Tried to reset to a closed stream"); + + currentBaos = baosList.get(baosListIndex = 0); + for (var baos : baosList) { + baos.reset(); } - return builder.toString(); } - public String toString(Charset charset) throws UnsupportedEncodingException { - return toString(charset.toString()); + /** + * Clears the state of the object by removing all {@link ByteArrayOutputStream}s + * from the baosList except for the first one. The baosListIndex is set to 1 + * and the currentBaos variable is reassigned to the first ByteArrayOutputStream + * in the baosList. + */ + public void clear() throws IOException { + if (closed) throw new IOException("Tried to clear to a closed stream"); + + if (baosList.size() > 1) + baosList.subList(1, this.baosList.size()).clear(); + currentBaos = baosList.get(baosListIndex = 0); + currentBaos.reset(); } - public Long countMatches(char s) { - //read - long count=0; - for(ByteArrayOutputStream baos : this.baos){ - for(byte b : baos.toByteArray()){ - if(b==s){ - count++; - } - } - } - return count; + @Override + public void close() throws IOException { + this.closed = true; } } \ No newline at end of file diff --git a/src/main/java/org/aksw/iguana/commons/numbers/NumberUtils.java b/src/main/java/org/aksw/iguana/commons/numbers/NumberUtils.java deleted file mode 100644 index 2c8629039..000000000 --- a/src/main/java/org/aksw/iguana/commons/numbers/NumberUtils.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.aksw.iguana.commons.numbers; - -/** - * Utils class for everything with numbers - * - * @author f.conrads - * - */ -public class NumberUtils { - - /** - * Returns either a long represantation of the String nm or null. - * - * @param nm String which should be parsed - * @return String as a long representation if String is a Long, otherwise null - */ - public static Long getLong(String nm) { - try { - Long ret = Long.parseLong(nm); - return ret; - }catch(NumberFormatException e) {} - return null; - } - - /** - * Returns either a double representation of the String nm or null. - * - * @param nm String which should be parsed - * @return String as a double representation if String is a double, otherwise null - */ - public static Double getDouble(String nm) { - try { - return Double.parseDouble(nm); - } catch (NumberFormatException | NullPointerException ignored) { - } - return null; - } - -} diff --git a/src/main/java/org/aksw/iguana/commons/rdf/IONT.java b/src/main/java/org/aksw/iguana/commons/rdf/IONT.java index b9027f9c0..bc06b1548 100644 --- a/src/main/java/org/aksw/iguana/commons/rdf/IONT.java +++ b/src/main/java/org/aksw/iguana/commons/rdf/IONT.java @@ -1,6 +1,6 @@ package org.aksw.iguana.commons.rdf; -import org.aksw.iguana.cc.tasks.stresstest.metrics.Metric; +import org.aksw.iguana.cc.metrics.Metric; import org.apache.jena.rdf.model.Resource; import org.apache.jena.rdf.model.ResourceFactory; @@ -9,7 +9,6 @@ public class IONT { public static final String PREFIX = "iont"; public static final Resource suite = ResourceFactory.createResource(NS + "Suite"); - public static final Resource experiment = ResourceFactory.createResource(NS + "Experiment"); public static final Resource dataset = ResourceFactory.createResource(NS + "Dataset"); public static final Resource task = ResourceFactory.createResource(NS + "Task"); public static final Resource connection = ResourceFactory.createResource(NS + "Connection"); @@ -20,11 +19,7 @@ public class IONT { public static final Resource metric = ResourceFactory.createResource(NS + "Metric"); public static Resource getMetricClass(Metric metric) { - // TODO: compare with stresstest class + // TODO: compare with stresstest class (stresstest class as a subclass of Task is iont:Stresstest while QPS for example is iont:metric/QPS) return ResourceFactory.createResource(NS + "metric/" + metric.getAbbreviation()); } - - public static Resource getClass(String classname) { - return ResourceFactory.createResource(NS + classname); - } } diff --git a/src/main/java/org/aksw/iguana/commons/rdf/IPROP.java b/src/main/java/org/aksw/iguana/commons/rdf/IPROP.java index 81eeddd20..dcda72e89 100644 --- a/src/main/java/org/aksw/iguana/commons/rdf/IPROP.java +++ b/src/main/java/org/aksw/iguana/commons/rdf/IPROP.java @@ -1,6 +1,6 @@ package org.aksw.iguana.commons.rdf; -import org.aksw.iguana.cc.tasks.stresstest.metrics.Metric; +import org.aksw.iguana.cc.metrics.Metric; import org.apache.jena.rdf.model.Property; import org.apache.jena.rdf.model.ResourceFactory; @@ -8,9 +8,6 @@ public class IPROP { public static final String NS = IGUANA_BASE.NS + "properties" + "/"; public static final String PREFIX = "iprop"; - private IPROP() { - } - /** * The RDF-friendly version of the IPROP namespace * with trailing / character. @@ -23,44 +20,13 @@ public static Property createMetricProperty(Metric metric) { return ResourceFactory.createProperty(NS + metric.getAbbreviation()); } - /* - * SPARQL query properties - */ - public static final Property aggregations; - public static final Property filter; - public static final Property groupBy; - public static final Property having; - public static final Property offset; - public static final Property optional; - public static final Property orderBy; - public static final Property triples; - public static final Property union; - - /* - * Query Stats - */ - public static final Property failed; - public static final Property penalizedQPS; - public static final Property QPS; - public static final Property queryExecution; - public static final Property timeOuts; - - public static final Property totalTime; - public static final Property unknownException; - public static final Property wrongCodes; public static final Property succeeded = ResourceFactory.createProperty(NS, "succeeded"); - /* - * Each Query Stats - */ - public static final Property code; - public static final Property queryID; - public static final Property resultSize; - public static final Property run; - public static final Property success; - public static final Property time; + public static final Property responseBodyHash = ResourceFactory.createProperty(NS, "responseBodyHash"); + public static final Property responseBody = ResourceFactory.createProperty(NS, "responseBody"); + public static final Property startTime = ResourceFactory.createProperty(NS, "startTime"); + public static final Property httpCode = ResourceFactory.createProperty(NS, "httpCode"); - public static final Property experiment = ResourceFactory.createProperty(NS, "experiment"); public static final Property dataset = ResourceFactory.createProperty(NS, "dataset"); public static final Property task = ResourceFactory.createProperty(NS, "task"); public static final Property connection = ResourceFactory.createProperty(NS, "connection"); @@ -78,34 +44,38 @@ public static Property createMetricProperty(Metric metric) { public static final Property startDate = ResourceFactory.createProperty(NS, "startDate"); public static final Property endDate = ResourceFactory.createProperty(NS, "endDate"); - static { + // Language Processor + public static final Property results = ResourceFactory.createProperty(NS, "results"); + public static final Property bindings = ResourceFactory.createProperty(NS, "bindings"); + public static final Property variable = ResourceFactory.createProperty(NS, "variable"); + public static final Property exception = ResourceFactory.createProperty(NS, "exception"); - // SPARQL query properties - aggregations = ResourceFactory.createProperty(NS, "aggregations"); - filter = ResourceFactory.createProperty(NS, "filter"); - groupBy = ResourceFactory.createProperty(NS, "groupBy"); - having = ResourceFactory.createProperty(NS, "having"); - offset = ResourceFactory.createProperty(NS, "offset"); - optional = ResourceFactory.createProperty(NS, "optional"); - orderBy = ResourceFactory.createProperty(NS, "orderBy"); - triples = ResourceFactory.createProperty(NS, "triples"); - union = ResourceFactory.createProperty(NS, "union"); - // Query Stats - failed = ResourceFactory.createProperty(NS, "failed"); - penalizedQPS = ResourceFactory.createProperty(NS, "penalizedQPS"); - QPS = ResourceFactory.createProperty(NS, "QPS"); - queryExecution = ResourceFactory.createProperty(NS, "queryExecution"); - timeOuts = ResourceFactory.createProperty(NS, "timeOuts"); + // SPARQL query properties + public static final Property aggregations = ResourceFactory.createProperty(NS, "aggregations"); + public static final Property filter = ResourceFactory.createProperty(NS, "filter"); + public static final Property groupBy = ResourceFactory.createProperty(NS, "groupBy"); + public static final Property having = ResourceFactory.createProperty(NS, "having"); + public static final Property offset = ResourceFactory.createProperty(NS, "offset"); + public static final Property optional = ResourceFactory.createProperty(NS, "optional"); + public static final Property orderBy = ResourceFactory.createProperty(NS, "orderBy"); + public static final Property triples = ResourceFactory.createProperty(NS, "triples"); + public static final Property union = ResourceFactory.createProperty(NS, "union"); - totalTime = ResourceFactory.createProperty(NS, "totalTime"); - unknownException = ResourceFactory.createProperty(NS, "unknownException"); - wrongCodes = ResourceFactory.createProperty(NS, "wrongCodes"); - // Each Query Stats - code = ResourceFactory.createProperty(NS, "code"); - queryID = ResourceFactory.createProperty(NS, "queryID"); - resultSize = ResourceFactory.createProperty(NS, "resultSize"); - run = ResourceFactory.createProperty(NS, "run"); - success = ResourceFactory.createProperty(NS, "success"); - time = ResourceFactory.createProperty(NS, "time"); - } + // Query Stats + public static final Property failed = ResourceFactory.createProperty(NS, "failed"); + public static final Property penalizedQPS = ResourceFactory.createProperty(NS, "penalizedQPS"); + public static final Property QPS = ResourceFactory.createProperty(NS, "QPS"); + public static final Property queryExecution = ResourceFactory.createProperty(NS, "queryExecution"); + public static final Property timeOuts = ResourceFactory.createProperty(NS, "timeOuts"); + public static final Property totalTime = ResourceFactory.createProperty(NS, "totalTime"); + public static final Property unknownException = ResourceFactory.createProperty(NS, "unknownException"); + public static final Property wrongCodes = ResourceFactory.createProperty(NS, "wrongCodes"); + + // Each Query Stats + public static final Property code = ResourceFactory.createProperty(NS, "code"); + public static final Property queryID = ResourceFactory.createProperty(NS, "queryID"); + public static final Property resultSize = ResourceFactory.createProperty(NS, "resultSize"); + public static final Property run = ResourceFactory.createProperty(NS, "run"); + public static final Property success = ResourceFactory.createProperty(NS, "success"); + public static final Property time = ResourceFactory.createProperty(NS, "time"); } diff --git a/src/main/java/org/aksw/iguana/commons/rdf/IRES.java b/src/main/java/org/aksw/iguana/commons/rdf/IRES.java index 49a01ea3d..c24768f68 100644 --- a/src/main/java/org/aksw/iguana/commons/rdf/IRES.java +++ b/src/main/java/org/aksw/iguana/commons/rdf/IRES.java @@ -1,47 +1,63 @@ package org.aksw.iguana.commons.rdf; -import org.aksw.iguana.cc.tasks.stresstest.metrics.Metric; +import org.aksw.iguana.cc.metrics.Metric; +import org.aksw.iguana.cc.worker.HttpWorker; import org.apache.jena.rdf.model.Resource; import org.apache.jena.rdf.model.ResourceFactory; import java.math.BigInteger; +/** + * Class containing the IRES vocabulary and methods to create RDF resources. + */ public class IRES { public static final String NS = IGUANA_BASE.NS + "resource" + "/"; public static final String PREFIX = "ires"; - private IRES() { - } - - /** - * The RDF-friendly version of the IRES namespace - * with trailing / character. - */ - public static String getURI() { - return NS; - } - public static Resource getResource(String id) { return ResourceFactory.createResource(NS + id); } - public static Resource getWorkerResource(String taskID, int workerID) { - return ResourceFactory.createResource(NS + taskID + "/" + workerID); + public static Resource getMetricResource(Metric metric) { + return ResourceFactory.createResource(NS + metric.getAbbreviation()); } - public static Resource getTaskQueryResource(String taskID, String queryID) { - return ResourceFactory.createResource(NS + taskID + "/" + queryID); + public static Resource getResponsebodyResource(long hash) { + return ResourceFactory.createResource(NS + "responseBody" + "/" + hash); } - public static Resource getWorkerQueryResource(String taskID, int workerID, String queryID) { - return ResourceFactory.createResource(NS + taskID + "/" + workerID + "/" + queryID); - } + public static class Factory { - public static Resource getMetricResource(Metric metric) { - return ResourceFactory.createResource(NS + metric.getAbbreviation()); - } + private final String suiteID; + private final String taskURI; + + public Factory(String suiteID, long taskID) { + this.suiteID = suiteID; + this.taskURI = NS + suiteID + "/" + taskID; + } + + public Resource getSuiteResource() { + return ResourceFactory.createResource(NS + suiteID); + } + + public Resource getTaskResource() { + return ResourceFactory.createResource(this.taskURI); + } + + public Resource getWorkerResource(HttpWorker worker) { + return ResourceFactory.createResource(this.taskURI + "/" + worker.getWorkerID()); + } + + public Resource getTaskQueryResource(String queryID) { + return ResourceFactory.createResource(this.taskURI + "/" + queryID); + } + + public Resource getWorkerQueryResource(HttpWorker worker, int index) { + return ResourceFactory.createResource(this.taskURI + "/" + worker.getWorkerID() + "/" + worker.config().queries().getQueryId(index)); + } - public static Resource getWorkerQueryRunResource(String taskID, int workerID, String queryID, BigInteger run) { - return ResourceFactory.createResource(NS + taskID + "/" + workerID + "/" + queryID + "/" + run); + public Resource getWorkerQueryRunResource(HttpWorker worker, int index, BigInteger run) { + return ResourceFactory.createResource(this.taskURI + "/" + worker.getWorkerID() + "/" + worker.config().queries().getQueryId(index) + "/" + run); + } } } diff --git a/src/main/java/org/aksw/iguana/commons/reflect/ShorthandMapper.java b/src/main/java/org/aksw/iguana/commons/reflect/ShorthandMapper.java deleted file mode 100644 index 46d84fe5d..000000000 --- a/src/main/java/org/aksw/iguana/commons/reflect/ShorthandMapper.java +++ /dev/null @@ -1,71 +0,0 @@ -package org.aksw.iguana.commons.reflect; - -import org.aksw.iguana.commons.annotation.Shorthand; -import org.reflections.Configuration; -import org.reflections.Reflections; -import org.reflections.scanners.*; -import org.reflections.util.ConfigurationBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -/** - * Maps the shorthand to the class names at the beginning of it's initialization. - * Thus it has to be done once. - * - */ -public class ShorthandMapper { - - public Logger LOGGER = LoggerFactory.getLogger(getClass()); - - private Map shortMap = new HashMap(); - - private static ShorthandMapper instance; - - public static ShorthandMapper getInstance(){ - if(instance==null){ - instance = new ShorthandMapper(); - } - return instance; - } - - - public ShorthandMapper(){ - this("org"); - } - - /** - * create mapping, but only searches in packages with the prefix - * @param prefix package prefix to check - */ - public ShorthandMapper(String prefix){ - try { - Configuration config = ConfigurationBuilder.build(prefix).addScanners(new TypeAnnotationsScanner()).addScanners(new SubTypesScanner()); - Reflections reflections = new Reflections(new String[]{"", prefix}); - - Set> annotatedClasses = reflections.getTypesAnnotatedWith(Shorthand.class); - LOGGER.info("Found {} annotated classes", annotatedClasses.size()); - LOGGER.info("Annotated Classes : {}", annotatedClasses.toString()); - ClassLoader cloader = ClassLoader.getSystemClassLoader(); - for (Class annotatedClass : annotatedClasses) { - Shorthand annotation = (Shorthand) annotatedClass.getAnnotation(Shorthand.class); - if (annotation == null) { - continue; - } - if (shortMap.containsKey(annotation.value())) { - LOGGER.warn("Shorthand Key {} for Class {} already exists, pointing to Class {}. ", annotation.value(), shortMap.get(annotation.value()), annotatedClass.getCanonicalName()); - } - shortMap.put(annotation.value(), annotatedClass.getCanonicalName()); - } - }catch(Exception e){ - LOGGER.error("Could not create shorthand mapping", e); - } - } - - public Map getShortMap() { - return shortMap; - } -} diff --git a/src/main/java/org/aksw/iguana/commons/script/ScriptExecutor.java b/src/main/java/org/aksw/iguana/commons/script/ScriptExecutor.java deleted file mode 100644 index 6f7aac7be..000000000 --- a/src/main/java/org/aksw/iguana/commons/script/ScriptExecutor.java +++ /dev/null @@ -1,105 +0,0 @@ -/** - * - */ -package org.aksw.iguana.commons.script; - -import org.apache.commons.exec.ExecuteException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.*; - -/** - * Class to execute Shell Scripts - * - * @author f.conrads - * - */ -public class ScriptExecutor { - - private static final Logger LOGGER = LoggerFactory.getLogger(ScriptExecutor.class); - - /** - * Will execute the given file with the provided arguments - * via Shell. - * - * @param file file to execute - * @param args arguments to execute file with - * @throws ExecuteException if script can't be executed - * @throws IOException if file IO errors - * @return Process return, 0 means everything worked fine - */ - public static int exec(String file, String[] args) throws ExecuteException, IOException{ - String fileName = new File(file).getAbsolutePath(); - - String[] shellCommand = new String[1 + (args == null ? 0 : args.length)]; - shellCommand[0] = fileName; - - if(args != null && args.length!=0) - { - System.arraycopy(args, 0, shellCommand, 1, args.length); - } - - return execute(shellCommand); - } - - /**Checks if file contains arguments itself - * - * @param file file to execute - * @param args arguments to execute file with - * @return Process return, 0 means everything worked fine - * @throws ExecuteException if script can't be executed - * @throws IOException if file IO errors - */ - public static int execSafe(String file, String[] args) throws ExecuteException, IOException{ - String actualScript = file; - String[] args2 = args; - if(file.contains(" ")){ - - String[] providedArguments = file.split("\\s+"); - args2 = new String[providedArguments.length-1+args.length]; - actualScript=providedArguments[0]; - int i=1; - for(i=1;i threadBuffer = ThreadLocal.withInitial(() -> new byte[bufferSize]); - - protected static final ThreadLocal threadByteArrayOutputStream = ThreadLocal.withInitial(() -> new ByteArrayOutputStream(bufferSize)); - - /** - * Fastest way to serialize a stream to UTF-8 according to this stackoverflow question. - * - * @param inputStream the stream to read from - * @return the content of inputStream as a string. - * @throws IOException from {@link InputStream#read()} - */ - static public ByteArrayOutputStream inputStream2String(InputStream inputStream) throws IOException { - ByteArrayOutputStream result = threadByteArrayOutputStream.get(); - result.reset(); - try { - inputStream2ByteArrayOutputStream(inputStream, null, -1.0, result); - } catch (TimeoutException e) { - // never happens - System.exit(-1); - } - return result; - } - - /** - * Fastest way to serialize a stream to UTF-8 according to this stackoverflow question. - * - * @param inputStream the stream to read from - * @param startTime a time when the computation started - * @param timeout delta from startTime when the computation must be completed. Otherwise, a TimeoutException may be thrown. Timeout check is deactivated if timeout is < 0. - * @return the content of inputStream as a string. - * @throws IOException from {@link InputStream#read()} - * @throws TimeoutException Maybe thrown any time after if startTime + timeout is exceeded - */ - static public ByteArrayOutputStream inputStream2String(InputStream inputStream, Instant startTime, double timeout) throws IOException, TimeoutException { - ByteArrayOutputStream result = new ByteArrayOutputStream(); - inputStream2ByteArrayOutputStream(inputStream, startTime, timeout, result); - return result; - } - - /** - * Fastest way to serialize a stream to UTF-8 according to this stackoverflow question. - * - * @param inputStream the stream to read from - * @param startTime a time when the computation started - * @param timeout delta from startTime when the computation must be completed. Otherwise, a TimeoutException may be thrown. Timeout check is deactivated if timeout is < 0. - * @param result the stream where the result is written to. - * @return size of the output stream - * @throws IOException from {@link InputStream#read()} - * @throws TimeoutException Maybe thrown any time after if startTime + timeout is exceeded - */ - public static long inputStream2ByteArrayOutputStream(InputStream inputStream, Instant startTime, double timeout, ByteArrayOutputStream result) throws IOException, TimeoutException { - assert (result != null); - boolean enable_timeout = timeout > 0; - byte[] buffer = threadBuffer.get(); - int length; - while ((length = inputStream.read(buffer)) != -1) { - if (enable_timeout && durationInMilliseconds(startTime, Instant.now()) > timeout) - throw new TimeoutException("reading the answer timed out"); - result.write(buffer, 0, length); - } - return result.size(); - } - - /** - * Fastest way to serialize a stream to UTF-8 according to this stackoverflow question. - * - * @param inputStream the stream to read from - * @param result the stream where the result is written to. - * @return size of the output stream - * @throws IOException from {@link InputStream#read()} - */ - public static long inputStream2ByteArrayOutputStream(InputStream inputStream, ByteArrayOutputStream result) throws IOException { - try { - return inputStream2ByteArrayOutputStream(inputStream, Instant.now(), -1, result); - } catch (TimeoutException e) { - //will never happen - return 0; - } - } - - /** - * reads a stream and throws away the result. - * - * @param inputStream the stream to read from - * @param timeout delta from startTime when the computation must be completed. Otherwise, a TimeoutException may be thrown. Timeout check is deactivated if timeout is < 0. - * @return size of the output stream - * @throws IOException from {@link InputStream#read()} - * @throws TimeoutException Maybe thrown any time after if startTime + timeout is exceeded - */ - static public long inputStream2Length(InputStream inputStream, Instant startTime, double timeout) throws IOException, TimeoutException { - byte[] buffer = threadBuffer.get(); - long length; - long ret = 0; - while ((length = inputStream.read(buffer)) != -1) { - if (durationInMilliseconds(startTime, Instant.now()) > timeout && timeout > 0) - throw new TimeoutException("reading the answer timed out"); - ret += length; - } - return ret; - } -} diff --git a/src/main/java/org/aksw/iguana/commons/time/TimeUtils.java b/src/main/java/org/aksw/iguana/commons/time/TimeUtils.java index 46c2e13fa..4a7777689 100644 --- a/src/main/java/org/aksw/iguana/commons/time/TimeUtils.java +++ b/src/main/java/org/aksw/iguana/commons/time/TimeUtils.java @@ -1,47 +1,37 @@ package org.aksw.iguana.commons.time; import org.apache.jena.datatypes.xsd.XSDDuration; +import org.apache.jena.datatypes.xsd.impl.XSDDateTimeStampType; +import org.apache.jena.datatypes.xsd.impl.XSDDateTimeType; import org.apache.jena.datatypes.xsd.impl.XSDDurationType; +import org.apache.jena.rdf.model.Literal; +import org.apache.jena.rdf.model.ResourceFactory; import java.math.BigDecimal; import java.math.BigInteger; import java.time.Duration; import java.time.Instant; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; /** - * Everythin related to time stuff + * Class related to the conversion of Java time objects to RDF literals. */ public class TimeUtils { - /** - * returns the current time in Nanoseconds as a long instead of a double - * @return current time in nanoseconds as a long - */ - public static long getTimeInNanoseconds() { - Instant now = Instant.now(); - return ((long)now.getNano() + now.getEpochSecond() * 1000000000 /*ns*/); + public static XSDDuration toXSDDurationInSeconds(Duration duration) { + return (XSDDuration) new XSDDurationType().parse("PT" + new BigDecimal(BigInteger.valueOf(duration.toNanos()), 9).toPlainString() + "S"); } - /** - * gets the current time in milliseconds - * @return the current time in ms - */ - public static double getTimeInMilliseconds() { - return getTimeInNanoseconds() / 1000000d /*ms*/; + public static Literal createTypedDurationLiteral(Duration duration) { + return ResourceFactory.createTypedLiteral(new XSDDurationType().parse(duration.toString())); } - /** - * returns the duration in MS between two Time Instants - * @param start Start time - * @param end end time - * @return duration in ms between start and end - */ - public static double durationInMilliseconds(Instant start, Instant end) { - Duration duration = Duration.between(start, end); - return ((long)duration.getNano() + duration.getSeconds() * 1000000000 /*ns*/) / 1000000d /*ms*/; + public static Literal createTypedInstantLiteral(Instant time) { + return ResourceFactory.createTypedLiteral(new XSDDateTimeStampType(null).parse(time.toString())); } - public static XSDDuration toXSDDurationInSeconds(Duration duration) { - return (XSDDuration) new XSDDurationType().parse("PT" + new BigDecimal(BigInteger.valueOf(duration.toNanos()), 9).toPlainString() + "S"); + public static Literal createTypedZonedDateTimeLiteral(ZonedDateTime time) { + return ResourceFactory.createTypedLiteral(new XSDDateTimeStampType(null).parse(time.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME))); } } diff --git a/src/test/java/org/aksw/iguana/cc/config/ConfigTest.java b/src/test/java/org/aksw/iguana/cc/config/ConfigTest.java deleted file mode 100644 index 68a922e11..000000000 --- a/src/test/java/org/aksw/iguana/cc/config/ConfigTest.java +++ /dev/null @@ -1,57 +0,0 @@ -package org.aksw.iguana.cc.config; - -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; - -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; - -/** - * Checks if the config is read correctly as YAML as well as JSON and checks if the corresponding Task could be created - */ -@RunWith(Parameterized.class) -public class ConfigTest { - - private final Boolean valid; - private final String file; - public Logger LOGGER = LoggerFactory.getLogger(getClass()); - - @Parameterized.Parameters - public static Collection data(){ - Collection testData = new ArrayList(); - testData.add(new Object[]{"src/test/resources/iguana.yml", false}); - testData.add(new Object[]{"src/test/resources/iguana.json", false}); - testData.add(new Object[]{"src/test/resources/iguana-valid.yml", true}); - testData.add(new Object[]{"src/test/resources/iguana-valid.json", true}); - return testData; - } - - public ConfigTest(String file, Boolean valid){ - this.file=file; - this.valid=valid; - } - - @Test - public void checkValidity() throws IOException { - IguanaConfig config = IguanaConfigFactory.parse(new File(file)); - if(valid){ - assertNotNull(config); - } - else { - assertNull(config); - } - config = IguanaConfigFactory.parse(new File(file), false); - assertNotNull(config); - } - - - -} diff --git a/src/test/java/org/aksw/iguana/cc/config/WorkflowTest.java b/src/test/java/org/aksw/iguana/cc/config/WorkflowTest.java deleted file mode 100644 index e9e107919..000000000 --- a/src/test/java/org/aksw/iguana/cc/config/WorkflowTest.java +++ /dev/null @@ -1,123 +0,0 @@ -package org.aksw.iguana.cc.config; - -import org.aksw.iguana.cc.tasks.MockupStorage; -import org.aksw.iguana.cc.tasks.stresstest.metrics.Metric; -import org.aksw.iguana.cc.tasks.stresstest.metrics.MetricManager; -import org.aksw.iguana.cc.tasks.stresstest.metrics.impl.*; -import org.aksw.iguana.cc.tasks.stresstest.storage.Storage; -import org.aksw.iguana.cc.tasks.stresstest.storage.StorageManager; -import org.aksw.iguana.cc.tasks.stresstest.storage.impl.NTFileStorage; -import org.apache.commons.io.FileUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; -import java.util.*; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -public class WorkflowTest { - private String file = "src/test/resources/config/mockupworkflow.yml"; - private String noDefaultFile = "src/test/resources/config/mockupworkflow-no-default.yml"; - private String preFile = "pre-shouldNotExist.txt"; - private String postFile = "post-shouldNotExist.txt"; - - private String expectedPreContent="TestSystem DatasetName testfile.txt\nTestSystem2 DatasetName testfile.txt\nTestSystem DatasetName2 testfile2.txt\nTestSystem2 DatasetName2 testfile2.txt\n"; - private String expectedPostContent="testfile.txt DatasetName TestSystem\ntestfile.txt DatasetName TestSystem2\ntestfile2.txt DatasetName2 TestSystem\ntestfile2.txt DatasetName2 TestSystem2\n"; - - @After - @Before - public void cleanUp(){ - File pre = new File(preFile); - File post = new File(postFile); - pre.delete(); - post.delete(); - StorageManager storageManager = StorageManager.getInstance(); - storageManager.getStorages().clear(); - MetricManager.setMetrics(new ArrayList<>()); - } - - @Test - public void hooks() throws IOException { - IguanaConfig config = IguanaConfigFactory.parse(new File(noDefaultFile), false); - //test if workflow was correct - config.start(); - File pre = new File(preFile); - File post = new File(postFile); - - String preContent = FileUtils.readFileToString(pre, "UTF-8"); - String postContent = FileUtils.readFileToString(post, "UTF-8"); - assertEquals(expectedPreContent, preContent); - assertEquals(expectedPostContent, postContent); - - } - - @Test - public void workflowTest() throws IOException { - IguanaConfig config = IguanaConfigFactory.parse(new File(file), false); - //test if workflow was correct - config.start(); - StorageManager storageManager = StorageManager.getInstance(); - Set storages = storageManager.getStorages(); - assertEquals(1, storages.size()); - Storage s = storages.iterator().next(); - assertTrue(s instanceof MockupStorage); - } - - @Test - public void noDefaultTest() throws IOException { - IguanaConfig config = IguanaConfigFactory.parse(new File(noDefaultFile), false); - //test if correct defaults were loaded - config.start(); - StorageManager storageManager = StorageManager.getInstance(); - Set storages = storageManager.getStorages(); - assertEquals(1, storages.size()); - Storage s = storages.iterator().next(); - assertTrue(s instanceof MockupStorage); - - List metrics = MetricManager.getMetrics(); - assertEquals(2, metrics.size()); - Set> seen = new HashSet<>(); - for(Metric m : metrics){ - seen.add(m.getClass()); - } - assertEquals(2, seen.size()); - - assertTrue(seen.contains(QMPH.class)); - assertTrue(seen.contains(QPS.class)); - } - - @Test - public void initTest() throws IOException { - String file = "src/test/resources/config/mockupworkflow-default.yml"; - IguanaConfig config = IguanaConfigFactory.parse(new File(file), false); - //test if correct defaults were loaded - config.start(); - StorageManager storageManager = StorageManager.getInstance(); - Set storages = storageManager.getStorages(); - assertEquals(1, storages.size()); - Storage s = storages.iterator().next(); - assertTrue(s instanceof NTFileStorage); - File del = new File(((NTFileStorage)s).getFileName()); - del.delete(); - - List metrics = MetricManager.getMetrics(); - assertEquals(6, metrics.size()); - Set> seen = new HashSet<>(); - for(Metric m : metrics){ - seen.add(m.getClass()); - } - assertEquals(6, seen.size()); - - assertTrue(seen.contains(QMPH.class)); - assertTrue(seen.contains(QPS.class)); - assertTrue(seen.contains(AvgQPS.class)); - assertTrue(seen.contains(NoQPH.class)); - assertTrue(seen.contains(NoQ.class)); - assertTrue(seen.contains(AggregatedExecutionStatistics.class)); - } - -} diff --git a/src/test/java/org/aksw/iguana/cc/config/elements/ConnectionConfigTest.java b/src/test/java/org/aksw/iguana/cc/config/elements/ConnectionConfigTest.java new file mode 100644 index 000000000..32034d23e --- /dev/null +++ b/src/test/java/org/aksw/iguana/cc/config/elements/ConnectionConfigTest.java @@ -0,0 +1,185 @@ +package org.aksw.iguana.cc.config.elements; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.net.URI; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.*; + +class ConnectionConfigTest { + private final ObjectMapper mapper = new ObjectMapper(); + + private static Stream testDeserializationData() { + return Stream.of( + Arguments.of(new ConnectionConfig( + "endpoint01", + "0.1", + null, + URI.create("http://example.com/sparql"), + null, null, null), + """ + { + "name":"endpoint01", + "endpoint":"http://example.com/sparql", + "version":"0.1", + "authentication": null, + "updateEndpoint":null, + "updateAuthentication":null, + "dataset":null + } + """ + ), + Arguments.of(new ConnectionConfig( + "endpoint01", + "0.1", + new DatasetConfig("MyData", "some.ttl"), + URI.create("http://example.com/sparql"), + null, null, null), + """ + {"name":"endpoint01","endpoint":"http://example.com/sparql","version":"0.1","authentication": null,"updateEndpoint":null,"dataset":{"name":"MyData","file":"some.ttl"}, "updateAuthentication": null} + """ + ), + Arguments.of(new ConnectionConfig( // test default values + "endpoint01", + null, + null, + URI.create("http://example.com/sparql"), + null, null, null), + """ + { + "name":"endpoint01", + "endpoint":"http://example.com/sparql" + } + """ + ), + Arguments.of(new ConnectionConfig( // test setting everything + "endpoint01", + "v2", + new DatasetConfig("dataset1", "some.ttl"), + URI.create("http://example.com/sparql"), + new ConnectionConfig.Authentication("user", "pass"), + URI.create("http://example.com/update"), + new ConnectionConfig.Authentication("user_update", "pass_update")), + """ + { + "name":"endpoint01", + "version": "v2", + "endpoint":"http://example.com/sparql", + "authentication": { + "user": "user", + "password": "pass" + }, + "updateEndpoint": "http://example.com/update", + "updateAuthentication": { + "user": "user_update", + "password": "pass_update" + }, + "dataset": { + "name": "dataset1", + "file": "some.ttl" + } + } + """ + ) + ); + } + + private static Stream testSerializationData() { + return Stream.of( + Arguments.of(new ConnectionConfig( + "endpoint01", + "0.1", + null, + URI.create("http://example.com/sparql"), + null, null, null), + """ + { + "name":"endpoint01", + "endpoint":"http://example.com/sparql", + "version":"0.1", + "authentication": null, + "updateEndpoint":null, + "updateAuthentication":null, + "dataset":null + } + """ + ), + Arguments.of(new ConnectionConfig( + "endpoint01", + "0.1", + new DatasetConfig("MyData", "some.ttl"), + URI.create("http://example.com/sparql"), + null, null, null), + """ + {"name":"endpoint01","endpoint":"http://example.com/sparql","version":"0.1","authentication": null,"updateEndpoint":null,"dataset":{"name":"MyData","file":"some.ttl"}, "updateAuthentication": null} + """ + ), + Arguments.of(new ConnectionConfig( // test default values + "endpoint01", + null, + null, + URI.create("http://example.com/sparql"), + null, null, null), + """ + { + "name":"endpoint01", + "endpoint":"http://example.com/sparql", + "version": null, + "dataset": null, + "authentication": null, + "updateAuthentication": null, + "updateEndpoint": null + } + """ + ), + Arguments.of(new ConnectionConfig( // test setting everything + "endpoint01", + "v2", + new DatasetConfig("dataset1", "some.ttl"), + URI.create("http://example.com/sparql"), + new ConnectionConfig.Authentication("user", "pass"), + URI.create("http://example.com/update"), + new ConnectionConfig.Authentication("user_update", "pass_update")), + """ + { + "name":"endpoint01", + "version": "v2", + "endpoint":"http://example.com/sparql", + "authentication": { + "user": "user", + "password": "pass" + }, + "updateEndpoint": "http://example.com/update", + "updateAuthentication": { + "user": "user_update", + "password": "pass_update" + }, + "dataset": { + "name": "dataset1", + "file": "some.ttl" + } + } + """ + ) + ); + } + + @ParameterizedTest + @MethodSource("testSerializationData") + public void testSerialization(ConnectionConfig config, String expectedJson) throws Exception { + final String actual = mapper.writeValueAsString(config); + assertEquals(mapper.readTree(expectedJson), mapper.readTree(actual)); + } + + @ParameterizedTest + @MethodSource("testDeserializationData") + public void testDeserialization(ConnectionConfig expected, String json) throws Exception { + final var actual = mapper.readValue(json, ConnectionConfig.class); + assertEquals(expected, actual); + } +} \ No newline at end of file diff --git a/src/test/java/org/aksw/iguana/cc/config/elements/DatasetConfigTest.java b/src/test/java/org/aksw/iguana/cc/config/elements/DatasetConfigTest.java new file mode 100644 index 000000000..9d09b7cb6 --- /dev/null +++ b/src/test/java/org/aksw/iguana/cc/config/elements/DatasetConfigTest.java @@ -0,0 +1,39 @@ +package org.aksw.iguana.cc.config.elements; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.*; + +class DatasetConfigTest { + + private final ObjectMapper mapper = new ObjectMapper(); + + private static Stream testData() { + return Stream.of( + Arguments.of( + new DatasetConfig("MyData", "some.ttl"), + """ + {"name":"MyData","file":"some.ttl"} + """ + ), + Arguments.of( + new DatasetConfig("MyData", null), + """ + {"name":"MyData"} + """ + ) + ); + } + + @ParameterizedTest + @MethodSource("testData") + public void testDeserialization(DatasetConfig expectedConfig, String json) throws Exception { + final var actualConfig = mapper.readValue(json, DatasetConfig.class); + assertEquals(expectedConfig, actualConfig); + } +} \ No newline at end of file diff --git a/src/test/java/org/aksw/iguana/cc/config/elements/StorageConfigTest.java b/src/test/java/org/aksw/iguana/cc/config/elements/StorageConfigTest.java new file mode 100644 index 000000000..0c99a46dd --- /dev/null +++ b/src/test/java/org/aksw/iguana/cc/config/elements/StorageConfigTest.java @@ -0,0 +1,51 @@ +package org.aksw.iguana.cc.config.elements; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.aksw.iguana.cc.storage.impl.CSVStorage; +import org.aksw.iguana.cc.storage.impl.RDFFileStorage; +import org.aksw.iguana.cc.storage.impl.TriplestoreStorage; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.*; + +class StorageConfigTest { + private final ObjectMapper mapper = new ObjectMapper(); + + private static Stream testData() { + return Stream.of( + Arguments.of(new RDFFileStorage.Config("some.ttl"), + """ + {"type":"rdf file","path":"some.ttl"} + """ + ), + Arguments.of(new CSVStorage.Config("csv_results/"), + """ + {"type":"csv file","directory":"csv_results/"} + """ + ), + Arguments.of(new TriplestoreStorage.Config("http://example.com/sparql", "user", "pass", "http://example.com/"), + """ + {"type":"triplestore","endpoint":"http://example.com/sparql", "user": "user", "password": "pass", "baseUri": "http://example.com/"} + """ + ) + ); + } + + @ParameterizedTest + @MethodSource("testData") + public void testSerialization(StorageConfig config, String expectedJson) throws Exception { + final String actual = mapper.writeValueAsString(config); + assertEquals(mapper.readTree(expectedJson), mapper.readTree(actual)); + } + + @ParameterizedTest + @MethodSource("testData") + public void testDeserialization(StorageConfig expectedConfig, String json) throws Exception { + final var actualConfig = mapper.readValue(json, StorageConfig.class); + assertEquals(expectedConfig, actualConfig); + } +} \ No newline at end of file diff --git a/src/test/java/org/aksw/iguana/cc/lang/MockCloseableHttpResponse.java b/src/test/java/org/aksw/iguana/cc/lang/MockCloseableHttpResponse.java deleted file mode 100644 index 5d7fc06e4..000000000 --- a/src/test/java/org/aksw/iguana/cc/lang/MockCloseableHttpResponse.java +++ /dev/null @@ -1,49 +0,0 @@ -package org.aksw.iguana.cc.lang; - -import org.apache.http.HttpStatus; -import org.apache.http.ProtocolVersion; -import org.apache.http.ReasonPhraseCatalog; -import org.apache.http.StatusLine; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.entity.BasicHttpEntity; -import org.apache.http.message.BasicHttpResponse; -import org.apache.http.message.BasicStatusLine; - -import java.io.*; -import java.net.URL; -import java.util.Locale; - -public class MockCloseableHttpResponse extends BasicHttpResponse implements CloseableHttpResponse { - - public MockCloseableHttpResponse(StatusLine statusline, ReasonPhraseCatalog catalog, Locale locale) { - super(statusline, catalog, locale); - } - - public MockCloseableHttpResponse(StatusLine statusline) { - super(statusline); - } - - public MockCloseableHttpResponse(ProtocolVersion ver, int code, String reason) { - super(ver, code, reason); - } - - @Override - public void close() throws IOException { - - } - - public static CloseableHttpResponse buildMockResponse(String data, String contentType) throws FileNotFoundException, UnsupportedEncodingException { - ProtocolVersion protocolVersion = new ProtocolVersion("HTTP", 1, 1); - String reasonPhrase = "OK"; - StatusLine statusline = new BasicStatusLine(protocolVersion, HttpStatus.SC_OK, reasonPhrase); - MockCloseableHttpResponse mockResponse = new MockCloseableHttpResponse(statusline); - BasicHttpEntity entity = new BasicHttpEntity(); - entity.setContentType(contentType); - //entity.setContentType(contentType); - URL url = Thread.currentThread().getContextClassLoader().getResource("response.txt"); - InputStream instream = new ByteArrayInputStream(data.getBytes()); - entity.setContent(instream); - mockResponse.setEntity(entity); - return mockResponse; - } -} diff --git a/src/test/java/org/aksw/iguana/cc/lang/RDFLanguageProcessorTest.java b/src/test/java/org/aksw/iguana/cc/lang/RDFLanguageProcessorTest.java deleted file mode 100644 index 37bb4176d..000000000 --- a/src/test/java/org/aksw/iguana/cc/lang/RDFLanguageProcessorTest.java +++ /dev/null @@ -1,65 +0,0 @@ -package org.aksw.iguana.cc.lang; - -import org.aksw.iguana.cc.lang.impl.RDFLanguageProcessor; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.jena.rdf.model.Model; -import org.apache.jena.rdf.model.ModelFactory; -import org.apache.jena.rdf.model.ResourceFactory; -import org.apache.jena.riot.Lang; -import org.json.simple.parser.ParseException; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.xml.sax.SAXException; - -import javax.xml.parsers.ParserConfigurationException; -import java.io.IOException; -import java.io.StringWriter; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.Collection; - -import static org.junit.Assert.assertEquals; - -@RunWith(Parameterized.class) -public class RDFLanguageProcessorTest { - - private static Logger LOGGER = LoggerFactory.getLogger(RDFLanguageProcessorTest.class); - private final Lang lang; - private final Model m; - - @Parameterized.Parameters - public static Collection data() throws IllegalAccessException { - Collection testData = new ArrayList(); - for(Field langField : Lang.class.getFields()) { - Lang susLang = (Lang)langField.get(Lang.class); - if(susLang.equals(Lang.RDFTHRIFT) || susLang.equals(Lang.TRIX) || susLang.equals(Lang.SHACLC) || susLang.equals(Lang.TSV) || susLang.equals(Lang.CSV) || susLang.equals(Lang.RDFNULL)) { - //cannot test them as model doesn't allow them to write - continue; - } - testData.add(new Object[]{susLang}); - } - return testData; - } - - public RDFLanguageProcessorTest(Lang lang){ - this.lang = lang; - this.m = ModelFactory.createDefaultModel(); - m.add(ResourceFactory.createResource("uri://test"), ResourceFactory.createProperty("uri://prop1"), "abc"); - m.add(ResourceFactory.createResource("uri://test"), ResourceFactory.createProperty("uri://prop2"), "abc2"); - LOGGER.info("Testing Lanuage {} Content-Type: {}", lang.getName(), lang.getContentType()); - } - - @Test - public void testCorrectModel() throws IOException, ParserConfigurationException, SAXException, ParseException { - StringWriter sw = new StringWriter(); - m.write(sw, lang.getName(), null); - CloseableHttpResponse response = MockCloseableHttpResponse.buildMockResponse(sw.toString(), lang.getContentType().getContentTypeStr()); - RDFLanguageProcessor processor = new RDFLanguageProcessor(); - assertEquals(2, processor.getResultSize(response).longValue()); - } - - -} diff --git a/src/test/java/org/aksw/iguana/cc/lang/SPARQLLanguageProcessorTest.java b/src/test/java/org/aksw/iguana/cc/lang/SPARQLLanguageProcessorTest.java deleted file mode 100644 index 43a5da094..000000000 --- a/src/test/java/org/aksw/iguana/cc/lang/SPARQLLanguageProcessorTest.java +++ /dev/null @@ -1,144 +0,0 @@ -package org.aksw.iguana.cc.lang; - -import org.aksw.iguana.cc.lang.impl.SPARQLLanguageProcessor; -import org.apache.jena.ext.com.google.common.collect.Lists; -import org.apache.jena.query.Query; -import org.apache.jena.query.QueryFactory; -import org.apache.jena.rdf.model.Model; -import org.apache.jena.rdf.model.ModelFactory; -import org.json.simple.parser.ParseException; -import org.junit.Test; -import org.xml.sax.SAXException; - -import javax.xml.parsers.ParserConfigurationException; -import java.io.ByteArrayOutputStream; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.nio.charset.StandardCharsets; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -public class SPARQLLanguageProcessorTest { - - private String jsonResult = "{\n" + - " \"head\": { \"vars\": [ \"book\" , \"title\" ]\n" + - " } ,\n" + - " \"results\": { \n" + - " \"bindings\": [\n" + - " {\n" + - " \"book\": { \"type\": \"uri\" , \"value\": \"http://example.org/book/book3\" } ,\n" + - " \"title\": { \"type\": \"literal\" , \"value\": \"Example Book 3\" }\n" + - " } ,\n" + - " {\n" + - " \"book\": { \"type\": \"uri\" , \"value\": \"http://example.org/book/book2\" } ,\n" + - " \"title\": { \"type\": \"literal\" , \"value\": \"Example Book 2\" }\n" + - " } ,\n" + - " {\n" + - " \"book\": { \"type\": \"uri\" , \"value\": \"http://example.org/book/book1\" } ,\n" + - " \"title\": { \"type\": \"literal\" , \"value\": \"Example Book 1\" }\n" + - " }\n" + - " ]\n" + - " }\n" + - "}"; - private String xmlResult = "\n" + - "\n" + - " \n" + - " \n" + - " \n" + - " \n" + - "\n" + - " \n" + - " \n" + - " test1\n" + - " ... \n" + - " \n" + - "\n" + - " \n" + - " test2\n" + - " ... \n" + - " \n" + - " \n" + - " \n" + - "\n" + - ""; - - - - - @Test - public void checkJSON() throws ParseException, IOException { - ByteArrayOutputStream bbaos = new ByteArrayOutputStream(); - bbaos.write(jsonResult.getBytes()); - assertEquals(3, SPARQLLanguageProcessor.getJsonResultSize(bbaos)); - //test if valid json response provide 0 bindings - try { - //check if invalid json throws exception - bbaos = new ByteArrayOutputStream(); - bbaos.write("{ \"a\": \"b\"}".getBytes()); - SPARQLLanguageProcessor.getJsonResultSize(bbaos); - assertTrue("Should have thrown an error", false); - }catch(Exception e){ - assertTrue(true); - } - try { - //check if invalid json throws exception - bbaos = new ByteArrayOutputStream(); - bbaos.write("{ \"a\": \"b\"".getBytes()); - SPARQLLanguageProcessor.getJsonResultSize(bbaos); - assertTrue("Should have thrown an error", false); - }catch(Exception e){ - assertTrue(true); - } - } - - @Test - public void checkXML() throws IOException, SAXException, ParserConfigurationException { - ByteArrayOutputStream bbaos = new ByteArrayOutputStream(); - bbaos.write(xmlResult.getBytes(StandardCharsets.UTF_8)); - assertEquals(2, SPARQLLanguageProcessor.getXmlResultSize(bbaos)); - //test if valid xml response provide 0 bindings - try { - //check if invalid xml throws exception - bbaos = new ByteArrayOutputStream(); - bbaos.write("b".getBytes()); - SPARQLLanguageProcessor.getJsonResultSize(bbaos); - assertTrue("Should have thrown an error", false); - }catch(Exception e){ - assertTrue(true); - } - try { - //check if invalid xml throws exception - bbaos = new ByteArrayOutputStream(); - bbaos.write("{ \"a\": \"b\"".getBytes()); - SPARQLLanguageProcessor.getJsonResultSize(bbaos); - assertTrue("Should have thrown an error", false); - }catch(Exception e){ - assertTrue(true); - } - } - - @Test - public void checkResultSize() throws IOException, ParserConfigurationException, SAXException, ParseException { - SPARQLLanguageProcessor languageProcessor = new SPARQLLanguageProcessor(); - assertEquals(3, languageProcessor.getResultSize(MockCloseableHttpResponse.buildMockResponse(jsonResult, SPARQLLanguageProcessor.QUERY_RESULT_TYPE_JSON)).longValue()); - assertEquals(2, languageProcessor.getResultSize(MockCloseableHttpResponse.buildMockResponse(xmlResult, SPARQLLanguageProcessor.QUERY_RESULT_TYPE_XML)).longValue()); - assertEquals(4, languageProcessor.getResultSize(MockCloseableHttpResponse.buildMockResponse("a\na\na\nb", "text/plain")).longValue()); - } - - - @Test - public void checkGeneratedStatsModel() throws IOException { - Query q = QueryFactory.create("SELECT * {?s ?p ?o. ?o ?q ?t. FILTER(?t = \"abc\")} GROUP BY ?s"); - QueryWrapper wrapped = new QueryWrapper(q, "abc0"); - SPARQLLanguageProcessor languageProcessor = new SPARQLLanguageProcessor(); - Model actual = languageProcessor.generateTripleStats(Lists.newArrayList(wrapped),"query","1/1/2"); - Model expected = ModelFactory.createDefaultModel(); - expected.read(new FileReader("src/test/resources/querystats.nt"), null, "N-TRIPLE"); - assertEquals(expected.size(), actual.size()); - expected.remove(actual); - actual.write(new FileWriter("test2.nt"), "N-TRIPLE"); - assertEquals(0, expected.size()); - } -} diff --git a/src/test/java/org/aksw/iguana/cc/mockup/MockupConnection.java b/src/test/java/org/aksw/iguana/cc/mockup/MockupConnection.java new file mode 100644 index 000000000..3e6d7bb05 --- /dev/null +++ b/src/test/java/org/aksw/iguana/cc/mockup/MockupConnection.java @@ -0,0 +1,19 @@ +package org.aksw.iguana.cc.mockup; + +import org.aksw.iguana.cc.config.elements.ConnectionConfig; + +import java.net.URI; + +public class MockupConnection { + + /** + * Creates a connection config with the given parameters + * + * @param name The name of the connection + * @param endpoint The endpoint of the connection + * @param datasetName The name of the dataset + */ + public static ConnectionConfig createConnectionConfig(String name, String datasetName, String endpoint) { + return new ConnectionConfig(name, "", null, URI.create(endpoint), null, null, null); + } +} diff --git a/src/test/java/org/aksw/iguana/cc/mockup/MockupQueryHandler.java b/src/test/java/org/aksw/iguana/cc/mockup/MockupQueryHandler.java new file mode 100644 index 000000000..6988f0ab9 --- /dev/null +++ b/src/test/java/org/aksw/iguana/cc/mockup/MockupQueryHandler.java @@ -0,0 +1,41 @@ +package org.aksw.iguana.cc.mockup; + +import org.aksw.iguana.cc.query.handler.QueryHandler; +import org.aksw.iguana.cc.query.selector.QuerySelector; +import org.aksw.iguana.cc.query.selector.impl.LinearQuerySelector; + + +public class MockupQueryHandler extends QueryHandler { + private final int id; + private final int queryNumber; + + public MockupQueryHandler(int id, int queryNumber) { + super(); + this.queryNumber = queryNumber; + this.id = id; + } + + @Override + public String getQueryId(int i) { + return "MockQueryHandler" + this.id + ":" + i; + } + + @Override + public String[] getAllQueryIds() { + String[] out = new String[queryNumber]; + for (int i = 0; i < queryNumber; i++) { + out[i] = getQueryId(i); + } + return out; + } + + @Override + public int getQueryCount() { + return queryNumber; + } + + @Override + public QuerySelector getQuerySelectorInstance() { + return new LinearQuerySelector(queryNumber); + } +} diff --git a/src/test/java/org/aksw/iguana/cc/mockup/MockupStorage.java b/src/test/java/org/aksw/iguana/cc/mockup/MockupStorage.java new file mode 100644 index 000000000..ef15adf9e --- /dev/null +++ b/src/test/java/org/aksw/iguana/cc/mockup/MockupStorage.java @@ -0,0 +1,18 @@ +package org.aksw.iguana.cc.mockup; + +import org.aksw.iguana.cc.storage.Storage; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; + +public class MockupStorage implements Storage { + private Model resultModel = ModelFactory.createDefaultModel(); + + @Override + public void storeResult(Model data) { + resultModel = data; + } + + public Model getResultModel() { + return resultModel; + } +} diff --git a/src/test/java/org/aksw/iguana/cc/mockup/MockupWorker.java b/src/test/java/org/aksw/iguana/cc/mockup/MockupWorker.java new file mode 100644 index 000000000..9950c9f9d --- /dev/null +++ b/src/test/java/org/aksw/iguana/cc/mockup/MockupWorker.java @@ -0,0 +1,118 @@ +package org.aksw.iguana.cc.mockup; + +import org.aksw.iguana.cc.config.elements.ConnectionConfig; +import org.aksw.iguana.cc.config.elements.DatasetConfig; +import org.aksw.iguana.cc.query.handler.QueryHandler; +import org.aksw.iguana.cc.worker.HttpWorker; + +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.CompletableFuture; + +public class MockupWorker extends HttpWorker { + public record Config( + CompletionTarget completionTarget, + String acceptHeader, + Integer number, + Boolean parseResults, + QueryHandler queries, + ConnectionConfig connection, + Duration timeout + ) implements HttpWorker.Config {} + + + /** + * All values except the workerID and queries may be null. I recommend to use the MockupQueryHandler. + * I would also recommend to set the connection, if you want to use the StresstestResultProcessor. + */ + public MockupWorker(long workerID, CompletionTarget target, String acceptHeader, Integer number, Boolean parseResults, QueryHandler queries, ConnectionConfig connection, Duration timeout) { + super(workerID, null, new Config( + target, + acceptHeader, + number, + parseResults, + queries, + connection, + timeout + ) + ); + } + + /** + * All other values will be set to null. This is the bare minimum to make it work with the StresstestResultProcessor. + */ + public MockupWorker(long workerID, QueryHandler queries, String connectionName, String connectionVersion, String datasetName, Duration timeout) { + super(workerID, null, new Config( + null, + null, + null, + null, + queries, + new ConnectionConfig(connectionName, connectionVersion, new DatasetConfig(datasetName, null), null, null, null, null), + timeout + )); + } + + @Override + public CompletableFuture start() { + return null; + } + + public static List createWorkerResults(QueryHandler queries, List workers) { + final var startTime = ZonedDateTime.of(2023, 10, 11, 14, 14, 10, 0, ZoneId.of("UTC")); + final var endTime = ZonedDateTime.of(2023, 10, 12, 15, 15, 15, 0, ZoneId.of("UTC")); + + final var queryNumber = queries.getQueryCount(); + + Instant time = Instant.parse("2023-10-21T20:48:06.399Z"); + + final var results = new ArrayList(); + for (var worker : workers) { + final var exectutionStats = new ArrayList(); + for (int queryID = 0; queryID < queryNumber; queryID++) { + // successful execution + final var sucHttpCode = Optional.of(200); + final var sucDuration = Duration.ofSeconds(2); + final var sucLength = OptionalLong.of(1000); + final var responseBodyHash = OptionalLong.of(123); + time = time.plusSeconds(1); + exectutionStats.add(new ExecutionStats(queryID, time, sucDuration, sucHttpCode, sucLength, responseBodyHash, Optional.empty())); + + // failed execution (http error) + var failHttpCode = Optional.of(404); + var failDuration = Duration.ofMillis(500); + var failLength = OptionalLong.empty(); + var failResponseBodyHash = OptionalLong.empty(); + var failException = new Exception("httperror"); + time = time.plusSeconds(1); + exectutionStats.add(new ExecutionStats(queryID, time, failDuration, failHttpCode, failLength, failResponseBodyHash, Optional.of(failException))); + + // failed execution + failHttpCode = Optional.of(200); + failDuration = Duration.ofSeconds(5); + failLength = OptionalLong.empty(); + failResponseBodyHash = OptionalLong.of(456); + failException = new Exception("io_exception"); + time = time.plusSeconds(1); + exectutionStats.add(new ExecutionStats(queryID, time, failDuration, failHttpCode, failLength, failResponseBodyHash, Optional.of(failException))); + } + results.add(new Result(worker.getWorkerID(), exectutionStats, startTime, endTime)); + } + return results; + } + + public static List createWorkers(int idOffset, int workerNumber, QueryHandler queries, String connectionName, String connectionVersion, String datasetName) { + final var workers = new ArrayList(); + for (int i = idOffset; i < workerNumber + idOffset; i++) { + workers.add(new MockupWorker(i, queries, connectionName, connectionVersion, datasetName, Duration.ofSeconds(2))); + } + return workers; + } + +} diff --git a/src/test/java/org/aksw/iguana/cc/model/QueryResultHashKeyTest.java b/src/test/java/org/aksw/iguana/cc/model/QueryResultHashKeyTest.java deleted file mode 100644 index e2c1cc538..000000000 --- a/src/test/java/org/aksw/iguana/cc/model/QueryResultHashKeyTest.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.aksw.iguana.cc.model; - -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.UUID; - -import static org.junit.Assert.*; - -@RunWith(Parameterized.class) -public class QueryResultHashKeyTest { - - - private final String queryID; - private final long uniqueKey; - - @Parameterized.Parameters - public static Collection data(){ - Collection testData = new ArrayList(); - testData.add(new Object[]{"sparql1", 1}); - testData.add(new Object[]{"sparql2", 122323l}); - testData.add(new Object[]{"update", 122323l}); - testData.add(new Object[]{UUID.randomUUID().toString(), 122323l}); - testData.add(new Object[]{"", 0}); - return testData; - } - - public QueryResultHashKeyTest(String queryID, long uniqueKey){ - this.queryID=queryID; - this.uniqueKey=uniqueKey; - } - - @Test - public void checkEquals(){ - QueryResultHashKey key = new QueryResultHashKey(queryID, uniqueKey); - assertTrue(key.equals(key)); - assertFalse(key.equals(null)); - assertFalse(key.equals(queryID)); - assertFalse(key.equals(uniqueKey)); - QueryResultHashKey that = new QueryResultHashKey(queryID, uniqueKey); - assertEquals(key, that); - that = new QueryResultHashKey(queryID+"abc", uniqueKey); - assertNotEquals(key, that); - that = new QueryResultHashKey(queryID, uniqueKey+1); - assertNotEquals(key, that); - } - - -} diff --git a/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerConfigTest.java b/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerConfigTest.java new file mode 100644 index 000000000..ace718d9f --- /dev/null +++ b/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerConfigTest.java @@ -0,0 +1,141 @@ +package org.aksw.iguana.cc.query.handler; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.*; + +class QueryHandlerConfigTest { + private final ObjectMapper mapper = new ObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_NULL); + + + + private static Stream testDeserializationData() { + return Stream.of( + Arguments.of(new QueryHandler.Config("some.queries", + QueryHandler.Config.Format.FOLDER, + "", + true, + QueryHandler.Config.Order.LINEAR, + 100L, + QueryHandler.Config.Language.SPARQL + ), + """ + {"path":"some.queries","format":"folder","caching":true,"order":"linear","seed": 100, "lang":"SPARQL"} + """ + ), + Arguments.of(new QueryHandler.Config("some.queries", + QueryHandler.Config.Format.ONE_PER_LINE, + "", + true, + QueryHandler.Config.Order.LINEAR, + 0L, + QueryHandler.Config.Language.SPARQL + ), + """ + {"path":"some.queries"} + """ + ), + Arguments.of(new QueryHandler.Config("some.queries", + QueryHandler.Config.Format.FOLDER, + "", + true, + QueryHandler.Config.Order.RANDOM, + 42L, + QueryHandler.Config.Language.SPARQL + ), + """ + {"path":"some.queries","format":"folder","caching":true,"order":"random","seed":42,"lang":"SPARQL"} + """ + ), + Arguments.of(new QueryHandler.Config("some.queries", + QueryHandler.Config.Format.SEPARATOR, + "\n", + true, + QueryHandler.Config.Order.RANDOM, + 42L, + QueryHandler.Config.Language.SPARQL + ), + """ + {"path":"some.queries","format":"separator", "separator": "\\n", "caching":true,"order":"random","seed":42,"lang":"SPARQL"} + """ + ) + ); + } + + private static Stream testSerializationData() { + return Stream.of( + Arguments.of(new QueryHandler.Config("some.queries", + QueryHandler.Config.Format.FOLDER, + "", + true, + QueryHandler.Config.Order.LINEAR, + 100L, + QueryHandler.Config.Language.SPARQL + ), + """ + {"path":"some.queries","separator": "", "format":"folder","caching":true,"order":"linear","seed": 100, "lang":"SPARQL"} + """ + ), + Arguments.of(new QueryHandler.Config("some.queries", + QueryHandler.Config.Format.ONE_PER_LINE, + "", + true, + QueryHandler.Config.Order.LINEAR, + 0L, + QueryHandler.Config.Language.SPARQL + ), + """ + {"path":"some.queries", "format":"one-per-line","separator":"","caching":true,"order":"linear","seed":0,"lang":"SPARQL"} + """ + ), + Arguments.of(new QueryHandler.Config("some.queries", + QueryHandler.Config.Format.FOLDER, + "", + true, + QueryHandler.Config.Order.RANDOM, + 42L, + QueryHandler.Config.Language.SPARQL + ), + """ + {"path":"some.queries","format":"folder","separator":"","caching":true,"order":"random","seed":42,"lang":"SPARQL"} + """ + ), + Arguments.of(new QueryHandler.Config("some.queries", + QueryHandler.Config.Format.SEPARATOR, + "\n", + true, + QueryHandler.Config.Order.RANDOM, + 42L, + QueryHandler.Config.Language.SPARQL + ), + """ + {"path":"some.queries","format":"separator", "separator": "\\n", "caching":true,"order":"random","seed":42,"lang":"SPARQL"} + """ + ) + ); + } + + @ParameterizedTest + @MethodSource("testSerializationData") + public void testSerialisation(QueryHandler.Config config, String expectedJson) throws Exception { + + final String actual = mapper.writeValueAsString(config); + System.out.println(actual); + assertEquals(mapper.readTree(expectedJson), mapper.readTree(actual)); + } + + @ParameterizedTest + @MethodSource("testDeserializationData") + public void testDeserialization(QueryHandler.Config expected, String json) throws Exception { + final var actual = mapper.readValue(json, QueryHandler.Config.class); + + assertEquals(expected, actual); + } +} \ No newline at end of file diff --git a/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerTest.java b/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerTest.java index 04ec00596..4235c1df2 100644 --- a/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerTest.java +++ b/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerTest.java @@ -1,152 +1,185 @@ package org.aksw.iguana.cc.query.handler; -import org.aksw.iguana.cc.utils.ServerMock; -import org.apache.commons.io.FileUtils; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.simpleframework.http.core.ContainerServer; -import org.simpleframework.transport.connect.SocketConnection; - -import java.io.File; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.aksw.iguana.cc.query.selector.impl.LinearQuerySelector; +import org.aksw.iguana.cc.query.selector.impl.RandomQuerySelector; +import org.aksw.iguana.cc.query.source.QuerySource; +import org.aksw.iguana.cc.query.source.impl.FileLineQuerySource; +import org.aksw.iguana.cc.query.source.impl.FileSeparatorQuerySource; +import org.aksw.iguana.cc.query.source.impl.FolderQuerySource; +import org.aksw.iguana.cc.query.source.impl.FolderQuerySourceTest; +import org.junit.jupiter.api.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + import java.io.IOException; -import java.net.InetSocketAddress; -import java.net.SocketAddress; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.*; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.*; -@RunWith(Parameterized.class) public class QueryHandlerTest { - private static final int FAST_SERVER_PORT = 8024; - private static final String CACHE_FOLDER = UUID.randomUUID().toString(); - private static ContainerServer fastServer; - private static SocketConnection fastConnection; + static Path parentFolder; + static Path tempDir; + static Path tempFileSep; + static Path tempFileLine; + + static List queries; + static List folderQueries; + + public static List data() { + final var out = new ArrayList(); + final var caching = List.of(true, false); + + for (var cache : caching) { + out.add(Arguments.of(String.format(""" + {"path":"%s","format":"folder","order":"linear","lang":"SPARQL", "caching": %s} + """, tempDir.toString().replaceAll("\\\\", "\\\\\\\\"), cache), + FolderQuerySource.class)); + out.add(Arguments.of(String.format(""" + {"path":"%s","format":"one-per-line","order":"linear","lang":"SPARQL", "caching": %s} + """, tempFileLine.toString().replaceAll("\\\\", "\\\\\\\\"), cache), + FileLineQuerySource.class)); + out.add(Arguments.of(String.format(""" + {"path":"%s","format":"separator", "separator": "\\n###\\n", "order":"linear","lang":"SPARQL", "caching": %s} + """, tempFileSep.toString().replaceAll("\\\\", "\\\\\\\\"), cache), + FileSeparatorQuerySource.class)); + } + + return out; + } - private final QueryHandler queryHandler; - private final Map config; - private final String[] expected; + @BeforeAll + public static void createFolder() throws IOException { + parentFolder = Files.createTempDirectory("iguana-query-handler-test"); + tempDir = Files.createTempDirectory(parentFolder, "folder-query-source-test-dir"); + tempFileSep = Files.createTempFile(parentFolder, "Query", ".txt"); + tempFileLine = Files.createTempFile(parentFolder, "Query", ".txt"); + + queries = new LinkedList<>(); + folderQueries = new LinkedList<>(); + + for (int i = 0; i < 10; i++) { + final Path queryFile = Files.createTempFile(tempDir, "Query", ".txt"); + final String content = UUID.randomUUID().toString(); + Files.writeString(queryFile, content); + Files.writeString(tempFileSep, content + "\n###\n", StandardCharsets.UTF_8, StandardOpenOption.APPEND); + Files.writeString(tempFileLine, content + "\n", StandardCharsets.UTF_8, StandardOpenOption.APPEND); + queries.add(new FolderQuerySourceTest.Query(queryFile, content)); + folderQueries.add(new FolderQuerySourceTest.Query(queryFile, content)); + } + // Queries in the folder are expected in alphabetic order of the file names. + Collections.sort(folderQueries); + } + @AfterAll + public static void removeFolder() throws IOException { + org.apache.commons.io.FileUtils.deleteDirectory(parentFolder.toFile()); + } - public QueryHandlerTest(Map config, String[] expected) { - this.queryHandler = new QueryHandler(config, 0); // workerID 0 results in correct seed for RandomSelector - this.config = config; - this.expected = expected; + @ParameterizedTest + @MethodSource("data") + public void testDeserialization(String json, Class sourceType) throws Exception { + final var mapper = new ObjectMapper(); + QueryHandler queryHandler = assertDoesNotThrow(() -> mapper.readValue(json, QueryHandler.class)); + final var selector = queryHandler.getQuerySelectorInstance(); + assertTrue(selector instanceof LinearQuerySelector); + assertEquals(queries.size(), queryHandler.getQueryCount()); + assertNotEquals(0, queryHandler.hashCode()); + for (int i = 0; i < queryHandler.getQueryCount(); i++) { + final var wrapper = queryHandler.getNextQuery(selector); + assertEquals(i, selector.getCurrentIndex()); + if (FolderQuerySource.class.isAssignableFrom(sourceType)) + assertEquals(folderQueries.get(i).content(), wrapper.query()); + else + assertEquals(queries.get(i).content(), wrapper.query()); + assertEquals(i, wrapper.index()); + } } - @Parameterized.Parameters - public static Collection data() throws IOException { - String le = org.aksw.iguana.cc.utils.FileUtils.getLineEnding("src/test/resources/query/source/queries.txt"); - - String[] opl = new String[]{"QUERY 1 {still query 1}", "QUERY 2 {still query 2}", "QUERY 3 {still query 3}", "QUERY 1 {still query 1}"}; - String[] folder = new String[]{"QUERY 1 {" + le + "still query 1" + le + "}", "QUERY 2 {" + le + "still query 2" + le + "}", "QUERY 3 {" + le + "still query 3" + le + "}", "QUERY 1 {" + le + "still query 1" + le + "}"}; - String[] separator = new String[]{"QUERY 1 {" + le + "still query 1" + le + "}", "QUERY 2 {" + le + "still query 2" + le + "}", "QUERY 3 {" + le + "still query 3" + le + "}", "QUERY 1 {" + le + "still query 1" + le + "}"}; - - Collection testData = new ArrayList<>(); - - // Defaults: one-per-line, caching, linear - Map config0 = new HashMap<>(); - config0.put("location", "src/test/resources/query/source/queries.txt"); - testData.add(new Object[]{config0, opl}); - - // Defaults: caching, linear - Map config1 = new HashMap<>(); - config1.put("location", "src/test/resources/query/source/query-folder"); - config1.put("format", "folder"); - testData.add(new Object[]{config1, folder}); - - // Defaults: separator("###"), caching, linear - Map config2 = new HashMap<>(); - config2.put("location", "src/test/resources/query/source/separated-queries-default.txt"); - config2.put("format", "separator"); - testData.add(new Object[]{config2, separator}); - - Map config3 = new HashMap<>(); - config3.put("location", "src/test/resources/query/source/separated-queries-default.txt"); - Map format3 = new HashMap<>(); - format3.put("separator", "###"); - config3.put("format", format3); - config3.put("caching", false); - config3.put("order", "random"); - testData.add(new Object[]{config3, separator}); - - // Defaults: one-per-line, caching - Map config4 = new HashMap<>(); - config4.put("location", "src/test/resources/query/source/queries.txt"); - Map random4 = new HashMap<>(); - random4.put("seed", 0); - Map order4 = new HashMap<>(); - order4.put("random", random4); - config4.put("order", order4); - testData.add(new Object[]{config4, opl}); - - String[] expectedInstances = new String[]{"SELECT ?book {?book ?o}", "SELECT ?book {?book ?o}", "SELECT ?book {?book ?o}", "SELECT ?book {?book ?o}"}; - Map config5 = new HashMap<>(); - config5.put("location", "src/test/resources/query/pattern-query.txt"); - Map pattern5 = new HashMap<>(); - pattern5.put("endpoint", "http://localhost:8024"); - pattern5.put("outputFolder", CACHE_FOLDER); - config5.put("pattern", pattern5); - testData.add(new Object[]{config5, expectedInstances}); - - - return testData; + @ParameterizedTest + @MethodSource("data") + public void testQueryStreamWrapper(String json, Class sourceType) throws IOException { + final var mapper = new ObjectMapper(); + QueryHandler queryHandler = assertDoesNotThrow(() -> mapper.readValue(json, QueryHandler.class)); + final var selector = queryHandler.getQuerySelectorInstance(); + assertTrue(selector instanceof LinearQuerySelector); + assertEquals(queries.size(), queryHandler.getQueryCount()); + assertNotEquals(0, queryHandler.hashCode()); + for (int i = 0; i < queryHandler.getQueryCount(); i++) { + final var wrapper = queryHandler.getNextQueryStream(selector); + assertEquals(i, selector.getCurrentIndex()); + final var acutalQuery = new String(wrapper.queryInputStream().readAllBytes(), StandardCharsets.UTF_8); + if (FolderQuerySource.class.isAssignableFrom(sourceType)) + assertEquals(folderQueries.get(i).content(), acutalQuery); + else + assertEquals(queries.get(i).content(), acutalQuery); + assertEquals(i, wrapper.index()); + } } - @BeforeClass - public static void startServer() throws IOException { - ServerMock fastServerContainer = new ServerMock(); - fastServer = new ContainerServer(fastServerContainer); - fastConnection = new SocketConnection(fastServer); - SocketAddress address1 = new InetSocketAddress(FAST_SERVER_PORT); - fastConnection.connect(address1); + @ParameterizedTest + @MethodSource("data") + public void testQueryStringWrapper(String json, Class sourceType) throws IOException { + final var mapper = new ObjectMapper(); + QueryHandler queryHandler = assertDoesNotThrow(() -> mapper.readValue(json, QueryHandler.class)); + final var selector = queryHandler.getQuerySelectorInstance(); + assertTrue(selector instanceof LinearQuerySelector); + assertEquals(queries.size(), queryHandler.getQueryCount()); + assertNotEquals(0, queryHandler.hashCode()); + for (int i = 0; i < queryHandler.getQueryCount(); i++) { + final var wrapper = queryHandler.getNextQuery(selector); + assertEquals(i, selector.getCurrentIndex()); + if (FolderQuerySource.class.isAssignableFrom(sourceType)) + assertEquals(folderQueries.get(i).content(), wrapper.query()); + else + assertEquals(queries.get(i).content(), wrapper.query()); + assertEquals(i, wrapper.index()); + } } - @AfterClass - public static void stopServer() throws IOException { - fastConnection.close(); - fastServer.stop(); - FileUtils.deleteDirectory(new File(CACHE_FOLDER)); + @ParameterizedTest + @MethodSource("data") + public void testQueryIDs(String json, Class sourceType) { + final var mapper = new ObjectMapper(); + QueryHandler queryHandler = assertDoesNotThrow(() -> mapper.readValue(json, QueryHandler.class)); + final var selector = queryHandler.getQuerySelectorInstance(); + assertTrue(selector instanceof LinearQuerySelector); + assertEquals(queries.size(), queryHandler.getQueryCount()); + assertNotEquals(0, queryHandler.hashCode()); + final var allQueryIDs = queryHandler.getAllQueryIds(); + for (int i = 0; i < queryHandler.getQueryCount(); i++) { + assertEquals(queryHandler.hashCode() + ":" + i, allQueryIDs[i]); + assertEquals(allQueryIDs[i], queryHandler.getQueryId(i)); + } } @Test - public void getNextQueryTest() throws IOException { - // Assumes, that the order is correct has only stored values for random retrieval - Object order = config.getOrDefault("order", null); - if (order != null) { - Collection queries = new HashSet<>(); - for (int i = 0; i < 4; i++) { - StringBuilder query = new StringBuilder(); - StringBuilder queryID = new StringBuilder(); - this.queryHandler.getNextQuery(query, queryID); - queries.add(query.toString()); + public void testRandomQuerySelectorSeedConsistency() throws IOException { + String[] json = new String[2]; + json[0] = String.format(""" + {"path":"%s","format":"folder","order":"random", "seed": 100,"lang":"SPARQL"} + """, tempDir.toString().replaceAll("\\\\", "\\\\\\\\")); // windows + json[1] = String.format(""" + {"path":"%s","format":"one-per-line","order":"random", "seed": 100,"lang":"SPARQL"} + """, tempFileLine.toString().replaceAll("\\\\", "\\\\\\\\")); // this tests need to different configuration, because instances of the query handler are cached + + final var mapper = new ObjectMapper(); + List[] indices = new ArrayList[2]; + for (int i = 0; i < 2; i++) { + QueryHandler queryHandler = mapper.readValue(json[i], QueryHandler.class); + final var selector = queryHandler.getQuerySelectorInstance(); + assertTrue(selector instanceof RandomQuerySelector); + indices[i] = new ArrayList<>(); + for (int j = 0; j < 100000; j++) { + indices[i].add(selector.getNextIndex()); } - assertTrue(Arrays.asList(this.expected).containsAll(queries)); - return; } - - StringBuilder query = new StringBuilder(); - StringBuilder queryID = new StringBuilder(); - this.queryHandler.getNextQuery(query, queryID); - assertEquals(this.expected[0], query.toString()); - - query = new StringBuilder(); - queryID = new StringBuilder(); - this.queryHandler.getNextQuery(query, queryID); - assertEquals(this.expected[1], query.toString()); - - query = new StringBuilder(); - queryID = new StringBuilder(); - this.queryHandler.getNextQuery(query, queryID); - assertEquals(this.expected[2], query.toString()); - - query = new StringBuilder(); - queryID = new StringBuilder(); - this.queryHandler.getNextQuery(query, queryID); - assertEquals(this.expected[3], query.toString()); + assertEquals(indices[0], indices[1]); } } \ No newline at end of file diff --git a/src/test/java/org/aksw/iguana/cc/query/list/QueryListTest.java b/src/test/java/org/aksw/iguana/cc/query/list/QueryListTest.java new file mode 100644 index 000000000..b1da8ffac --- /dev/null +++ b/src/test/java/org/aksw/iguana/cc/query/list/QueryListTest.java @@ -0,0 +1,142 @@ +package org.aksw.iguana.cc.query.list; + +import org.aksw.iguana.cc.query.list.impl.FileBasedQueryList; +import org.aksw.iguana.cc.query.list.impl.InMemQueryList; +import org.aksw.iguana.cc.query.source.QuerySource; +import org.aksw.iguana.cc.query.source.impl.FileLineQuerySource; +import org.aksw.iguana.cc.query.source.impl.FileSeparatorQuerySource; +import org.aksw.iguana.cc.query.source.impl.FolderQuerySource; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Named; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.*; + +class QueryListTest { + private enum QuerySourceType { + FILE_LINE, + FILE_SEPARATOR, + FOLDER, + } + + static Path tempDir; + static List cachedArguments = null; + + private static QueryList createQueryList(Class queryListClass,QuerySource querySource) { + try { + return (QueryList) queryListClass.getConstructor(QuerySource.class).newInstance(querySource); + } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + throw new RuntimeException(e); + } + } + + @BeforeAll + public static void createFolder() throws IOException { + tempDir = Files.createTempDirectory("iguana-folder-query-source-test-dir"); + } + + @AfterAll + public static void deleteFolder() throws IOException { + FileUtils.deleteDirectory(tempDir.toFile()); + } + + public static List data() throws IOException { + if (cachedArguments != null) + return cachedArguments; + + final var queryListClasses = List.of(InMemQueryList.class, FileBasedQueryList.class); + final var querySources = List.of(QuerySourceType.FILE_SEPARATOR, QuerySourceType.FILE_LINE, QuerySourceType.FOLDER); + final var sizes = List.of(1, 2, 10, 100, 1000); + + final var out = new ArrayList(); + for (var size : sizes) { + for (var querySourceType : querySources) { + for (var queryListClass : queryListClasses) { + final var queries = new ArrayList(); + for (int i = 0; i < size; i++) { + final String queryString = UUID.randomUUID().toString(); + queries.add(queryString); + } + QuerySource querySource = null; + switch (querySourceType) { + case FOLDER -> { + final var queryDir = Files.createTempDirectory(tempDir, "query-dir"); + for (int i = 0; i < size; i++) { + String filePrefix = String.format("Query-%09d.txt", i); // to ensure that the order from the queries List is the same as the order of the files in the folder + final Path queryFile = Files.createTempFile(queryDir, filePrefix, ".txt"); + Files.write(queryFile, queries.get(i).getBytes()); + } + querySource = new FolderQuerySource(queryDir); + } + case FILE_LINE -> { + final var queryFile = Files.createTempFile(tempDir, "Query", ".txt"); + Files.write(queryFile, String.join("\n", queries).getBytes()); + querySource = new FileLineQuerySource(queryFile); + } + case FILE_SEPARATOR -> { + final var queryFile = Files.createTempFile(tempDir, "Query", ".txt"); + Files.write(queryFile, String.join("\n###\n", queries).getBytes()); + querySource = new FileSeparatorQuerySource(queryFile, "\n###\n"); + } + } + String querySourceConfigString = String.format("[ type=%s, size=%d ]", querySourceType, size); + out.add(Arguments.of(Named.of(queryListClass.getSimpleName(), queryListClass), Named.of(querySourceConfigString, querySource), queries)); + } + } + } + cachedArguments = out; + return out; + } + + public void testIllegalArguments() { + assertThrows(NullPointerException.class, () -> new InMemQueryList(null)); + assertThrows(NullPointerException.class, () -> new FileBasedQueryList(null)); + } + + @ParameterizedTest(name = "[{index}] queryListClass={0}, querySourceConfig={1}") + @MethodSource("data") + public void testGetQuery(Class queryListClass, QuerySource querySource, List expectedQueries) throws IOException { + final var queryList = createQueryList(queryListClass, querySource); + for (int i = 0; i < expectedQueries.size(); i++) { + final var expectedQuery = expectedQueries.get(i); + assertEquals(expectedQuery, queryList.getQuery(i)); + } + } + + @ParameterizedTest(name = "[{index}] queryListClass={0}, querySourceConfig={1}") + @MethodSource("data") + public void testGetQueryStream(Class queryListClass, QuerySource querySource, List expectedQueries) throws IOException { + final var queryList = createQueryList(queryListClass, querySource); + for (int i = 0; i < expectedQueries.size(); i++) { + final var expectedQuery = expectedQueries.get(i); + final var queryString = new String(queryList.getQueryStream(i).readAllBytes(), "UTF-8"); + assertEquals(expectedQuery, queryString); + } + } + + @ParameterizedTest(name = "[{index}] queryListClass={0}, querySourceConfig={1}") + @MethodSource("data") + public void testSize(Class queryListClass, QuerySource querySource, List expectedQueries) { + final var queryList = createQueryList(queryListClass, querySource); + assertEquals(expectedQueries.size(), queryList.size()); + } + + @ParameterizedTest(name = "[{index}] queryListClass={0}, querySourceConfig={1}") + @MethodSource("data") + public void testHashcode(Class queryListClass, QuerySource querySource, List expectedQueries) { + final var queryList = createQueryList(queryListClass, querySource); + assertTrue(queryList.hashCode() != 0); + } +} \ No newline at end of file diff --git a/src/test/java/org/aksw/iguana/cc/query/pattern/PatternBasedQueryHandlerTest.java b/src/test/java/org/aksw/iguana/cc/query/pattern/PatternBasedQueryHandlerTest.java deleted file mode 100644 index 2c082531c..000000000 --- a/src/test/java/org/aksw/iguana/cc/query/pattern/PatternBasedQueryHandlerTest.java +++ /dev/null @@ -1,141 +0,0 @@ -package org.aksw.iguana.cc.query.pattern; - -import org.aksw.iguana.cc.query.source.QuerySource; -import org.aksw.iguana.cc.query.source.impl.FileLineQuerySource; -import org.apache.commons.io.FileUtils; -import org.apache.jena.ext.com.google.common.collect.Lists; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.io.File; -import java.io.IOException; -import java.io.PrintWriter; -import java.nio.file.Files; -import java.util.*; -import java.util.stream.Stream; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -@RunWith(Parameterized.class) -public class PatternBasedQueryHandlerTest { - private final String dir = UUID.randomUUID().toString(); - private String[] queryStr; - private File queriesFile; - - public PatternBasedQueryHandlerTest(String[] queryStr) { - this.queryStr = queryStr; - } - - @Parameterized.Parameters - public static Collection data() { - Collection testData = new ArrayList<>(); - testData.add(new Object[]{new String[]{"SELECT * {?s ?p ?o}"}}); - testData.add(new Object[]{new String[]{"SELECT * {?s ?p ?o}"}}); - - return testData; - } - - @Before - public void createFolder() throws IOException { - //File f = new File(this.dir); - //f.mkdir(); - String queryFile = UUID.randomUUID().toString(); - File f = new File(queryFile); - f.createNewFile(); - try (PrintWriter pw = new PrintWriter(f)) { - for (String query : this.queryStr) { - pw.println(query); - } - } - //remove empty lines after printing them, so the expected asserts will correctly assume that the empty limes are ignored - List tmpList = Lists.newArrayList(this.queryStr); - Iterator it = tmpList.iterator(); - while (it.hasNext()) { - if (it.next().isEmpty()) { - it.remove(); - } - } - this.queryStr = tmpList.toArray(new String[]{}); - this.queriesFile = f; - f.deleteOnExit(); - } - - @After - public void removeFolder() throws IOException { - File f = new File(this.dir); - FileUtils.deleteDirectory(f); - } - - - @Test - public void testQueryCreation() throws IOException { - QuerySource originalSource = getQuerySource(); - PatternHandler ph = new PatternHandler(getConfig(), originalSource); - QuerySource qs = ph.generateQuerySource(); - - //check if folder exist this.dir/hashCode/ with |queries| files - int hashcode = originalSource.hashCode(); - File f = new File(this.dir + File.separator + hashcode); - File outDir = new File(this.dir); - assertTrue(outDir.exists()); - assertTrue(outDir.isDirectory()); - assertTrue(f.isFile()); - - assertEquals(1, outDir.listFiles().length); - - int expectedNoOfQueries = this.queryStr.length; - assertEquals(expectedNoOfQueries, qs.size()); - - try (Stream stream = Files.lines(f.toPath())) { - assertEquals(expectedNoOfQueries, stream.count()); - } - - for (int i = 0; i < expectedNoOfQueries; i++) { - assertEquals(this.queryStr[i], qs.getQuery(i)); - } - - FileUtils.deleteDirectory(outDir); - } - - @Test - public void testCaching() throws IOException { - QuerySource originalSource = getQuerySource(); - PatternHandler ph = new PatternHandler(getConfig(), originalSource); - ph.generateQuerySource(); - - int hashcode = originalSource.hashCode(); - File f = new File(this.dir + File.separator + hashcode); - assertTrue(f.exists()); - assertTrue(f.isFile()); - - int contentHash = org.aksw.iguana.cc.utils.FileUtils.getHashcodeFromFileContent(f.getAbsolutePath()); - Map attr = Files.readAttributes(f.toPath(), "basic:creationTime"); - - PatternHandler ph2 = new PatternHandler(getConfig(), originalSource); - ph2.generateQuerySource(); - - int contentHash2 = org.aksw.iguana.cc.utils.FileUtils.getHashcodeFromFileContent(f.getAbsolutePath()); - assertEquals(contentHash, contentHash2); - - Map attr2 = Files.readAttributes(f.toPath(), "basic:creationTime"); - assertEquals(attr.get("creationTime"), attr2.get("creationTime")); - - FileUtils.deleteDirectory(new File(this.dir)); - } - - private Map getConfig() { - Map config = new HashMap<>(); - config.put("endpoint", "http://test.com"); - config.put("outputFolder", this.dir); - config.put("limit", 5); - return config; - } - - private QuerySource getQuerySource() { - return new FileLineQuerySource(this.queriesFile.getAbsolutePath()); - } -} diff --git a/src/test/java/org/aksw/iguana/cc/query/pattern/PatternHandlerTest.java b/src/test/java/org/aksw/iguana/cc/query/pattern/PatternHandlerTest.java deleted file mode 100644 index c618815d9..000000000 --- a/src/test/java/org/aksw/iguana/cc/query/pattern/PatternHandlerTest.java +++ /dev/null @@ -1,113 +0,0 @@ -package org.aksw.iguana.cc.query.pattern; - -import org.aksw.iguana.cc.query.source.QuerySource; -import org.aksw.iguana.cc.query.source.impl.FileLineQuerySource; -import org.aksw.iguana.cc.utils.ServerMock; -import org.apache.jena.ext.com.google.common.collect.Lists; -import org.apache.jena.ext.com.google.common.collect.Sets; -import org.apache.jena.query.ParameterizedSparqlString; -import org.apache.jena.query.Query; -import org.apache.jena.query.QueryFactory; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.simpleframework.http.core.ContainerServer; -import org.simpleframework.transport.connect.SocketConnection; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.net.SocketAddress; -import java.util.*; - -import static org.junit.Assert.assertEquals; - -@RunWith(Parameterized.class) -public class PatternHandlerTest { - - private static final int FAST_SERVER_PORT = 8024; - private static ContainerServer fastServer; - private static SocketConnection fastConnection; - private final String service; - private final String queryStr; - private final Query expectedConversionQuery; - private final String[] vars; - private final String expectedReplacedQuery; - private final List expectedInstances; - private final String dir = UUID.randomUUID().toString(); - - - public PatternHandlerTest(String queryStr, String expectedConversionStr, String expectedReplacedQuery, String[] vars, String[] expectedInstances) { - this.service = "http://localhost:8024"; - - this.queryStr = queryStr; - this.expectedConversionQuery = QueryFactory.create(expectedConversionStr); - this.vars = vars; - this.expectedReplacedQuery = expectedReplacedQuery; - this.expectedInstances = Lists.newArrayList(expectedInstances); - } - - @Parameterized.Parameters - public static Collection data() { - Collection testData = new ArrayList<>(); - testData.add(new Object[]{"SELECT * {?s ?p ?o}", "SELECT * {?s ?p ?o}", "SELECT * {?s ?p ?o}", new String[]{}, new String[]{"SELECT * {?s ?p ?o}"}}); - testData.add(new Object[]{"SELECT ?book {?book %%var0%% ?o}", "SELECT DISTINCT ?var0 {?book ?var0 ?o} LIMIT 2000", "SELECT ?book {?book ?var0 ?o}", new String[]{"var0"}, new String[]{"SELECT ?book {?book ?o}", "SELECT ?book {?book ?o}"}}); - testData.add(new Object[]{"SELECT ?book {?book %%var0%% %%var1%%}", "SELECT DISTINCT ?var1 ?var0 {?book ?var0 ?var1} LIMIT 2000", "SELECT ?book {?book ?var0 ?var1}", new String[]{"var0", "var1"}, new String[]{"SELECT ?book {?book \"Example Book 2\"}", "SELECT ?book {?book \"Example Book 1\"}"}}); - - return testData; - } - - @BeforeClass - public static void startServer() throws IOException { - ServerMock fastServerContainer = new ServerMock(); - fastServer = new ContainerServer(fastServerContainer); - fastConnection = new SocketConnection(fastServer); - SocketAddress address1 = new InetSocketAddress(FAST_SERVER_PORT); - fastConnection.connect(address1); - } - - @AfterClass - public static void stopServer() throws IOException { - fastConnection.close(); - fastServer.stop(); - } - - @Test - public void testReplacement() { - Set varNames = new HashSet<>(); - String replacedQuery = getHandler().replaceVars(this.queryStr, varNames); - assertEquals(this.expectedReplacedQuery, replacedQuery); - assertEquals(Sets.newHashSet(vars), varNames); - } - - - @Test - public void testPatternExchange() { - List instances = getHandler().generateQueries(this.queryStr); - assertEquals(this.expectedInstances, instances); - } - - @Test - public void testConversion() { - // convert query - // retrieve instances - PatternHandler qh = getHandler(); - - ParameterizedSparqlString pss = new ParameterizedSparqlString(); - pss.setCommandText(qh.replaceVars(this.queryStr, Sets.newHashSet())); - - Query q = qh.convertToSelect(pss, Sets.newHashSet(this.vars)); - assertEquals(this.expectedConversionQuery, q); - } - - private PatternHandler getHandler() { - Map config = new HashMap<>(); - config.put("endpoint", this.service); - config.put("outputFolder", this.dir); - - QuerySource qs = new FileLineQuerySource("src/test/resources/workers/single-query.txt"); - - return new PatternHandler(config, qs); - } -} diff --git a/src/test/java/org/aksw/iguana/cc/query/selector/impl/LinearQuerySelectorTest.java b/src/test/java/org/aksw/iguana/cc/query/selector/impl/LinearQuerySelectorTest.java index 8f6d18947..ca508685b 100644 --- a/src/test/java/org/aksw/iguana/cc/query/selector/impl/LinearQuerySelectorTest.java +++ b/src/test/java/org/aksw/iguana/cc/query/selector/impl/LinearQuerySelectorTest.java @@ -1,23 +1,29 @@ package org.aksw.iguana.cc.query.selector.impl; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; public class LinearQuerySelectorTest { - private LinearQuerySelector linearQuerySelector; - - @Before - public void setUp() { - this.linearQuerySelector = new LinearQuerySelector(5); + @ParameterizedTest() + @ValueSource(ints = {1, 2, 3, 4}) + public void getNextIndexTest(int size) { + final var linearQuerySelector = new LinearQuerySelector(size); + assertEquals(-1, linearQuerySelector.getCurrentIndex()); + for (int i = 0; i < 10; i++) { + int currentIndex = linearQuerySelector.getNextIndex(); + assertEquals(i % size, currentIndex); + assertEquals(currentIndex, linearQuerySelector.getCurrentIndex()); + } } @Test - public void getNextIndexTest() { - for (int i = 0; i < 10; i++) { - assertEquals(i % 5, this.linearQuerySelector.getNextIndex()); - } + public void ThrowOnLinearQuerySelectorSizeZero() { + final var size = 0; + assertThrows(IllegalArgumentException.class, () -> new LinearQuerySelector(size)); } } \ No newline at end of file diff --git a/src/test/java/org/aksw/iguana/cc/query/selector/impl/RandomQuerySelectorTest.java b/src/test/java/org/aksw/iguana/cc/query/selector/impl/RandomQuerySelectorTest.java new file mode 100644 index 000000000..2353d983c --- /dev/null +++ b/src/test/java/org/aksw/iguana/cc/query/selector/impl/RandomQuerySelectorTest.java @@ -0,0 +1,39 @@ +package org.aksw.iguana.cc.query.selector.impl; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.junit.jupiter.api.Assertions.*; + +public class RandomQuerySelectorTest { + + @Test + public void testGetIndex() { + final var selector = new RandomQuerySelector(100, 10); + for (int i = 0; i < 10000; i++) { + int currentIndex = selector.getNextIndex(); + assertTrue(0 <= currentIndex && currentIndex < 100); + assertEquals(currentIndex, selector.getCurrentIndex()); + } + } + + @ParameterizedTest + @ValueSource(ints = {-1, 0}) + public void testThrowingOnIllegalSize(int size) { + assertThrows(IllegalArgumentException.class, () -> new RandomQuerySelector(-1, 0)); + assertThrows(IllegalArgumentException.class, () -> new RandomQuerySelector(0, 0)); + } + + @ParameterizedTest + @ValueSource(ints = {1, 2, 3, 100000}) + public void testSeedConsistency(int size) { + final var selector = new RandomQuerySelector(size, 0); + final var selector2 = new RandomQuerySelector(size, 0); + for (int i = 0; i < 100000; i++) { + final var nextIndex = selector.getNextIndex(); + final var nextIndex2 = selector2.getNextIndex(); + assert nextIndex == nextIndex2; + } + } +} diff --git a/src/test/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySourceTest.java b/src/test/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySourceTest.java index 7801fec80..7e3ce487c 100644 --- a/src/test/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySourceTest.java +++ b/src/test/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySourceTest.java @@ -1,49 +1,114 @@ package org.aksw.iguana.cc.query.source.impl; -import org.aksw.iguana.cc.utils.FileUtils; -import org.junit.Test; +import org.apache.commons.io.FileUtils; +import org.apache.commons.text.StringEscapeUtils; +import org.junit.jupiter.api.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; +import java.util.function.Function; +import java.util.stream.IntStream; + +import static org.junit.jupiter.api.Assertions.*; -import static org.junit.Assert.assertEquals; public class FileLineQuerySourceTest { + private record SourceConfig(int size, String lineEnding, boolean overshoot) { + @Override + public String toString() { + return "{ size: " + size + ", overshoot: " + overshoot + ", line_ending: " + StringEscapeUtils.escapeJava(lineEnding) + " }"; + } + } - private static final String PATH = "src/test/resources/query/source/queries.txt"; + private final static Function queryTemplate = (i) -> "Query " + i + " {still query " + i + "}"; - private final FileLineQuerySource querySource; + private static Path directory; + private static List cachedArguments = null; - public FileLineQuerySourceTest() { - this.querySource = new FileLineQuerySource(PATH); + private static String createFileContent(int size, String lineEnding, boolean overshoot) { + final var stringBuilder = new StringBuilder(); + int limit = overshoot ? size : size - 1; + for (int i = 0; i < limit; i++) { + stringBuilder.append(queryTemplate.apply(i)).append(lineEnding); + } + if (!overshoot) { + stringBuilder.append(queryTemplate.apply(size - 1)); + } + return stringBuilder.toString(); } - @Test - public void sizeTest() { - assertEquals(3, this.querySource.size()); + public static List createTestSource() throws IOException { + if (cachedArguments != null) { + return cachedArguments; + } + List output = new ArrayList<>(); + int[] sizes = { 1, 1000 }; + String[] lineEndings = { "\n", "\r\n", "\r" }; + boolean[] overshoots = { false, true }; + for (int size : sizes) { + for (String lineEnding : lineEndings) { + for (boolean overshoot : overshoots) { + final var fileContent = createFileContent(size, lineEnding, overshoot); + final var filePath = Files.createTempFile(directory, "Query", ".txt"); + Files.writeString(filePath, fileContent); + final var querySource = new FileLineQuerySource(filePath); + output.add(Arguments.of(querySource, new SourceConfig(size, lineEnding, overshoot))); + } + } + } + cachedArguments = output; + return output; } - @Test - public void getQueryTest() throws IOException { - assertEquals("QUERY 1 {still query 1}", this.querySource.getQuery(0)); - assertEquals("QUERY 2 {still query 2}", this.querySource.getQuery(1)); - assertEquals("QUERY 3 {still query 3}", this.querySource.getQuery(2)); + @BeforeAll + public static void createTempDirectory() throws IOException { + directory = Files.createTempDirectory("iguana-file-line-query-source-test-dir"); + } + + @AfterAll + public static void deleteTempDirectory() throws IOException { + FileUtils.deleteDirectory(directory.toFile()); } @Test - public void getAllQueriesTest() throws IOException { - List expected = new ArrayList<>(3); - expected.add("QUERY 1 {still query 1}"); - expected.add("QUERY 2 {still query 2}"); - expected.add("QUERY 3 {still query 3}"); + public void testInitialization() throws IOException { + assertThrows(NullPointerException.class, () -> new FileLineQuerySource(null)); + assertDoesNotThrow(() -> new FileLineQuerySource(Files.createTempFile(directory, "Query", ".txt"))); + final var notEmptyFile = Files.createTempFile(directory, "Query", ".txt"); + Files.writeString(notEmptyFile, "not empty"); + assertDoesNotThrow(() -> new FileLineQuerySource(notEmptyFile)); + } - assertEquals(expected, this.querySource.getAllQueries()); + @ParameterizedTest(name = "[{index}] config = {1}") + @MethodSource("createTestSource") + public void sizeTest(FileLineQuerySource querySource, SourceConfig config) throws IOException { + assertEquals(config.size, querySource.size()); } - @Test - public void getHashcodeTest() { - int expected = FileUtils.getHashcodeFromFileContent(PATH); - assertEquals(expected, this.querySource.hashCode()); + @ParameterizedTest(name = "[{index}] config = {1}") + @MethodSource("createTestSource") + public void getQueryTest(FileLineQuerySource querySource, SourceConfig config) throws IOException { + for (int i = 0; i < config.size; i++) { + assertEquals(queryTemplate.apply(i), querySource.getQuery(i)); + } + } + + @ParameterizedTest(name = "[{index}] config = {1}") + @MethodSource("createTestSource") + public void getAllQueriesTest(FileLineQuerySource querySource, SourceConfig config) throws IOException { + List expected = IntStream.range(0, config.size).mapToObj(i -> queryTemplate.apply(i)).toList(); + assertEquals(expected, querySource.getAllQueries()); + } + + @ParameterizedTest(name = "[{index}] config = {1}") + @MethodSource("createTestSource") + public void getHashcodeTest(FileLineQuerySource querySource, SourceConfig config) { + assertTrue(querySource.hashCode() != 0); } } \ No newline at end of file diff --git a/src/test/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySourceTest.java b/src/test/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySourceTest.java index 07acffe18..0b90506d2 100644 --- a/src/test/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySourceTest.java +++ b/src/test/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySourceTest.java @@ -1,70 +1,129 @@ package org.aksw.iguana.cc.query.source.impl; -import org.aksw.iguana.cc.utils.FileUtils; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.apache.commons.io.FileUtils; +import org.apache.commons.text.StringEscapeUtils; +import org.junit.jupiter.api.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; -import java.util.Collection; import java.util.List; +import java.util.function.BiFunction; +import java.util.stream.IntStream; + +import static org.junit.jupiter.api.Assertions.*; -import static org.junit.Assert.assertEquals; -@RunWith(Parameterized.class) public class FileSeparatorQuerySourceTest { + private record SourceConfig(int size, String lineEnding, boolean overshoot, String separator) { + @Override + public String toString() { + return "{ size: " + size + + ", overshoot: " + overshoot + + ", line_ending: " + StringEscapeUtils.escapeJava(lineEnding) + + ", separator: " + StringEscapeUtils.escapeJava(separator) + " }"; + } + } - private final FileSeparatorQuerySource querySource; + private final static BiFunction queryTemplate = (i, le) -> "Query " + i + " {" + le + "still query " + i + le + "}"; - private final String path; + private static Path directory; + private static List cachedArguments = null; - public FileSeparatorQuerySourceTest(String path, String separator) { - this.path = path; + private static String createFileContent(int size, String lineEnding, boolean overshoot, String separator) { + final var stringBuilder = new StringBuilder(); + int limit = overshoot ? size : size - 1; + for (int i = 0; i < limit; i++) { + stringBuilder.append(queryTemplate.apply(i, lineEnding)).append(separator); + } + if (!overshoot) { + stringBuilder.append(queryTemplate.apply(size - 1, lineEnding)); + } + return stringBuilder.toString(); + } - if (separator == null) { - this.querySource = new FileSeparatorQuerySource(this.path); - } else { - this.querySource = new FileSeparatorQuerySource(this.path, separator); + public static List createTestSource() throws IOException { + if (cachedArguments != null) { + return cachedArguments; + } + List output = new ArrayList<>(); + int[] sizes = { 1, 1000 }; + String[] lineEndings = { "\n", "\r\n", "\r" }; + boolean[] overshoots = { false, true }; // tests if there is no empty query at the end + String[] separators = { "\n\t\t", "\n###\n", "###", ""}; + for (int size : sizes) { + for (String lineEnding : lineEndings) { + for (boolean overshoot : overshoots) { + for (String separator : separators) { + String fileContent; + if (separator.isEmpty()) + fileContent = createFileContent(size, lineEnding, overshoot, lineEnding + lineEnding); // make empty lines + else + fileContent = createFileContent(size, lineEnding, overshoot, separator); + final var filePath = Files.createTempFile(directory, "Query", ".txt"); + Files.writeString(filePath, fileContent); + FileSeparatorQuerySource querySource; + if (separator.equals("###")) + querySource = new FileSeparatorQuerySource(filePath); // test default separator + else + querySource = new FileSeparatorQuerySource(filePath, separator); + output.add(Arguments.of(querySource, new SourceConfig(size, lineEnding, overshoot, separator))); + } + } + } } + cachedArguments = output; + return output; } - @Parameterized.Parameters - public static Collection data() { - Collection testData = new ArrayList<>(); - testData.add(new Object[]{"src/test/resources/query/source/separated-queries-default.txt", null}); - testData.add(new Object[]{"src/test/resources/query/source/separated-queries-space.txt", ""}); + @BeforeAll + public static void createTempDirectory() throws IOException { + directory = Files.createTempDirectory("iguana-file-line-query-source-test-dir"); + } - return testData; + @AfterAll + public static void deleteTempDirectory() throws IOException { + FileUtils.deleteDirectory(directory.toFile()); } @Test - public void sizeTest() { - assertEquals(3, this.querySource.size()); + public void testInitialization() throws IOException { + assertThrows(NullPointerException.class, () -> new FileSeparatorQuerySource(null)); + assertDoesNotThrow(() -> new FileSeparatorQuerySource(Files.createTempFile(directory, "Query", ".txt"), "###")); + final var notEmptyFile = Files.createTempFile(directory, "Query", ".txt"); + Files.writeString(notEmptyFile, "not empty"); + assertDoesNotThrow(() -> new FileSeparatorQuerySource(notEmptyFile)); + assertDoesNotThrow(() -> new FileSeparatorQuerySource(notEmptyFile, "\n\n\n")); } - @Test - public void getQueryTest() throws IOException { - String le = FileUtils.getLineEnding(this.path); - assertEquals("QUERY 1 {" + le + "still query 1" + le + "}", this.querySource.getQuery(0)); - assertEquals("QUERY 2 {" + le + "still query 2" + le + "}", this.querySource.getQuery(1)); - assertEquals("QUERY 3 {" + le + "still query 3" + le + "}", this.querySource.getQuery(2)); + @ParameterizedTest(name = "[{index}] config = {1}") + @MethodSource("createTestSource") + public void sizeTest(FileSeparatorQuerySource querySource, SourceConfig config) throws IOException { + assertEquals(config.size, querySource.size()); } - @Test - public void getAllQueriesTest() throws IOException { - List expected = new ArrayList<>(3); - String le = FileUtils.getLineEnding(this.path); - expected.add("QUERY 1 {" + le + "still query 1" + le + "}"); - expected.add("QUERY 2 {" + le + "still query 2" + le + "}"); - expected.add("QUERY 3 {" + le + "still query 3" + le + "}"); - - assertEquals(expected, this.querySource.getAllQueries()); + @ParameterizedTest(name = "[{index}] config = {1}") + @MethodSource("createTestSource") + public void getQueryTest(FileSeparatorQuerySource querySource, SourceConfig config) throws IOException { + for (int i = 0; i < config.size; i++) { + assertEquals(queryTemplate.apply(i, config.lineEnding), querySource.getQuery(i)); + } } - @Test - public void getHashcodeTest() { - int expected = FileUtils.getHashcodeFromFileContent(this.path); - assertEquals(expected, this.querySource.hashCode()); + @ParameterizedTest(name = "[{index}] config = {1}") + @MethodSource("createTestSource") + public void getAllQueriesTest(FileSeparatorQuerySource querySource, SourceConfig config) throws IOException { + List expected = IntStream.range(0, config.size).mapToObj(i -> queryTemplate.apply(i, config.lineEnding)).toList(); + assertEquals(expected, querySource.getAllQueries()); + } + + @ParameterizedTest(name = "[{index}] config = {1}") + @MethodSource("createTestSource") + public void getHashcodeTest(FileSeparatorQuerySource querySource, SourceConfig config) { + assertTrue(querySource.hashCode() != 0); } } \ No newline at end of file diff --git a/src/test/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySourceTest.java b/src/test/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySourceTest.java index ef58c6101..811cd26c5 100644 --- a/src/test/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySourceTest.java +++ b/src/test/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySourceTest.java @@ -1,8 +1,10 @@ package org.aksw.iguana.cc.query.source.impl; -import org.junit.*; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -11,35 +13,12 @@ import java.util.*; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(Parameterized.class) public class FolderQuerySourceTest { + static Path tempDir; - Path tempDir; - TestConfig testConfig; - - public FolderQuerySourceTest(TestConfig testConfig) { - this.testConfig = testConfig; - } - - public static class TestConfig { - - public TestConfig(int numberOfQueries) { - this.numberOfQueries = numberOfQueries; - } - - int numberOfQueries; - } - - public static class Query implements Comparable { - public Query(Path queryFile, String content) { - this.queryFile = queryFile; - this.content = content; - } - - Path queryFile; - String content; + public record Query(Path queryFile, String content) implements Comparable { @Override public int compareTo(Query other) { @@ -47,47 +26,47 @@ public int compareTo(Query other) { } } - List queries; - - - @Parameterized.Parameters - public static Collection data() { - return List.of(new TestConfig(0), - new TestConfig(1), - new TestConfig(2), - new TestConfig(5)); + public static List data() throws IOException { + final var sizes = List.of(1, 2, 10, 100, 1000); + final var out = new ArrayList(); + for (int size : sizes) { + final var queries = new LinkedList(); + final var queryDir = Files.createTempDirectory(tempDir, "query-dir"); + for (int i = 0; i < size; i++) { + final Path queryFile = Files.createTempFile(queryDir, "Query", ".txt"); + final String content = UUID.randomUUID().toString(); + Files.write(queryFile, content.getBytes(StandardCharsets.UTF_8)); + queries.add(new Query(queryFile, content)); + } + // Queries in the folder are expected in alphabetic order of the file names. + Collections.sort(queries); + out.add(Arguments.of(queryDir, queries)); + } + return out; } - @Before - public void createFolder() throws IOException { - this.tempDir = Files.createTempDirectory("folder-query-source-test-dir"); - - this.queries = new LinkedList<>(); - for (int i = 0; i < testConfig.numberOfQueries; i++) { - final Path queryFile = Files.createTempFile(tempDir, "Query", ".txt"); - final String content = UUID.randomUUID().toString(); - Files.write(queryFile, content.getBytes(StandardCharsets.UTF_8)); - this.queries.add(new Query(queryFile, content)); - } - // Queries in the folder are expected in alphabetic order of the file names. - Collections.sort(this.queries); + @BeforeAll + public static void createFolder() throws IOException { + tempDir = Files.createTempDirectory("iguana-folder-query-source-test-dir"); } - @After - public void removeFolder() throws IOException { - org.apache.commons.io.FileUtils.deleteDirectory(this.tempDir.toFile()); + + @AfterAll + public static void removeFolder() throws IOException { + org.apache.commons.io.FileUtils.deleteDirectory(tempDir.toFile()); } - @Test - public void testFolderQuerySource() throws IOException { - FolderQuerySource querySource = new FolderQuerySource(tempDir.toString()); + @ParameterizedTest + @MethodSource("data") + public void testFolderQuerySource(Path tempDir, List expectedQueries) throws IOException { + FolderQuerySource querySource = new FolderQuerySource(tempDir); - assertEquals(this.queries.size(), querySource.size()); + assertEquals(expectedQueries.size(), querySource.size()); for (int i = 0; i < querySource.size(); i++) { - assertEquals(queries.get(i).content, querySource.getQuery(i)); + assertEquals(expectedQueries.get(i).content, querySource.getQuery(i)); } - assertEquals(queries.stream().map(q -> q.content).collect(Collectors.toList()), querySource.getAllQueries()); + assertEquals(expectedQueries.stream().map(q -> q.content).collect(Collectors.toList()), querySource.getAllQueries()); } } \ No newline at end of file diff --git a/src/test/java/org/aksw/iguana/cc/storage/impl/CSVStorageTest.java b/src/test/java/org/aksw/iguana/cc/storage/impl/CSVStorageTest.java new file mode 100644 index 000000000..db1d5ff5f --- /dev/null +++ b/src/test/java/org/aksw/iguana/cc/storage/impl/CSVStorageTest.java @@ -0,0 +1,140 @@ +package org.aksw.iguana.cc.storage.impl; + +import com.opencsv.CSVReader; +import com.opencsv.exceptions.CsvException; +import org.aksw.iguana.cc.mockup.MockupQueryHandler; +import org.aksw.iguana.cc.mockup.MockupWorker; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.FileReader; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.*; + +public class CSVStorageTest extends StorageTest { + private static final String EXPECTED_FILES_DIR = "src/test/resources/test-data/csv-storage-test/"; + + public static List data() { + final var workersTask1 = List.of( + MockupWorker.createWorkers(0, 2, new MockupQueryHandler(0, 10), "test-connection-1", "v1.0.0", "test-dataset-1"), + MockupWorker.createWorkers(2, 2, new MockupQueryHandler(1, 10), "test-connection-2", "v1.1.0", "test-dataset-2") + ); + + final var workersTask2 = List.of( + MockupWorker.createWorkers(0, 2, new MockupQueryHandler(2, 5), "test-connection-3", "v1.2.0", "test-dataset-3"), + MockupWorker.createWorkers(2, 2, new MockupQueryHandler(3, 5), "test-connection-4", "v1.3.0", "test-dataset-4") + ); + + return List.of(Arguments.of(List.of(createTaskResult(workersTask1, 0, "123"), createTaskResult(workersTask2, 1, "123")))); + } + + @ParameterizedTest + @MethodSource("data") + protected void testCSVStorage(List results) throws IOException { + final var storage = new CSVStorage(tempDir.toString(), getMetrics(), "123"); + for (var result : results) + storage.storeResult(result.resultModel()); + + final var expectedFiles = Path.of(EXPECTED_FILES_DIR); + final var actualFiles = tempDir; + + try (var files = Files.list(expectedFiles)) { + files.forEach( + x -> { + try { + compareFile(x, actualFiles.resolve(x.getFileName())); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + ); + } + + storage.storeData(new TestStorable()); + final var path = tempDir.resolve("suite-123").resolve("task-1").resolve("csv-folder").toFile(); + assertTrue(path.exists()); + assertTrue(path.isDirectory()); + assertEquals(2, path.listFiles().length); + for (var file : path.listFiles()) { + if (file.getName().equals("csv-file-1.csv")) + assertEquals(2, Files.readAllLines(file.toPath()).size()); + else if (file.getName().equals("csv-file-2.csv")) + assertEquals(3, Files.readAllLines(file.toPath()).size()); + else + throw new RuntimeException("Unexpected file name: " + file.getName()); + } + } + + private void compareFile(Path expected, Path actual) throws IOException { + if (Files.isDirectory(expected)) { + assertTrue(Files.isDirectory(actual), String.format("Expected directory %s but found file %s", expected, actual)); + assertEquals(expected.getFileName(), actual.getFileName()); + try (var files = Files.list(expected)) { + files.forEach(x -> { + try { + compareFile(x, actual.resolve(x.getFileName())); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + } else if (Files.isRegularFile(expected)) { + assertTrue(Files.isRegularFile(actual), String.format("Expected file %s but found directory %s", expected, actual)); + assertEquals(expected.getFileName(), actual.getFileName()); + compareCSVFiles(expected, actual); + } else { + throw new RuntimeException(String.format("Expected file or directory %s but found nothing", expected)); + } + } + + private void compareCSVFiles(Path expected, Path actual) throws IOException { + try (CSVReader readerExpected = new CSVReader(new FileReader(expected.toFile())); + CSVReader readerActual = new CSVReader(new FileReader(actual.toFile()))) { + + String[] headerExpected; + String[] headerActual; + try { + headerExpected = readerExpected.readNext(); + headerActual = readerActual.readNext(); + } catch (CsvException e) { + throw new RuntimeException(String.format("CSV format in the header of file %s is malformed.", expected), e); + } + + assertEquals(headerExpected.length, headerActual.length, String.format("Headers don't match. Actual: %s, Expected: %s", Arrays.toString(headerActual), Arrays.toString(headerExpected))); + for (int i = 0; i < headerExpected.length; i++) { + assertEquals(headerExpected[i], headerActual[i], String.format("Headers don't match. Actual: %s, Expected: %s", Arrays.toString(headerActual), Arrays.toString(headerExpected))); + } + + List expectedValues; + List actualValues; + + try { + expectedValues = new ArrayList<>(readerExpected.readAll()); + actualValues = new ArrayList<>(readerActual.readAll()); + } catch (CsvException e) { + throw new RuntimeException(String.format("CSV format in file %s is malformed.", expected), e); + } + + for (String[] expectedLine : expectedValues) { + List sameLines = actualValues.stream().filter(x -> { + for (int i = 0; i < expectedLine.length; i++) { + if (!expectedLine[i].equals(x[i])) return false; + } + return true; + }).toList(); + + assertFalse(sameLines.isEmpty(), String.format("Line (%s) not found in actual file", Arrays.toString(expectedLine))); + actualValues.remove(sameLines.get(0)); + } + assertTrue(actualValues.isEmpty(), String.format("Actual file contains more lines than expected. Lines: %s", actualValues.stream().map(x -> "[" + String.join(", ", x) + "]").collect(Collectors.joining("\n")))); + } + } +} diff --git a/src/test/java/org/aksw/iguana/cc/storage/impl/RDFFileStorageTest.java b/src/test/java/org/aksw/iguana/cc/storage/impl/RDFFileStorageTest.java new file mode 100644 index 000000000..e044ecfbc --- /dev/null +++ b/src/test/java/org/aksw/iguana/cc/storage/impl/RDFFileStorageTest.java @@ -0,0 +1,56 @@ +package org.aksw.iguana.cc.storage.impl; + +import org.aksw.iguana.cc.mockup.MockupQueryHandler; +import org.aksw.iguana.cc.mockup.MockupWorker; +import org.apache.jena.rdf.model.*; +import org.apache.jena.riot.RDFDataMgr; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.ArrayList; +import java.util.List; + +/** + * This Test class extends the StorageTest class and tests the RDFFileStorage class. + */ +public class RDFFileStorageTest extends StorageTest { + public static List data() { + final var arguments = new ArrayList(); + + final var paths = new ArrayList<>(List.of("rdf-file-storage-test1.ttl", "rdf-file-storage-test1.nt", "rdf-file-storage-test1.nt", "")); + + final var queryHandler1 = new MockupQueryHandler(0, 10); + final var queryHandler2 = new MockupQueryHandler(1, 10); + + final var workers = List.of( + MockupWorker.createWorkers(0, 2, queryHandler1, "test-connection-1", "v1.0.0", "test-dataset-1"), + MockupWorker.createWorkers(2, 2, queryHandler2, "test-connection-1", "v1.0.0", "test-dataset-1") + ); + final var task1 = createTaskResult(workers, 0, "0"); + final var task2 = createTaskResult(workers, 1, "0"); + + // test file creation + for (String path : paths) { + arguments.add(Arguments.of(tempDir.resolve(path).toString(), List.of(task1), task1.resultModel())); + } + + // test file appending + Model concatenatedModel = ModelFactory.createDefaultModel().add(task1.resultModel()).add(task2.resultModel()); + arguments.add(Arguments.of(tempDir.resolve("rdf-file-storage-test2.ttl").toString(), List.of(task1, task2), concatenatedModel)); + return arguments; + } + + @ParameterizedTest + @MethodSource("data") + public void testRDFFileStorage(String path, List results, Model expectedModel) { + final var rdfStorage = new RDFFileStorage(path); + for (var result : results) { + rdfStorage.storeResult(result.resultModel()); + } + path = rdfStorage.getFileName(); + Model actualModel = RDFDataMgr.loadModel(path); + Assertions.assertTrue(actualModel.isIsomorphicWith(expectedModel)); + } +} diff --git a/src/test/java/org/aksw/iguana/cc/storage/impl/StorageTest.java b/src/test/java/org/aksw/iguana/cc/storage/impl/StorageTest.java new file mode 100644 index 000000000..6fe07a015 --- /dev/null +++ b/src/test/java/org/aksw/iguana/cc/storage/impl/StorageTest.java @@ -0,0 +1,120 @@ +package org.aksw.iguana.cc.storage.impl; + +import org.aksw.iguana.cc.lang.LanguageProcessor; +import org.aksw.iguana.cc.metrics.Metric; +import org.aksw.iguana.cc.metrics.impl.*; +import org.aksw.iguana.cc.mockup.MockupWorker; +import org.aksw.iguana.cc.storage.Storable; +import org.aksw.iguana.cc.storage.Storage; +import org.aksw.iguana.cc.mockup.MockupStorage; +import org.aksw.iguana.cc.tasks.impl.StresstestResultProcessor; +import org.aksw.iguana.cc.worker.HttpWorker; +import org.apache.commons.io.FileUtils; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.*; +import java.util.function.Supplier; + + +public abstract class StorageTest { + @BeforeAll + public static void createFolder() throws IOException { + tempDir = Files.createTempDirectory("iguana-storage-test-dir"); + } + + @AfterAll + public static void deleteFolder() throws IOException { + FileUtils.deleteDirectory(tempDir.toFile()); + } + + public static class TestStorable implements Storable.AsCSV, Storable.AsRDF { + + @Override + public Storable.CSVData toCSV() { + final var data = new Storable.CSVData("csv-folder", List.of( + new Storable.CSVData.CSVFileData("csv-file-1", new String[][]{{"header-1", "header-2"}, {"randomString", "100"}}), + new Storable.CSVData.CSVFileData("csv-file-2", new String[][]{{"header-1", "header-2"}, {"randomString-2", "200"}, {"randomString-3", "300"}}) + )); + return data; + } + + @Override + public Model toRDF() { + Model m = ModelFactory.createDefaultModel(); + m.add(m.createResource("http://example.org/subject"), m.createProperty("http://example.org/predicate"), m.createResource("http://example.org/object")); + return m; + } + } + + @BeforeEach + public void resetDate() { + someDateTime = GregorianCalendar.from(ZonedDateTime.ofInstant(Instant.parse("2023-10-21T20:48:06.399Z"), ZoneId.of("Europe/Berlin"))); + } + + protected record TaskResult(Model resultModel, List workerResults) {} + + protected static Path tempDir; + + private static Calendar someDateTime = GregorianCalendar.from(ZonedDateTime.ofInstant(Instant.parse("2023-10-21T20:48:06.399Z"), ZoneId.of("Europe/Berlin"))); + + private static Calendar getDateTime() { + someDateTime.add(Calendar.MINUTE, 1); + someDateTime.add(Calendar.SECOND, 18); + return someDateTime; + } + + public static List getMetrics() { + return List.of( + new AggregatedExecutionStatistics(), + new AvgQPS(), + new EachExecutionStatistic(), + new NoQ(), + new NoQPH(), + new PAvgQPS(1000), + new PQPS(1000), + new QMPH(), + new QPS() + ); + } + + // Argument is a List that contains lists of workers with the same configuration. + protected static TaskResult createTaskResult(List> workers, int taskID, String suiteID) { + final var queryIDs = new ArrayList(); + for (var list : workers) { + queryIDs.addAll(List.of(list.get(0).config().queries().getAllQueryIds())); + } + + final var metrics = getMetrics(); + final var storages = new ArrayList(); + final Supplier>> supplier = HashMap::new; + + final var ls = new MockupStorage(); + storages.add(ls); + + final var flatWorkerList = workers.stream().flatMap(Collection::stream).toList(); + + final var srp = new StresstestResultProcessor(suiteID, taskID, flatWorkerList, queryIDs, metrics, storages, supplier); + + final var workerResults = new ArrayList(); + for (var list : workers) { + workerResults.addAll(MockupWorker.createWorkerResults(list.get(0).config().queries(), list)); + } + + srp.process(workerResults); + Calendar startTime = (Calendar) getDateTime().clone(); + srp.calculateAndSaveMetrics(startTime, getDateTime()); + + return new TaskResult(ls.getResultModel(), workerResults); + } + +} diff --git a/src/test/java/org/aksw/iguana/cc/storage/impl/TriplestoreStorageTest.java b/src/test/java/org/aksw/iguana/cc/storage/impl/TriplestoreStorageTest.java new file mode 100644 index 000000000..a33d135cf --- /dev/null +++ b/src/test/java/org/aksw/iguana/cc/storage/impl/TriplestoreStorageTest.java @@ -0,0 +1,70 @@ +package org.aksw.iguana.cc.storage.impl; + +import com.github.tomakehurst.wiremock.common.Slf4jNotifier; +import com.github.tomakehurst.wiremock.core.Options; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; +import com.github.tomakehurst.wiremock.junit5.WireMockExtension; +import com.github.tomakehurst.wiremock.stubbing.ServeEvent; +import org.aksw.iguana.cc.mockup.MockupQueryHandler; +import org.aksw.iguana.cc.mockup.MockupWorker; +import org.aksw.iguana.cc.worker.HttpWorker; +import org.apache.jena.riot.RDFDataMgr; +import org.apache.jena.update.UpdateFactory; +import org.apache.jena.update.UpdateRequest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.io.StringWriter; +import java.net.URI; +import java.net.URISyntaxException; +import java.time.Duration; +import java.util.List; +import java.util.UUID; + +import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TriplestoreStorageTest extends StorageTest { + + @RegisterExtension + public static WireMockExtension wm = WireMockExtension.newInstance() + .options(new WireMockConfiguration().useChunkedTransferEncoding(Options.ChunkedEncodingPolicy.NEVER).dynamicPort().notifier(new Slf4jNotifier(true))) + .failOnUnmatchedRequests(true) + .build(); + + @Test + public void dataTest() throws URISyntaxException { + final var uuid = UUID.randomUUID(); + wm.stubFor(post(urlEqualTo("/ds/sparql")) + .willReturn(aResponse() + .withStatus(200))) + .setId(uuid); + + final List> worker = List.of(List.of( + new MockupWorker( + 0, + new MockupQueryHandler(1, 2), + "conneciton", + "v2", + "wikidata", + Duration.ofSeconds(2)) + )); + final var testData = createTaskResult(worker, 0, "1"); + + final var uri = new URI("http://localhost:" + wm.getPort() + "/ds/sparql"); + final var storage = new TriplestoreStorage(String.valueOf(uri)); + storage.storeResult(testData.resultModel()); + + List allServeEvents = wm.getAllServeEvents(); + ServeEvent request = allServeEvents.get(0); + String body = request.getRequest().getBodyAsString(); + + StringWriter nt = new StringWriter(); + RDFDataMgr.write(nt, testData.resultModel(), org.apache.jena.riot.Lang.NTRIPLES); + + UpdateRequest updateRequestActual = UpdateFactory.create(body); + UpdateRequest updateRequestExpected = UpdateFactory.create().add("INSERT DATA { " + nt + " }"); + + assertTrue(updateRequestExpected.iterator().next().equalTo(updateRequestActual.iterator().next(), null)); + } +} diff --git a/src/test/java/org/aksw/iguana/cc/suite/IguanaSuiteParserTest.java b/src/test/java/org/aksw/iguana/cc/suite/IguanaSuiteParserTest.java new file mode 100644 index 000000000..9ab39b009 --- /dev/null +++ b/src/test/java/org/aksw/iguana/cc/suite/IguanaSuiteParserTest.java @@ -0,0 +1,36 @@ +package org.aksw.iguana.cc.suite; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; + +class IguanaSuiteParserTest { + + public static Stream validData() throws IOException { + final var dir = Path.of("./src/test/resources/suite-configs/valid/"); + return Files.list(dir).map(Arguments::of); + } + + public static Stream invalidData() throws IOException { + final var dir = Path.of("./src/test/resources/suite-configs/invalid/"); + return Files.list(dir).map(Arguments::of); + } + + @ParameterizedTest + @MethodSource("validData") + public void testValidDeserialization(Path config) throws IOException { + Assertions.assertTrue(IguanaSuiteParser.validateConfig(config)); + } + + @ParameterizedTest + @MethodSource("invalidData") + public void testInvalidDeserialization(Path config) throws IOException { + Assertions.assertFalse(IguanaSuiteParser.validateConfig(config)); + } +} \ No newline at end of file diff --git a/src/test/java/org/aksw/iguana/cc/tasks/MockupStorage.java b/src/test/java/org/aksw/iguana/cc/tasks/MockupStorage.java deleted file mode 100644 index 145a72570..000000000 --- a/src/test/java/org/aksw/iguana/cc/tasks/MockupStorage.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.aksw.iguana.cc.tasks; - -import org.aksw.iguana.cc.tasks.stresstest.storage.Storage; -import org.apache.jena.rdf.model.Model; -import org.apache.jena.rdf.model.ModelFactory; - -public class MockupStorage implements Storage { - private Model m = ModelFactory.createDefaultModel(); - - @Override - public void storeResult(Model data) { - m.add(data); - } -} diff --git a/src/test/java/org/aksw/iguana/cc/tasks/MockupTask.java b/src/test/java/org/aksw/iguana/cc/tasks/MockupTask.java deleted file mode 100644 index a077916b8..000000000 --- a/src/test/java/org/aksw/iguana/cc/tasks/MockupTask.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.aksw.iguana.cc.tasks; - - -public class MockupTask extends AbstractTask{ - - public MockupTask(String empty){ - } - - - @Override - public void execute() { - } - - -} diff --git a/src/test/java/org/aksw/iguana/cc/tasks/ServerMock.java b/src/test/java/org/aksw/iguana/cc/tasks/ServerMock.java deleted file mode 100644 index b70b0d4a4..000000000 --- a/src/test/java/org/aksw/iguana/cc/tasks/ServerMock.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.aksw.iguana.cc.tasks; - -import org.simpleframework.http.Request; -import org.simpleframework.http.Response; -import org.simpleframework.http.Status; -import org.simpleframework.http.core.Container; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; - -/** - * Server Mock representing a TS - * - * @author f.conrads - * - */ -public class ServerMock implements Container { - - private static final Logger LOGGER = LoggerFactory.getLogger(ServerMock.class); - private String actualContent; - - - @Override - public void handle(Request request, Response resp) { - String content=null; - try { - content = request.getContent(); - } catch (IOException e) { - LOGGER.error("Got exception.", e); - } - this.actualContent=content; - resp.setCode(Status.OK.code); - try { - resp.getOutputStream().close(); - } catch (IOException e) { - LOGGER.error("Could not close Response Output Stream"); - } - } - - /** - * @return the actualContent - */ - public String getActualContent() { - return actualContent; - } - - /** - * @param actualContent the actualContent to set - */ - public void setActualContent(String actualContent) { - this.actualContent = actualContent; - } - -} diff --git a/src/test/java/org/aksw/iguana/cc/tasks/storage/impl/CSVStorageTest.java b/src/test/java/org/aksw/iguana/cc/tasks/storage/impl/CSVStorageTest.java deleted file mode 100644 index 054d6e347..000000000 --- a/src/test/java/org/aksw/iguana/cc/tasks/storage/impl/CSVStorageTest.java +++ /dev/null @@ -1,201 +0,0 @@ -package org.aksw.iguana.cc.tasks.storage.impl; - -import com.opencsv.CSVReader; -import com.opencsv.exceptions.CsvException; -import org.aksw.iguana.cc.config.IguanaConfig; -import org.aksw.iguana.cc.tasks.stresstest.metrics.*; -import org.aksw.iguana.cc.tasks.stresstest.metrics.impl.AggregatedExecutionStatistics; -import org.aksw.iguana.cc.tasks.stresstest.metrics.impl.NoQ; -import org.aksw.iguana.cc.tasks.stresstest.metrics.impl.NoQPH; -import org.aksw.iguana.cc.tasks.stresstest.metrics.impl.QPS; -import org.aksw.iguana.cc.tasks.stresstest.storage.impl.CSVStorage; -import org.aksw.iguana.commons.rdf.IONT; -import org.aksw.iguana.commons.rdf.IPROP; -import org.aksw.iguana.commons.rdf.IRES; -import org.apache.commons.io.FileUtils; -import org.apache.jena.rdf.model.Model; -import org.apache.jena.rdf.model.ModelFactory; -import org.apache.jena.rdf.model.Resource; -import org.apache.jena.rdf.model.ResourceFactory; -import org.apache.jena.vocabulary.RDF; -import org.apache.jena.vocabulary.RDFS; -import org.junit.jupiter.api.*; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; - -import java.io.FileReader; -import java.io.IOException; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.*; - -public class CSVStorageTest { - CSVStorage storage; - Path folder; - Path suiteFolder; - - @BeforeEach - public void setup() throws IOException { - this.folder = Files.createTempDirectory("iguana-CSVStorage-test"); - this.suiteFolder = folder.resolve(IguanaConfig.getSuiteID()); - } - - public static Arguments createTestData1() { - // Entry records should store metric values in the same order as the metrics in the following list. - List metrics = List.of(new NoQ(), new NoQPH(), new QPS(), new AggregatedExecutionStatistics()); - - // First Task has 2 Worker - Resource datasetRes = IRES.getResource("dataset1"); - Resource experimentRes = IRES.getResource("suite/experiment"); - Resource taskRes = IRES.getResource("suite/experiment/task1"); - Resource conRes = IRES.getResource("triplestore1"); - Resource workerRes1 = IRES.getResource("worker1"); - Resource workerRes2 = IRES.getResource("worker2"); - Resource taskQueryRes = IRES.getResource("task-query"); - Resource workerQueryRes1 = IRES.getResource("worker-query-1"); - Resource workerQueryRes2 = IRES.getResource("worker-query-2"); - Resource queryRes1 = IRES.getResource("query1"); - Resource queryRes2 = IRES.getResource("query2"); - - Model m = ModelFactory.createDefaultModel(); - - m.add(experimentRes, RDF.type, IONT.experiment); - m.add(experimentRes, IPROP.dataset, datasetRes); - m.add(experimentRes, IPROP.task, taskRes); - m.add(datasetRes, RDFS.label, ResourceFactory.createTypedLiteral("dataset1")); - m.add(datasetRes, RDF.type, IONT.dataset); - m.add(conRes, RDF.type, IONT.connection); - m.add(conRes, RDFS.label, ResourceFactory.createTypedLiteral("triplestore1")); - m.add(conRes, IPROP.version, ResourceFactory.createTypedLiteral("v1")); - m.add(taskRes, RDF.type, IONT.task); - m.add(taskRes, IPROP.connection, conRes); - m.add(taskRes, IPROP.startDate, ResourceFactory.createTypedLiteral("now")); - m.add(taskRes, IPROP.endDate, ResourceFactory.createTypedLiteral("then")); - m.add(taskRes, IPROP.workerResult, workerRes1); - m.add(taskRes, IPROP.workerResult, workerRes2); - m.add(taskRes, IPROP.noOfWorkers, ResourceFactory.createTypedLiteral(2)); - m.add(taskRes, IPROP.createMetricProperty(new NoQ()), ResourceFactory.createTypedLiteral(BigInteger.valueOf(2))); - m.add(taskRes, IPROP.createMetricProperty(new NoQPH()), ResourceFactory.createTypedLiteral(BigDecimal.valueOf(20,2))); - m.add(taskRes, IPROP.query, taskQueryRes); - - m.add(workerRes1, RDF.type, IONT.worker); - m.add(workerRes1, IPROP.workerID, ResourceFactory.createTypedLiteral(BigInteger.valueOf(1))); - m.add(workerRes1, IPROP.workerType, ResourceFactory.createTypedLiteral("SPARQL")); - m.add(workerRes1, IPROP.noOfQueries, ResourceFactory.createTypedLiteral(BigInteger.valueOf(2))); - m.add(workerRes1, IPROP.timeOut, ResourceFactory.createTypedLiteral(BigInteger.valueOf(100))); - m.add(workerRes1, IPROP.createMetricProperty(new NoQ()), ResourceFactory.createTypedLiteral(BigInteger.valueOf(8))); - m.add(workerRes1, IPROP.createMetricProperty(new NoQPH()), ResourceFactory.createTypedLiteral(BigDecimal.valueOf(108))); - m.add(workerRes1, IPROP.query, workerQueryRes1); - - m.add(workerRes2, RDF.type, IONT.worker); - m.add(workerRes2, IPROP.workerID, ResourceFactory.createTypedLiteral(BigInteger.valueOf(2))); - m.add(workerRes2, IPROP.workerType, ResourceFactory.createTypedLiteral("LQRAPS")); - m.add(workerRes2, IPROP.noOfQueries, ResourceFactory.createTypedLiteral(BigInteger.valueOf(1))); - m.add(workerRes2, IPROP.timeOut, ResourceFactory.createTypedLiteral(BigInteger.valueOf(50))); - m.add(workerRes2, IPROP.createMetricProperty(new NoQ()), ResourceFactory.createTypedLiteral(BigDecimal.valueOf(0))); - m.add(workerRes2, IPROP.createMetricProperty(new NoQPH()), ResourceFactory.createTypedLiteral(BigDecimal.valueOf(0))); - m.add(workerRes2, IPROP.query, workerQueryRes2); - - m.add(taskQueryRes, IPROP.createMetricProperty(new QPS()), ResourceFactory.createTypedLiteral(BigDecimal.valueOf(72000, 2))); - m.add(taskQueryRes, IPROP.succeeded, ResourceFactory.createTypedLiteral(BigInteger.valueOf(2))); - m.add(taskQueryRes, IPROP.failed, ResourceFactory.createTypedLiteral(BigInteger.valueOf(0))); - m.add(taskQueryRes, IPROP.totalTime, ResourceFactory.createTypedLiteral(BigInteger.valueOf(12345))); - m.add(taskQueryRes, IPROP.resultSize, ResourceFactory.createTypedLiteral(BigInteger.valueOf(1000))); - m.add(taskQueryRes, IPROP.wrongCodes, ResourceFactory.createTypedLiteral(BigInteger.valueOf(0))); - m.add(taskQueryRes, IPROP.timeOuts, ResourceFactory.createTypedLiteral(BigInteger.valueOf(0))); - m.add(taskQueryRes, IPROP.unknownException, ResourceFactory.createTypedLiteral(BigInteger.valueOf(0))); - m.add(taskQueryRes, IPROP.queryID, queryRes1); - m.add(taskQueryRes, IPROP.createMetricProperty(new QPS()), ResourceFactory.createTypedLiteral(BigDecimal.valueOf(72000, 2))); - - m.add(workerQueryRes1, IPROP.createMetricProperty(new QPS()), ResourceFactory.createTypedLiteral(BigDecimal.valueOf(10))); - m.add(workerQueryRes1, IPROP.succeeded, ResourceFactory.createTypedLiteral(BigInteger.valueOf(1))); - m.add(workerQueryRes1, IPROP.failed, ResourceFactory.createTypedLiteral(BigInteger.valueOf(2))); - m.add(workerQueryRes1, IPROP.totalTime, ResourceFactory.createTypedLiteral(BigInteger.valueOf(100))); - m.add(workerQueryRes1, IPROP.resultSize, ResourceFactory.createTypedLiteral(BigInteger.valueOf(98))); - m.add(workerQueryRes1, IPROP.wrongCodes, ResourceFactory.createTypedLiteral(BigInteger.valueOf(3))); - m.add(workerQueryRes1, IPROP.timeOuts, ResourceFactory.createTypedLiteral(BigInteger.valueOf(4))); - m.add(workerQueryRes1, IPROP.unknownException, ResourceFactory.createTypedLiteral(BigInteger.valueOf(5))); - m.add(workerQueryRes1, IPROP.queryID, queryRes1); - - // workerQueryRes2 isn't complete, therefore won't be saved - m.add(workerQueryRes2, IPROP.queryID, queryRes2); - - Path testFileFolder = Path.of("src/test/resources/storage/csv_test_files/"); - - return Arguments.of(Named.of(String.format("One simple tasks with one faulty entry. | ExpectedFolder: %s | Metrics: %s", testFileFolder, metrics.stream().map(Metric::getAbbreviation).toList()), new Suite(List.of(m), metrics, testFileFolder))); - } - - @AfterEach - public void cleanup() throws IOException { - FileUtils.deleteDirectory(this.folder.toFile()); - } - - public static Stream data() { - return Stream.of( - createTestData1() - ); - } - - @ParameterizedTest - @MethodSource("data") - @DisplayName("Test CSVStorage") - public void testCSVStorage(Suite suite) throws IOException { - for (Model m : suite.taskResults) { - // Metrics need to be set here, because the CSVStorage uses the manager to store the results - MetricManager.setMetrics(suite.metrics); - - // Test Initialisation - assertDoesNotThrow(() -> storage = new CSVStorage(this.folder.toAbsolutePath().toString()), "Initialisation failed"); - assertTrue(Files.exists(this.suiteFolder), String.format("Result folder (%s) doesn't exist", this.suiteFolder)); - storage.storeResult(m); - - List expectedFiles; - try (Stream s = Files.list(suite.expectedFolder)) { - expectedFiles = s.toList(); - } - - for (Path expectedFile : expectedFiles) { - Path actualFile = suiteFolder.resolve(expectedFile.getFileName()); - assertTrue(Files.exists(actualFile), String.format("File (%s) doesn't exist", actualFile)); - assertDoesNotThrow(() -> compareCSVFiles(expectedFile, actualFile)); - } - } - } - - private void compareCSVFiles(Path expected, Path actual) throws IOException, CsvException { - try (CSVReader readerExpected = new CSVReader(new FileReader(expected.toFile())); - CSVReader readerActual = new CSVReader(new FileReader(actual.toFile()))) { - String[] headerExpected = readerExpected.readNext(); - String[] headerActual = readerActual.readNext(); - assertEquals(headerExpected.length, headerActual.length, String.format("Headers don't match. Actual: %s, Expected: %s", Arrays.toString(headerActual), Arrays.toString(headerExpected))); - for (int i = 0; i < headerExpected.length; i++) { - assertEquals(headerExpected[i], headerActual[i], String.format("Headers don't match. Actual: %s, Expected: %s", Arrays.toString(headerActual), Arrays.toString(headerExpected))); - } - - List expectedValues = new ArrayList<>(readerExpected.readAll()); - List actualValues = new ArrayList<>(readerActual.readAll()); - - for (String[] expectedLine : expectedValues) { - List sameLines = actualValues.stream().filter(x -> { - for (int i = 0; i < expectedLine.length; i++) { - if (!expectedLine[i].equals(x[i])) return false; - } - return true; - }).toList(); - - assertFalse(sameLines.isEmpty(), String.format("Line (%s) not found in actual file", Arrays.toString(expectedLine))); - actualValues.remove(sameLines.get(0)); - } - assertTrue(actualValues.isEmpty(), String.format("Actual file contains more lines than expected. Lines: %s", actualValues.stream().map(x -> "[" + String.join(", ", x) + "]").collect(Collectors.joining("\n")))); - } - } - - private record Suite(List taskResults, List metrics, Path expectedFolder) {} -} diff --git a/src/test/java/org/aksw/iguana/cc/tasks/storage/impl/NTFileStorageTest.java b/src/test/java/org/aksw/iguana/cc/tasks/storage/impl/NTFileStorageTest.java deleted file mode 100644 index 79739c6a3..000000000 --- a/src/test/java/org/aksw/iguana/cc/tasks/storage/impl/NTFileStorageTest.java +++ /dev/null @@ -1,71 +0,0 @@ -/** - * - */ -package org.aksw.iguana.cc.tasks.storage.impl; - -import org.aksw.iguana.cc.tasks.stresstest.storage.impl.NTFileStorage; -import org.aksw.iguana.cc.tasks.stresstest.storage.Storage; -import org.apache.jena.rdf.model.*; -import org.apache.jena.vocabulary.RDFS; -import org.junit.Test; - -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * - * This will test the NTFileStorage in short. - * - * @author f.conrads - * - */ -public class NTFileStorageTest { - - - @Test - public void dataTest() throws IOException{ - Storage store = new NTFileStorage("results_test2.nt"); - - new File("results_test2.nt").delete(); - - Model m = ModelFactory.createDefaultModel(); - m.read(new FileReader("src/test/resources/nt/results_test1.nt"), null, "N-TRIPLE"); - - store.storeResult(m); - assertEqual("results_test2.nt","src/test/resources/nt/results_test1.nt", true); - new File("results_test2.nt").delete(); - - } - - /** - * Checks if two ntriple files are equal by loading them into a model and check if they have the same size - * and by removing the actual model from the expected, if the new size after removal equals 0 they are the same - * - * @param actualFile - * @param expectedFile - * @throws IOException - */ - public void assertEqual(String actualFile, String expectedFile, boolean ignoreDate) throws IOException{ - Model expected = ModelFactory.createDefaultModel(); - expected.read(new FileReader(expectedFile), null, "N-TRIPLE"); - Model actual = ModelFactory.createDefaultModel(); - actual.read(new FileReader(actualFile), null, "N-TRIPLE"); - assertEquals(expected.size(), actual.size()); - expected.remove(actual); - if(!ignoreDate){ - //Remove startDate as they are different, just check if actual contains a start date - Property startDate =ResourceFactory.createProperty(RDFS.getURI()+"startDate"); - assertTrue(actual.contains(null, startDate, (RDFNode)null)); - List stmts = expected.listStatements(null, startDate, (RDFNode)null).toList(); - assertEquals(1, stmts.size()); - expected.remove(stmts); - } - - assertEquals(0, expected.size()); - } -} diff --git a/src/test/java/org/aksw/iguana/cc/tasks/storage/impl/RDFFileStorageTest.java b/src/test/java/org/aksw/iguana/cc/tasks/storage/impl/RDFFileStorageTest.java deleted file mode 100644 index 757d0e6a0..000000000 --- a/src/test/java/org/aksw/iguana/cc/tasks/storage/impl/RDFFileStorageTest.java +++ /dev/null @@ -1,74 +0,0 @@ -/** - * - */ -package org.aksw.iguana.cc.tasks.storage.impl; - -import org.aksw.iguana.cc.tasks.stresstest.storage.impl.RDFFileStorage; -import org.aksw.iguana.cc.tasks.stresstest.storage.Storage; -import org.apache.jena.rdf.model.*; -import org.apache.jena.riot.RDFLanguages; -import org.apache.jena.vocabulary.RDFS; -import org.junit.Test; - -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * - * This will test the RDFFileStorage in short. - * - * @author l.conrads - * - */ -public class RDFFileStorageTest { - - - @Test - public void dataTest() throws IOException{ - Storage store = new RDFFileStorage("results_test2.ttl"); - - new File("results_test2.ttl").delete(); - - Model m = ModelFactory.createDefaultModel(); - m.read(new FileReader("src/test/resources/nt/results_test1.nt"), null, "N-TRIPLE"); - - store.storeResult(m); - - assertEqual("results_test2.ttl","src/test/resources/nt/results_test1.nt", true); - new File("results_test2.ttl").delete(); - - } - - - /** - * Checks if two ntriple files are equal by loading them into a model and check if they have the same size - * and by removing the actual model from the expected, if the new size after removal equals 0 they are the same - * - * @param actualFile - * @param expectedFile - * @throws IOException - */ - public void assertEqual(String actualFile, String expectedFile, boolean ignoreDate) throws IOException{ - Model expected = ModelFactory.createDefaultModel(); - expected.read(new FileReader(expectedFile), null, "N-TRIPLE"); - Model actual = ModelFactory.createDefaultModel(); - actual.read(new FileReader(actualFile), null, RDFLanguages.filenameToLang(actualFile).getName()); - assertEquals(expected.size(), actual.size()); - expected.remove(actual); - if(!ignoreDate){ - //Remove startDate as they are different, just check if actual contains a start date - Property startDate =ResourceFactory.createProperty(RDFS.getURI()+"startDate"); - assertTrue(actual.contains(null, startDate, (RDFNode)null)); - List stmts = expected.listStatements(null, startDate, (RDFNode)null).toList(); - assertEquals(1, stmts.size()); - expected.remove(stmts); - } - - assertEquals(0, expected.size()); - } -} diff --git a/src/test/java/org/aksw/iguana/cc/tasks/storage/impl/TriplestoreStorageTest.java b/src/test/java/org/aksw/iguana/cc/tasks/storage/impl/TriplestoreStorageTest.java deleted file mode 100644 index c1d883f87..000000000 --- a/src/test/java/org/aksw/iguana/cc/tasks/storage/impl/TriplestoreStorageTest.java +++ /dev/null @@ -1,84 +0,0 @@ -/** - * - */ -package org.aksw.iguana.cc.tasks.storage.impl; - -import org.aksw.iguana.cc.tasks.stresstest.storage.impl.TriplestoreStorage; -import org.aksw.iguana.commons.constants.COMMON; -import org.aksw.iguana.cc.tasks.ServerMock; -import org.apache.jena.rdf.model.Model; -import org.apache.jena.rdf.model.ModelFactory; -import org.apache.jena.rdf.model.ResourceFactory; -import org.junit.After; -import org.junit.Test; -import org.simpleframework.http.core.ContainerServer; -import org.simpleframework.transport.connect.SocketConnection; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.net.SocketAddress; - -import static org.junit.Assert.assertEquals; - -/** - * Will test if the TriplestoreStorage sends the correct INSERT command to a Mock Server - * - * @author f.conrads - * - */ -public class TriplestoreStorageTest { - - private static final int FAST_SERVER_PORT = 8023; - private ServerMock fastServerContainer; - private ContainerServer fastServer; - private SocketConnection fastConnection; - - private String metaExp = "INSERT DATA {\n" + - " .\n" + - " .\n" + - " .\n" + - " \"dbpedia\" .\n" + - " .\n" + - " \"virtuoso\" .\n" + - " .\n" + - " \"???\"^^ .\n" + - " .\n" + - " .\n" + - " .\n" + - " .\n" + - " .\n" + - "}"; - - private String dataExp = "INSERT DATA {\n"+ -" \"c\" .\n"+ -"}"; - - /** - * @throws IOException - */ - @After - public void close() throws IOException { - fastConnection.close(); - } - - /** - * @throws IOException - */ - @Test - public void dataTest() throws IOException{ - fastServerContainer = new ServerMock(); - fastServer = new ContainerServer(fastServerContainer); - fastConnection = new SocketConnection(fastServer); - SocketAddress address1 = new InetSocketAddress(FAST_SERVER_PORT); - fastConnection.connect(address1); - - String host = "http://localhost:8023"; - TriplestoreStorage store = new TriplestoreStorage(host, host); - - Model m = ModelFactory.createDefaultModel(); - m.add(ResourceFactory.createResource(COMMON.RES_BASE_URI+"a"), ResourceFactory.createProperty(COMMON.PROP_BASE_URI+"b") , "c"); - store.storeResult(m); - assertEquals(dataExp.trim(),fastServerContainer.getActualContent().trim()); - } - -} diff --git a/src/test/java/org/aksw/iguana/cc/tasks/stresstest/StresstestTest.java b/src/test/java/org/aksw/iguana/cc/tasks/stresstest/StresstestTest.java deleted file mode 100644 index 1ab15f22f..000000000 --- a/src/test/java/org/aksw/iguana/cc/tasks/stresstest/StresstestTest.java +++ /dev/null @@ -1,134 +0,0 @@ -package org.aksw.iguana.cc.tasks.stresstest; - -import org.aksw.iguana.cc.config.elements.ConnectionConfig; -import org.aksw.iguana.cc.tasks.MockupStorage; -import org.aksw.iguana.cc.tasks.stresstest.metrics.MetricManager; -import org.aksw.iguana.cc.tasks.stresstest.metrics.impl.EachExecutionStatistic; -import org.aksw.iguana.cc.worker.MockupWorker; -import org.aksw.iguana.cc.worker.Worker; -import org.aksw.iguana.cc.tasks.stresstest.storage.StorageManager; -import org.junit.Ignore; -import org.junit.Test; - -import java.time.Instant; -import java.util.*; - -import static org.junit.Assert.assertEquals; - -public class StresstestTest { - - // test correct # of worker creation, meta data and warmup - private final String[] queries = new String[]{"a", "b"}; - private final String[] queries2 = new String[]{"b", "c"}; - - private List> getWorkers(int threads, String[] queries) { - List> workers = new ArrayList<>(); - Map workerConfig = new HashMap<>(); - workerConfig.put("className", MockupWorker.class.getCanonicalName()); - workerConfig.put("stringQueries", queries); - workerConfig.put("threads", threads); - workers.add(workerConfig); - return workers; - } - - private ConnectionConfig getConnection() { - ConnectionConfig con = new ConnectionConfig(); - con.setName("test"); - con.setEndpoint("test/sparql"); - return con; - } - - private void init(){ - StorageManager storageManager = StorageManager.getInstance(); - MetricManager.setMetrics(List.of(new EachExecutionStatistic())); - MockupStorage storage = new MockupStorage(); - storageManager.addStorage(storage); - } - - @Test - public void checkStresstestNoQM() { - - Stresstest task = new Stresstest(getWorkers(2, this.queries), 10); - task.init(new String[]{"1", "1/1", "1/1/1"}, "test", getConnection()); - - init(); - - task.execute(); - - //2 queries in mix, 10 executions on 2 workers -> 40 queries - assertEquals(40, task.getExecutedQueries()); - } - - @Test - public void checkStresstestTL() { - - Stresstest task = new Stresstest(5000, getWorkers(2, this.queries)); - - task.init(new String[]{"1", "1/1", "1/1/1"}, "test", getConnection()); - - init(); - - Instant start = Instant.now(); - task.execute(); - Instant end = Instant.now(); - //give about 200milliseconds time for init and end stuff - assertEquals(5000.0, end.toEpochMilli() - start.toEpochMilli(), 300.0); - - } - - @Test - @Ignore("This test doesn't always pass. It expects a timing that is not guaranteed (or necessary).") - public void warmupTest() { - //check if not executing - Stresstest task = new Stresstest(5000, getWorkers(2, this.queries)); - - task.init(new String[]{"1", "1/1", "1/1/1"}, "test", getConnection()); - Instant start = Instant.now(); - assertEquals(0, task.warmup()); - Instant end = Instant.now(); - assertEquals(0.0, end.toEpochMilli() - start.toEpochMilli(), 5.0); - //check if executing - - Map warmup = new LinkedHashMap<>(); - warmup.put("workers", getWorkers(2, this.queries)); - warmup.put("timeLimit", 350); - - task = new Stresstest(5000, getWorkers(2, this.queries), warmup); - - task.init(new String[]{"1", "1/1", "1/1/1"}, "test", getConnection()); - start = Instant.now(); - long queriesExecuted = task.warmup(); - end = Instant.now(); - // might sadly be 400 or 500 as the warmup works in 100th steps, also overhead, as long as executed Queries are 6 its fine - assertEquals(350.0, end.toEpochMilli() - start.toEpochMilli(), 250.0); - //each worker could execute 3 query - assertEquals(6, queriesExecuted); - - } - - @Test - public void workerCreationTest() { - List> worker = getWorkers(2, this.queries); - worker.addAll(getWorkers(1, this.queries2)); - Stresstest task = new Stresstest(5000, worker); - - task.init(new String[]{"1", "1/1", "1/1/1"}, "test", getConnection()); - List workers = task.workers; - assertEquals(3, workers.size()); - int q1 = 0; - int q2 = 0; - // alittle bit hacky but should be sufficient - for (Worker w : workers) { - MockupWorker mockupWorker = (MockupWorker) w; - String[] queries = mockupWorker.getStringQueries(); - if (Arrays.hashCode(queries) == Arrays.hashCode(this.queries)) { - q1++; - } else if (Arrays.hashCode(queries) == Arrays.hashCode(this.queries2)) { - q2++; - } - } - assertEquals(2, q1); - assertEquals(1, q2); - - } -} diff --git a/src/test/java/org/aksw/iguana/cc/utils/CLIProcessManagerTest.java b/src/test/java/org/aksw/iguana/cc/utils/CLIProcessManagerTest.java deleted file mode 100644 index f16cf6319..000000000 --- a/src/test/java/org/aksw/iguana/cc/utils/CLIProcessManagerTest.java +++ /dev/null @@ -1,64 +0,0 @@ -package org.aksw.iguana.cc.utils; - -import org.junit.Ignore; -import org.junit.Test; - -import java.io.IOException; - -import static org.junit.Assert.*; - -@Ignore("CLI doesn't work right now") -public class CLIProcessManagerTest { - - @Test - public void execTest() throws InterruptedException { - //create process - Process p = CLIProcessManager.createProcess("echo \"abc\"; wait 1m"); - //destroy process - assertTrue(p.isAlive()); - CLIProcessManager.destroyProcess(p); - //give OS a little bit of time to destroy process - Thread.sleep(50); - assertFalse(p.isAlive()); - - } - - @Test - public void countLinesSuccessfulTest() throws IOException, InterruptedException { - //create - Process p = CLIProcessManager.createProcess("echo \"abc\"; wait 100; echo \"t\nt\nabc: test ended suffix\"; wait 1m;"); - //count Lines until "test ended" occured - Thread.sleep(100); - assertTrue(CLIProcessManager.isReaderReady(p)); - - assertEquals(3, CLIProcessManager.countLinesUntilStringOccurs(p, "test ended", "failed")); - //destroy - CLIProcessManager.destroyProcess(p); - //give OS a little bit of time to destroy process - Thread.sleep(50); - assertFalse(p.isAlive()); - - } - - @Test - public void countLinesFailTest() throws IOException, InterruptedException { - //create - Process p = CLIProcessManager.createProcess("echo \"abc\"; wait 100; echo \"abc: failed suffix\"; wait 1m;"); - Thread.sleep(100); - assertTrue(CLIProcessManager.isReaderReady(p)); - //count Lines until "test ended" occured - try{ - CLIProcessManager.countLinesUntilStringOccurs(p, "test ended", "failed"); - assertTrue("Test did not end in IOException", false); - }catch (IOException e){ - assertTrue(true); - } - //destroy - CLIProcessManager.destroyProcess(p); - //give OS a little bit of time to destroy process - Thread.sleep(50); - assertFalse(p.isAlive()); - - } - -} diff --git a/src/test/java/org/aksw/iguana/cc/utils/FileUtilsTest.java b/src/test/java/org/aksw/iguana/cc/utils/FileUtilsTest.java index 80a567c68..f213d73e0 100644 --- a/src/test/java/org/aksw/iguana/cc/utils/FileUtilsTest.java +++ b/src/test/java/org/aksw/iguana/cc/utils/FileUtilsTest.java @@ -1,9 +1,9 @@ package org.aksw.iguana.cc.utils; -import org.junit.Test; -import org.junit.experimental.runners.Enclosed; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; import java.io.*; import java.nio.charset.StandardCharsets; @@ -15,157 +15,71 @@ import static java.nio.file.Files.createTempFile; import static org.apache.commons.io.FileUtils.writeStringToFile; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.*; -@RunWith(Enclosed.class) public class FileUtilsTest { - - @RunWith(Parameterized.class) - public static class TestGetLineEnding { - private static class TestData { - public Path file; - public String expectedLineEnding; - - public TestData(String expectedLineEnding) { - this.expectedLineEnding = expectedLineEnding; - - - } - } - - public TestGetLineEnding(String expectedLineEnding) throws IOException { - this.data = new TestData(expectedLineEnding); - this.data.file = createTempFile("TestGetLineEnding", ".txt"); - this.data.file.toFile().deleteOnExit(); - writeStringToFile(this.data.file.toFile(), "a" + this.data.expectedLineEnding + "b" + this.data.expectedLineEnding, StandardCharsets.UTF_8); - } - - private final TestData data; - - @Parameterized.Parameters - public static Collection data() { - return List.of( - "\n", /* unix */ - "\r", /* old mac */ - "\r\n" /* windows */ - ); - } - - @Test - public void testGetLineEndings() throws IOException { - assertEquals(FileUtils.getLineEnding(this.data.file.toString()), this.data.expectedLineEnding); + public static Path createTestFileWithLines(List content, String lineEnding) throws IOException { + final var file = createTempFile("getHashTest", ".txt"); + for (String s : content) { + writeStringToFile(file.toFile(), s + lineEnding, StandardCharsets.UTF_8, true); } + file.toFile().deleteOnExit(); + return file; } - @RunWith(Parameterized.class) - public static class TestIndexStream { - - private final TestData data; - - public TestIndexStream(TestData data) { - this.data = data; - } - - public static class TestData { - /** - * String to be separated - */ - String string; - /** - * Separating sequence - */ - String separator; - - /** - * List of [offset, length] arrays - */ - List index; - - public TestData(String string, String separator, List index) { - this.string = string; - this.separator = separator; - this.index = index; - } - } - @Parameterized.Parameters - public static Collection data() { - - - return List.of( - new TestData("", "a", Arrays.asList(new long[]{0, 0})), - new TestData("a", "a", Arrays.asList(new long[]{0, 0}, new long[]{1, 0})), - new TestData("abc", "b", Arrays.asList(new long[]{0, 1}, new long[]{2, 1})), - new TestData("1\n2", "\n", Arrays.asList(new long[]{0, 1}, new long[]{2, 1})), - new TestData("1\t2", "\t", Arrays.asList(new long[]{0, 1}, new long[]{2, 1})), - new TestData("abcbd", "b", Arrays.asList(new long[]{0, 1}, new long[]{2, 1}, new long[]{4, 1})), - new TestData("aab", "ab", Arrays.asList(new long[]{0, 1}, new long[]{3, 0})), - new TestData("aaaabaabaa", "ab", Arrays.asList(new long[]{0, 3}, new long[]{5, 1}, new long[]{8, 2})), - new TestData("1\n\t\n2", "\n\t\n", Arrays.asList(new long[]{0, 1}, new long[]{4, 1})) - - ); - } - - @Test - public void testIndexingStrings() throws IOException { - //check if hash abs works - - List index = FileUtils.indexStream(data.separator, new ByteArrayInputStream(data.string.getBytes())); - - assertEquals(data.index.size(), index.size()); - for (int i = 0; i < index.size(); i++) { - assertArrayEquals(data.index.get(i), index.get(i)); - } - } + public static Path createTestFileWithContent(String content) throws IOException { + final var file = createTempFile("getHashTest", ".txt"); + writeStringToFile(file.toFile(), content, StandardCharsets.UTF_8, false); + file.toFile().deleteOnExit(); + return file; } - @RunWith(Parameterized.class) - public static class ParameterizedTest { - private final Path file; - - private final String content; - - public ParameterizedTest(String content) throws IOException { - this.file = createTempFile("getHashTest", ".txt"); - writeStringToFile(this.file.toFile(), content, StandardCharsets.UTF_8); - this.file.toFile().deleteOnExit(); - - this.content = content; - } + @ParameterizedTest + @ValueSource(strings = {"\n", "\r", "\r\n"}) + public void testGetLineEndings(String ending) throws IOException { + final var file = createTestFileWithLines(List.of("a", "b"), ending); + assertEquals(FileUtils.getLineEnding(file), ending); + } - @Parameterized.Parameters - public static Collection data() { + public record IndexTestData( + String content, // String to be separated + String separator, + List indices // List of [offset, length] arrays + ) {} + + public static Collection data() { + return List.of( + new IndexTestData("", "a", Arrays.asList(new long[]{0, 0})), + new IndexTestData("a", "a", Arrays.asList(new long[]{0, 0}, new long[]{1, 0})), + new IndexTestData("abc", "b", Arrays.asList(new long[]{0, 1}, new long[]{2, 1})), + new IndexTestData("1\n2", "\n", Arrays.asList(new long[]{0, 1}, new long[]{2, 1})), + new IndexTestData("1\t2", "\t", Arrays.asList(new long[]{0, 1}, new long[]{2, 1})), + new IndexTestData("abcbd", "b", Arrays.asList(new long[]{0, 1}, new long[]{2, 1}, new long[]{4, 1})), + new IndexTestData("aab", "ab", Arrays.asList(new long[]{0, 1}, new long[]{3, 0})), + new IndexTestData("aaaabaabaa", "ab", Arrays.asList(new long[]{0, 3}, new long[]{5, 1}, new long[]{8, 2})), + new IndexTestData("1\n\t\n2", "\n\t\n", Arrays.asList(new long[]{0, 1}, new long[]{4, 1})) + ); + } - return Arrays.asList( - UUID.randomUUID().toString(), - UUID.randomUUID().toString(), - UUID.randomUUID().toString(), - UUID.randomUUID().toString() - ); + @ParameterizedTest + @MethodSource("data") + public void testIndexingStrings(IndexTestData data) throws IOException { + List index = FileUtils.indexStream(data.separator, new ByteArrayInputStream(data.content.getBytes())); + assertEquals(data.indices.size(), index.size()); + for (int i = 0; i < index.size(); i++) { + assertArrayEquals(data.indices.get(i), index.get(i)); } + } - @Test - public void getHashTest(){ - //check if hash abs works + @Test + public void getHashTest() throws IOException { + for (int i = 0; i < 10; i++) { + String content = UUID.randomUUID().toString(); + final var file = createTestFileWithContent(content); final int expected = Math.abs(content.hashCode()); - final int actual = FileUtils.getHashcodeFromFileContent(this.file.toString()); + final int actual = FileUtils.getHashcodeFromFileContent(file); assertTrue(actual >= 0); assertEquals(expected, actual); } } - - public static class NonParameterizedTest { - @Test - public void readTest() throws IOException { - - Path file = createTempFile("readTest", ".txt"); - file.toFile().deleteOnExit(); - String expectedString = UUID.randomUUID() + "\n\t\r" + UUID.randomUUID() + "\n"; - writeStringToFile(file.toFile(), expectedString, StandardCharsets.UTF_8); - - //read whole content - String actualString = FileUtils.readFile(file.toString()); - - assertEquals(expectedString, actualString); - } - } } diff --git a/src/test/java/org/aksw/iguana/cc/utils/IndexedQueryReaderTest.java b/src/test/java/org/aksw/iguana/cc/utils/IndexedQueryReaderTest.java index 5166940d5..b279c8153 100644 --- a/src/test/java/org/aksw/iguana/cc/utils/IndexedQueryReaderTest.java +++ b/src/test/java/org/aksw/iguana/cc/utils/IndexedQueryReaderTest.java @@ -1,99 +1,169 @@ package org.aksw.iguana.cc.utils; -import org.junit.Test; -import org.junit.experimental.runners.Enclosed; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; +import java.io.StringWriter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; import java.util.List; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -@RunWith(Enclosed.class) public class IndexedQueryReaderTest { - @RunWith(Parameterized.class) - public static class ParameterizedTest { + private static Path tempDir; - IndexedQueryReader reader; + @BeforeAll + public static void createTestFolder() throws IOException { + tempDir = Files.createTempDirectory("iguana-indexed-query-reader-test"); + } - @Parameterized.Parameters - public static Collection data() { - return Arrays.asList(new Object[][]{ - {"src/test/resources/readLineTestFile1.txt"}, - {"src/test/resources/readLineTestFile2.txt"}, - {"src/test/resources/readLineTestFile3.txt"} - }); - } + @AfterAll + public static void removeData() throws IOException { + org.apache.commons.io.FileUtils.deleteDirectory(tempDir.toFile()); + } - public ParameterizedTest(String path) throws IOException { - reader = IndexedQueryReader.make(path); + private record TestData ( + Path filepath, + String separator, + List expectedStrings + ) {} + + private static TestData createTestFile(String content, String separator, boolean emptyBegin, boolean leadingEmptyLine, int number, int spacing) throws IOException { + final var file = Files.createTempFile(tempDir, "line", "queries.txt"); + final var writer = new StringWriter(); + final var lines = new ArrayList(); + for (int i = (emptyBegin ? -1 : 0); i < (number * spacing) + 1; i++) { + if (i % spacing == 0) { + writer.append(content + i); + lines.add(content + i); + } + if (leadingEmptyLine || i != number * spacing) { + writer.append(separator); + } } + Files.writeString(file, writer.toString()); + return new TestData(file, separator, lines); + } - @Test - public void testIndexingWithLineEndings() throws IOException { - assertEquals("line 1", reader.readQuery(0)); - assertEquals("line 2", reader.readQuery(1)); - assertEquals("line 3", reader.readQuery(2)); - assertEquals("line 4", reader.readQuery(3)); + public static List indexWithLineEndingData() throws IOException { + final var out = new ArrayList(); + + final var numbers = List.of(1, 5, 10); + final var spacings = List.of(1, 2, 5, 10, 100, 1000000); + final var separators = List.of("\n", "\r\n", "\r"); + final var emptyBegins = List.of(true, false); + final var leadingEmptyLines = List.of(true, false); + + // cartesian product + for (var number : numbers) { + for (var spacing : spacings) { + for (var separator : separators) { + for (var emptyBegin : emptyBegins) { + for (var leadingEmptyLine : leadingEmptyLines) { + out.add(Arguments.of(createTestFile("line: ", separator, emptyBegin, leadingEmptyLine, number, spacing))); + } + } + } + } } - } - public static class NonParameterizedTest { - @Test - public void testIndexingWithBlankLines() throws IOException { - IndexedQueryReader reader = IndexedQueryReader.makeWithEmptyLines("src/test/resources/utils/indexingtestfile3.txt"); - String le = FileUtils.getLineEnding("src/test/resources/utils/indexingtestfile3.txt"); + return out; + } - assertEquals(" line 1" + le + "line 2", reader.readQuery(0)); - assertEquals("line 3", reader.readQuery(1)); - assertEquals("line 4" + le + "line 5", reader.readQuery(2)); + public static List indexWithBlankLinesData() throws IOException { + final var out = new ArrayList(); + + final var numbers = List.of(1, 5, 10, 100, 10000); + final var spacings = List.of(2); + final var separators = List.of("\n", "\r\n", "\r"); + final var emptyBegins = List.of(false); + final var leadingEmptyLines = List.of(false); + + // cartesian product + for (var number : numbers) { + for (var spacing : spacings) { + for (var separator : separators) { + for (var emptyBegin : emptyBegins) { + for (var leadingEmptyLine : leadingEmptyLines) { + out.add(Arguments.of(createTestFile(String.format("this is %s line: ", separator), separator, emptyBegin, leadingEmptyLine, number, spacing))); + out.add(Arguments.of(createTestFile("line: ", separator, emptyBegin, leadingEmptyLine, number, spacing))); + out.add(Arguments.of(createTestFile(String.format("make this %s three lines %s long: ", separator, separator), separator, emptyBegin, leadingEmptyLine, number, spacing))); + } + } + } + } } + + return out; } - @RunWith(Parameterized.class) - public static class TestCustomSeparator { - private static class TestData { - public String filepath; - public String separator; - public String[] expectedStrings; - - public TestData(String filepath, String separator, String[] expectedStrings) { - this.filepath = filepath; - this.separator = separator; - this.expectedStrings = expectedStrings; + public static List indexWithCustomSeparatorData() throws IOException { + final var out = new ArrayList(); + + final var numbers = List.of(1, 5, 10, 100, 10000); + final var spacings = List.of(1); + final var separators = List.of("\n", "\r\n", "\r", "\n+++\n", "\t\t\t", "test", "###$"); + final var emptyBegins = List.of(false); + final var leadingEmptyLines = List.of(false); + + // cartesian product + for (var number : numbers) { + for (var spacing : spacings) { + for (var separator : separators) { + for (var emptyBegin : emptyBegins) { + for (var leadingEmptyLine : leadingEmptyLines) { + out.add(Arguments.of(createTestFile("line: ", separator, emptyBegin, leadingEmptyLine, number, spacing))); + } + } + } } } - private TestData data; + final var file1 = Files.createTempFile(tempDir, "iguana", "queries.txt"); + final var file2 = Files.createTempFile(tempDir, "iguana", "queries.txt"); + Files.writeString(file1, "a####$b"); + Files.writeString(file2, "a21212111b"); + + out.add(Arguments.of(new TestData(file1, "###$", List.of("a#", "b")))); + out.add(Arguments.of(new TestData(file2, "211", List.of("a2121", "1b")))); + + return out; + } - public TestCustomSeparator(TestData data) { - this.data = data; + @ParameterizedTest + @MethodSource("indexWithLineEndingData") + public void testIndexingWithLineEndings(TestData data) throws IOException { + var reader = IndexedQueryReader.make(data.filepath); + for (int i = 0; i < data.expectedStrings.size(); i++) { + assertEquals(data.expectedStrings.get(i), reader.readQuery(i)); } + assertEquals(data.expectedStrings.size(), reader.size()); + } - @Parameterized.Parameters - public static Collection data() throws IOException { - // all the files should have the same line ending - String le = FileUtils.getLineEnding("src/test/resources/utils/indexingtestfile1.txt"); - return List.of( - new TestData("src/test/resources/utils/indexingtestfile1.txt", "#####" + le, new String[]{"line 1" + le, le + "line 2" + le}), - new TestData("src/test/resources/utils/indexingtestfile2.txt", "#####" + le, new String[]{"line 0" + le, "line 1" + le + "#####"}), - new TestData("src/test/resources/utils/indexingtestfile4.txt", "###$", new String[]{"a#", "b"}), - new TestData("src/test/resources/utils/indexingtestfile5.txt", "211", new String[]{"a21", "b"}) - ); + @ParameterizedTest + @MethodSource("indexWithBlankLinesData") + public void testIndexingWithBlankLines(TestData data) throws IOException { + IndexedQueryReader reader = IndexedQueryReader.makeWithEmptyLines(data.filepath); + for (int i = 0; i < data.expectedStrings.size(); i++) { + assertEquals(data.expectedStrings.get(i), reader.readQuery(i)); } + assertEquals(data.expectedStrings.size(), reader.size()); + } - @Test - public void testIndexingWithCustomSeparator() throws IOException { - IndexedQueryReader reader = IndexedQueryReader.makeWithStringSeparator(this.data.filepath, this.data.separator); - for (int i = 0; i < this.data.expectedStrings.length; i++) { - String read = reader.readQuery(i); - assertEquals(this.data.expectedStrings[i], read); - } - assertEquals(this.data.expectedStrings.length, reader.readQueries().size()); + @ParameterizedTest + @MethodSource("indexWithCustomSeparatorData") + public void testIndexingWithCustomSeparator(TestData data) throws IOException { + IndexedQueryReader reader = IndexedQueryReader.makeWithStringSeparator(data.filepath, data.separator); + for (int i = 0; i < data.expectedStrings.size(); i++) { + assertEquals(data.expectedStrings.get(i), reader.readQuery(i)); } + assertEquals(data.expectedStrings.size(), reader.size()); } } diff --git a/src/test/java/org/aksw/iguana/cc/utils/SPARQLQueryStatisticsTest.java b/src/test/java/org/aksw/iguana/cc/utils/SPARQLQueryStatisticsTest.java deleted file mode 100644 index be05c8757..000000000 --- a/src/test/java/org/aksw/iguana/cc/utils/SPARQLQueryStatisticsTest.java +++ /dev/null @@ -1,62 +0,0 @@ -package org.aksw.iguana.cc.utils; - -import org.apache.jena.query.Query; -import org.apache.jena.query.QueryFactory; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.util.ArrayList; -import java.util.Collection; - -import static org.junit.Assert.assertEquals; - -@RunWith(Parameterized.class) -public class SPARQLQueryStatisticsTest { - - - private final String query; - private final double size; - private final int[] stats; - - @Parameterized.Parameters - public static Collection data(){ - Collection testData = new ArrayList(); - testData.add(new Object[]{"SELECT * {?s ?p ?o}", 1, new int[]{0, 0, 0, 0, 0, 0, 0, 0, 1}}); - testData.add(new Object[]{"SELECT * {?s ?p ?o. ?o ?p1 ?t}", 1, new int[]{0, 0, 0, 0, 0, 0, 0, 0, 2}}); - testData.add(new Object[]{"SELECT * {?s ?p ?o. ?o ?p1 ?t. FILTER (?t = \"test\")}", 1, new int[]{0, 1, 0, 0, 0, 0, 0, 0, 2}}); - //implicit groupBY as aggr - testData.add(new Object[]{"SELECT (COUNT(?s) AS ?co) {?s ?p ?o. ?o ?p1 ?t. FILTER (?t = \"test\")}", 1, new int[]{1, 1, 0, 0, 0, 1, 0, 0, 2}}); - testData.add(new Object[]{"SELECT * {?s ?p ?o. ?o ?p1 ?t. FILTER (?t = \"test\")} ORDER BY ?s", 1, new int[]{0, 1, 0, 0, 0, 0, 0, 1, 2}}); - testData.add(new Object[]{"SELECT ?s {?s ?p ?o. ?o ?p1 ?t. FILTER (?t = \"test\")} GROUP BY ?s", 1, new int[]{0, 1, 0, 0, 0, 1, 0, 0, 2}}); - testData.add(new Object[]{"SELECT ?o {{?s ?p ?o OPTIONAL {?o ?u ?s} } UNION { ?o ?p1 ?t}} OFFSET 10", 1, new int[]{0, 0, 1, 1, 0, 0, 1, 0, 3}}); - //implicit groupBY as aggr - testData.add(new Object[]{"SELECT * {?s ?p ?o} HAVING(COUNT(?s) > 1)", 1, new int[]{1, 0, 0, 0, 1, 1, 0, 0, 1}}); - - return testData; - } - - public SPARQLQueryStatisticsTest(String query, double size, int[] stats){ - this.query=query; - this.size=size; - this.stats=stats; - } - - @Test - public void checkCorrectStats(){ - SPARQLQueryStatistics qs = new SPARQLQueryStatistics(); - Query q = QueryFactory.create(this.query); - qs.getStatistics(q); - assertEquals(stats[0], qs.aggr); - assertEquals(stats[1], qs.filter); - assertEquals(stats[2], qs.optional); - assertEquals(stats[3], qs.union); - assertEquals(stats[4], qs.having); - assertEquals(stats[5], qs.groupBy); - assertEquals(stats[6], qs.offset); - assertEquals(size, qs.size, 0); - assertEquals(stats[7], qs.orderBy); - assertEquals(stats[8], qs.triples); - } - -} diff --git a/src/test/java/org/aksw/iguana/cc/utils/ServerMock.java b/src/test/java/org/aksw/iguana/cc/utils/ServerMock.java deleted file mode 100644 index b4485ae3c..000000000 --- a/src/test/java/org/aksw/iguana/cc/utils/ServerMock.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.aksw.iguana.cc.utils; - -import org.aksw.iguana.cc.lang.impl.SPARQLLanguageProcessor; -import org.apache.commons.io.FileUtils; -import org.simpleframework.http.Request; -import org.simpleframework.http.Response; -import org.simpleframework.http.Status; -import org.simpleframework.http.core.Container; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; - -/** - * Server Mock representing a TS - * - * @author f.conrads - * - */ -public class ServerMock implements Container { - - private static final Logger LOGGER = LoggerFactory.getLogger(ServerMock.class); - private String actualContent; - - - @Override - public void handle(Request request, Response resp) { - String content=null; - try { - content = request.getContent(); - } catch (IOException e) { - LOGGER.error("Got exception.", e); - } - resp.setCode(Status.OK.code); - resp.setContentType(SPARQLLanguageProcessor.QUERY_RESULT_TYPE_JSON); - try { - //write answer - String resultStr = FileUtils.readFileToString(new File("src/test/resources/sparql-json-response.json"), "UTF-8"); - resp.getOutputStream().write(resultStr.getBytes()); - resp.getOutputStream().close(); - } catch (IOException e) { - LOGGER.error("Could not close Response Output Stream"); - } - } - - -} diff --git a/src/test/java/org/aksw/iguana/cc/worker/HTTPWorkerTest.java b/src/test/java/org/aksw/iguana/cc/worker/HTTPWorkerTest.java deleted file mode 100644 index a875295a1..000000000 --- a/src/test/java/org/aksw/iguana/cc/worker/HTTPWorkerTest.java +++ /dev/null @@ -1,217 +0,0 @@ -package org.aksw.iguana.cc.worker; - -import org.aksw.iguana.cc.config.elements.ConnectionConfig; -import org.aksw.iguana.cc.lang.impl.SPARQLLanguageProcessor; -import org.aksw.iguana.cc.model.QueryExecutionStats; -import org.aksw.iguana.cc.utils.FileUtils; -import org.aksw.iguana.cc.worker.impl.HttpGetWorker; -import org.aksw.iguana.cc.worker.impl.HttpPostWorker; -import org.aksw.iguana.cc.worker.impl.HttpWorker; -import org.junit.*; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.simpleframework.http.core.ContainerServer; -import org.simpleframework.transport.connect.SocketConnection; - -import java.io.File; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.net.SocketAddress; -import java.util.*; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import static org.junit.Assert.*; - -@RunWith(Parameterized.class) -public class HTTPWorkerTest { - - private static final int FAST_SERVER_PORT = 8025; - private static ContainerServer fastServer; - private static SocketConnection fastConnection; - private final String service; - private final Boolean isPost; - private final HashMap queries; - - private final String queriesFile = "src/test/resources/workers/single-query.txt"; - private final String responseType; - private final String parameter; - private final String query; - private final String queryID; - private final boolean isFail; - private String outputDir; - private final Integer fixedLatency; - private final Integer gaussianLatency; - - public HTTPWorkerTest(String query, String queryID, String responseType, String parameter, Integer fixedLatency, Integer gaussianLatency, Boolean isFail, Boolean isPost) { - this.query = query; - this.queryID = queryID; - this.responseType = responseType; - this.parameter = parameter; - this.isFail = isFail; - this.isPost = isPost; - this.fixedLatency = fixedLatency; - this.gaussianLatency = gaussianLatency; - this.service = "http://localhost:8025"; - - this.queries = new HashMap<>(); - this.queries.put("location", this.queriesFile); - - //warmup - getWorker("1").executeQuery("test", "test"); - } - - @Parameterized.Parameters - public static Collection data() { - Collection testData = new ArrayList<>(); - //get tests - testData.add(new Object[]{"Random Text", "doc1", "text/plain", "text", 100, 50, false, false}); - testData.add(new Object[]{UUID.randomUUID().toString(), UUID.randomUUID().toString(), "text/plain", "text", 100, 50, false, false}); - - testData.add(new Object[]{"Random Text", "doc1", "text/plain", "test", 100, 50, true, false}); - testData.add(new Object[]{"Random Text", "doc1", null, "text", 100, 50, false, false}); - - //post tests - testData.add(new Object[]{"Random Text", "doc1", "text/plain", "text", 100, 50, false, true}); - testData.add(new Object[]{UUID.randomUUID().toString(), UUID.randomUUID().toString(), "text/plain", "text", 100, 50, false, true}); - - testData.add(new Object[]{"Random Text", "doc1", "text/plain", "test", 100, 50, true, true}); - testData.add(new Object[]{"Random Text", "doc1", "text/plain", null, 100, 50, true, true}); - testData.add(new Object[]{"Random Text", "doc1", null, "text", 100, 50, false, true}); - - return testData; - } - - @BeforeClass - public static void startServer() throws IOException { - fastServer = new ContainerServer(new WorkerServerMock()); - fastConnection = new SocketConnection(fastServer); - SocketAddress address1 = new InetSocketAddress(FAST_SERVER_PORT); - fastConnection.connect(address1); - - } - - @AfterClass - public static void stopServer() throws IOException { - fastConnection.close(); - fastServer.stop(); - } - - @Before - public void setOutputDir() { - this.outputDir = UUID.randomUUID().toString(); - } - - @After - public void deleteFolder() throws IOException { - org.apache.commons.io.FileUtils.deleteDirectory(new File(this.outputDir)); - } - - @Test - public void testExecution() throws InterruptedException { - // check if correct param name was set - String taskID = "123/1/1/"; - - HttpWorker getWorker = getWorker(taskID); - - getWorker.executeQuery(this.query, this.queryID); - //as the result processing is in the background we have to wait for it. - Thread.sleep(1000); - Collection results = getWorker.popQueryResults(); - assertEquals(1, results.size()); - QueryExecutionStats p = results.iterator().next(); - - assertEquals(taskID, getWorker.taskID); - - assertEquals(this.queryID, p.queryID()); - if (isPost) { - assertEquals(200.0, p.executionTime(), 20.0); - } else { - assertEquals(100.0, p.executionTime(), 20.0); - } - if (isFail) { - assertEquals(-2L, p.responseCode()); - assertEquals(0L, p.resultSize()); - } else { - assertEquals(1L, p.responseCode()); - if (this.responseType != null && this.responseType.equals("text/plain")) { - assertEquals(4L, p.resultSize()); - } - if (this.responseType == null || this.responseType.equals(SPARQLLanguageProcessor.QUERY_RESULT_TYPE_JSON)) { - assertEquals(2L, p.resultSize()); - } - } - assertEquals(1, getWorker.getExecutedQueries()); - } - - private HttpWorker getWorker(String taskID) { - return getWorker(taskID, null, null); - } - - private HttpWorker getWorker(String taskID, Integer latencyFixed, Integer gaussianFixed) { - if (this.isPost) { - return new HttpPostWorker(taskID, 1, getConnection(), this.queries, null, null, latencyFixed, gaussianFixed, this.parameter, this.responseType, "application/json"); - } - return new HttpGetWorker(taskID, 1, getConnection(), this.queries, null, null, latencyFixed, gaussianFixed, this.parameter, this.responseType); - - } - - private ConnectionConfig getConnection() { - ConnectionConfig con = new ConnectionConfig(); - con.setName("test"); - con.setPassword("test"); - con.setUser("abc"); - con.setEndpoint(this.service); - con.setUpdateEndpoint(this.service); - return con; - } - - @Test - public void testWait() throws InterruptedException { - String taskID = "123/1/1/"; - HttpWorker getWorker = getWorker(taskID, this.fixedLatency, this.gaussianLatency); - ExecutorService executorService = Executors.newFixedThreadPool(1); - executorService.submit(getWorker); - long waitMS = 850; - Thread.sleep(waitMS); - getWorker.stopSending(); - executorService.shutdownNow(); - //get expected delay - int expectedDelay = 100 + this.fixedLatency + this.gaussianLatency; - if (this.isPost) { - expectedDelay += 100; - } - double expectedQueries = waitMS * 1.0 / expectedDelay; - double deltaUp = waitMS * 1.0 / (expectedDelay + this.gaussianLatency); - double deltaDown = waitMS * 1.0 / (expectedDelay - this.gaussianLatency); - double delta = Math.ceil((deltaDown - deltaUp) / 2); - assertEquals(expectedQueries, 1.0 * getWorker.getExecutedQueries(), delta); - } - - @Test - public void testWorkflow() throws InterruptedException { - // check as long as not endsignal - String taskID = "123/1/1/"; - int queryHash = FileUtils.getHashcodeFromFileContent(this.queriesFile); - - HttpWorker getWorker = getWorker(taskID); - ExecutorService executorService = Executors.newFixedThreadPool(1); - executorService.submit(getWorker); - Thread.sleep(450); - getWorker.stopSending(); - executorService.shutdownNow(); - // check correct executedQueries - long expectedSize = 4; - if (this.isPost) { - expectedSize = 2; - } - assertEquals(expectedSize, getWorker.getExecutedQueries()); - // check pop query results - Collection results = getWorker.popQueryResults(); - assertEquals(expectedSize, results.size()); - for (long i = 1; i < expectedSize; i++) { - assertTrue(getWorker.hasExecutedNoOfQueryMixes(i)); - } - assertFalse(getWorker.hasExecutedNoOfQueryMixes(expectedSize + 1)); - } -} diff --git a/src/test/java/org/aksw/iguana/cc/worker/MockupWorker.java b/src/test/java/org/aksw/iguana/cc/worker/MockupWorker.java deleted file mode 100644 index 2242f7bc0..000000000 --- a/src/test/java/org/aksw/iguana/cc/worker/MockupWorker.java +++ /dev/null @@ -1,56 +0,0 @@ -package org.aksw.iguana.cc.worker; - -import org.aksw.iguana.cc.config.elements.ConnectionConfig; -import org.aksw.iguana.cc.model.QueryExecutionStats; -import org.aksw.iguana.commons.annotation.Nullable; - -import java.util.HashMap; -import java.util.Map; - -public class MockupWorker extends AbstractWorker { - - private int counter = 0; - private final String[] queries; - - public MockupWorker(String[] stringQueries, Integer workerID, @Nullable Integer timeLimit, ConnectionConfig connection, String taskID) { - super(taskID, workerID, connection, getQueryConfig(), 0, timeLimit, 0, 0); - this.queries = stringQueries; - } - - public String[] getStringQueries() { - return queries; - } - - private static Map getQueryConfig() { - Map queryConfig = new HashMap<>(); - queryConfig.put("location", "src/test/resources/mockupq.txt"); - return queryConfig; - } - - @Override - public void executeQuery(String query, String queryID) { - long execTime = this.workerID * 10 + 100; - long responseCode; - long resultSize; - try { - Thread.sleep(execTime); - responseCode = 200; - resultSize = this.workerID * 100 + 100; - } catch (InterruptedException e) { - e.printStackTrace(); - responseCode = 400; - resultSize = 0; - } - super.addResults(new QueryExecutionStats(queryID, responseCode, execTime, resultSize)); - } - - @Override - public void getNextQuery(StringBuilder queryStr, StringBuilder queryID) { - if (this.counter >= this.queries.length) { - this.counter = 0; - } - queryStr.append(this.queries[this.counter]); - queryID.append("src/test/resources/mockupq.txt:").append(this.counter); - this.counter++; - } -} diff --git a/src/test/java/org/aksw/iguana/cc/worker/UPDATEWorkerTest.java b/src/test/java/org/aksw/iguana/cc/worker/UPDATEWorkerTest.java deleted file mode 100644 index 3b1310492..000000000 --- a/src/test/java/org/aksw/iguana/cc/worker/UPDATEWorkerTest.java +++ /dev/null @@ -1,175 +0,0 @@ -package org.aksw.iguana.cc.worker; - -import org.aksw.iguana.cc.config.elements.ConnectionConfig; -import org.aksw.iguana.cc.worker.impl.UPDATEWorker; -import org.aksw.iguana.cc.worker.impl.update.UpdateTimer; -import org.aksw.iguana.commons.time.TimeUtils; -import org.apache.commons.io.FileUtils; -import org.junit.*; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.simpleframework.http.core.ContainerServer; -import org.simpleframework.transport.connect.SocketConnection; - -import java.io.File; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.net.SocketAddress; -import java.time.Instant; -import java.util.*; - -import static org.junit.Assert.assertEquals; - - -@RunWith(Parameterized.class) -public class UPDATEWorkerTest { - - private static final int FAST_SERVER_PORT = 8025; - private static WorkerServerMock fastServerContainer; - private static ContainerServer fastServer; - private static SocketConnection fastConnection; - private final String service; - private final String timerStrategy; - private final Map queriesFile; - private final int expectedExec; - private String outputDir; - - public UPDATEWorkerTest(String timerStrategy, Map queriesFile, int expectedExec) { - this.service = "http://localhost:8025/test"; - this.timerStrategy = timerStrategy; - this.queriesFile = queriesFile; - this.expectedExec = expectedExec; - //warmup - Map warmupQueries = new HashMap<>(); - warmupQueries.put("location", "src/test/resources/workers/single-query.txt"); - UPDATEWorker worker = new UPDATEWorker("", 1, getConnection(), warmupQueries, null, null, null, null, null); - worker.executeQuery("INSERT DATA {", "1"); - fastServerContainer.getTimes().clear(); - fastServerContainer.getEncodedAuth().clear(); - } - - @Parameterized.Parameters - public static Collection data() { - Collection testData = new ArrayList<>(); - - Map queries0 = new HashMap<>(); - queries0.put("location", "src/test/resources/workers/updates"); - queries0.put("format", "folder"); - testData.add(new Object[]{"none", queries0, 4}); - - Map queries1 = new HashMap<>(); - queries1.put("location", "src/test/resources/workers/updates"); - queries1.put("format", "folder"); - testData.add(new Object[]{"fixed", queries1, 4}); - - Map queries2 = new HashMap<>(); - queries2.put("location", "src/test/resources/workers/updates"); - queries2.put("format", "folder"); - testData.add(new Object[]{"distributed", queries2, 4}); - - Map queries3 = new HashMap<>(); - queries3.put("location", "src/test/resources/workers/updates.txt"); - testData.add(new Object[]{"none", queries3, 3}); - - Map queries4 = new HashMap<>(); - queries4.put("location", "src/test/resources/workers/updates.txt"); - testData.add(new Object[]{"fixed", queries4, 3}); - - Map queries5 = new HashMap<>(); - queries5.put("location", "src/test/resources/workers/updates.txt"); - testData.add(new Object[]{"distributed", queries5, 3}); - return testData; - } - - @BeforeClass - public static void startServer() throws IOException { - fastServerContainer = new WorkerServerMock(true); - fastServer = new ContainerServer(fastServerContainer); - fastConnection = new SocketConnection(fastServer); - SocketAddress address1 = new InetSocketAddress(FAST_SERVER_PORT); - fastConnection.connect(address1); - - } - - @AfterClass - public static void stopServer() throws IOException { - fastConnection.close(); - fastServer.stop(); - } - - @Before - public void createDir() { - this.outputDir = UUID.randomUUID().toString(); - } - - @After - public void cleanup() throws IOException { - FileUtils.deleteDirectory(new File(this.outputDir)); - fastServerContainer.getTimes().clear(); - fastServerContainer.getEncodedAuth().clear(); - } - - // creds correct - // stop sending after iteration - // correct timer strategy - // correct waiting in sum - @Test - public void testWorkflow() throws InterruptedException { - String taskID = "124/1/1"; - int timeLimit = 2000; - ConnectionConfig con = getConnection(); - UPDATEWorker worker = new UPDATEWorker(taskID, 1, con, this.queriesFile, timeLimit, null, null, null, this.timerStrategy); - worker.run(); - Instant now = worker.startTime; - - Thread.sleep(2000); - assertEquals(this.expectedExec, worker.getExecutedQueries()); - - Set creds = fastServerContainer.getEncodedAuth(); - assertEquals(1, creds.size()); - assertEquals(con.getUser() + ":" + con.getPassword(), creds.iterator().next()); - List requestTimes = fastServerContainer.getTimes(); - long noOfQueries = worker.getNoOfQueries(); - Double fixedValue = timeLimit / noOfQueries * 1.0; - Instant pastInstant = requestTimes.get(0); - - long remainingQueries = noOfQueries - 1; - long remainingTime = timeLimit - Double.valueOf(TimeUtils.durationInMilliseconds(now, pastInstant)).longValue(); - for (int i = 1; i < requestTimes.size(); i++) { - //every exec needs about 200ms - Instant currentInstant = requestTimes.get(i); - double timeInMS = TimeUtils.durationInMilliseconds(pastInstant, currentInstant); - double expected = getQueryWaitTime(this.timerStrategy, fixedValue, remainingQueries, remainingTime); - assertEquals("Run " + i, expected, timeInMS, 200.0); - remainingTime = timeLimit - (100 + Double.valueOf(TimeUtils.durationInMilliseconds(now, currentInstant)).longValue()); - remainingQueries--; - pastInstant = currentInstant; - } - } - - private double getQueryWaitTime(String timerStrategy, Double fixedValue, long remainingQueries, long remainingTime) { - UpdateTimer.Strategy timer = UpdateTimer.Strategy.valueOf(timerStrategy.toUpperCase()); - switch (timer) { - case FIXED: - return fixedValue + 100.0; - case DISTRIBUTED: - return remainingTime * 1.0 / remainingQueries; - case NONE: - return 100.0; - - } - return 0; - } - - - private ConnectionConfig getConnection() { - ConnectionConfig con = new ConnectionConfig(); - con.setName("test"); - con.setEndpoint(this.service); - - con.setUpdateEndpoint(this.service); - con.setUser("testuser"); - con.setPassword("testpwd"); - return con; - } -} diff --git a/src/test/java/org/aksw/iguana/cc/worker/WorkerServerMock.java b/src/test/java/org/aksw/iguana/cc/worker/WorkerServerMock.java deleted file mode 100644 index 948a525cb..000000000 --- a/src/test/java/org/aksw/iguana/cc/worker/WorkerServerMock.java +++ /dev/null @@ -1,145 +0,0 @@ -package org.aksw.iguana.cc.worker; - -import org.aksw.iguana.cc.lang.impl.SPARQLLanguageProcessor; -import org.apache.commons.io.FileUtils; -import org.simpleframework.http.Request; -import org.simpleframework.http.Response; -import org.simpleframework.http.Status; -import org.simpleframework.http.core.Container; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.time.Instant; -import java.util.*; - -/** - * Server Mock - * - * @author f.conrads - * - */ -public class WorkerServerMock implements Container { - - private static final Logger LOGGER = LoggerFactory.getLogger(WorkerServerMock.class); - private final Boolean ignore; - - private List requestTimes = new ArrayList(); - private Set encodedAuth = new HashSet(); - - public WorkerServerMock() { - this(false); - } - - public WorkerServerMock(Boolean ignore){ - super(); - this.ignore =ignore; - } - - @Override - public void handle(Request request, Response resp) { - String content=null; - requestTimes.add(Instant.now()); - if(ignore){ - String authValue = request.getValue("Authorization").replace("Basic ", ""); - this.encodedAuth.add(new String(Base64.getDecoder().decode(authValue))); - waitForMS(95); - try { - content = request.getContent(); - }catch (IOException e){ - LOGGER.error("", e); - } - } - else if(request.getMethod().equals("GET")) { - waitForMS(95); - content=request.getParameter("text"); - } - else if(request.getMethod().equals("POST")){ - waitForMS(195); - try { - String postContent = request.getContent(); - if(postContent.startsWith("{ \"text\":")){ - content=postContent; - } - } catch (IOException e) { - LOGGER.error("", e); - } - } - - if(content!=null){ - handleOK(resp, request.getValue("accept")); - } - else{ - handleFail(resp, request.getValue("accept")); - } - - } - - private void waitForMS(long ms){ - try { - Thread.sleep(ms); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - - public void handleFail(Response resp, String acceptType){ - resp.setCode(Status.BAD_REQUEST.code); - String cType = acceptType; - if(acceptType==null){ - cType = SPARQLLanguageProcessor.QUERY_RESULT_TYPE_JSON; - } - resp.setContentType(cType); - try { - //write answer - resp.getOutputStream().write("".getBytes()); - resp.getOutputStream().close(); - } catch (IOException e) { - LOGGER.error("Could not close Response Output Stream"); - } - } - - public void handleUnAuthorized(Response resp){ - resp.setCode(Status.UNAUTHORIZED.code); - try { - //write answer - resp.getOutputStream().write("".getBytes()); - resp.getOutputStream().close(); - } catch (IOException e) { - LOGGER.error("Could not close Response Output Stream"); - } - } - - public void handleOK(Response resp, String acceptType){ - resp.setCode(Status.OK.code); - String cType = acceptType; - if(acceptType==null){ - cType = SPARQLLanguageProcessor.QUERY_RESULT_TYPE_JSON; - } - resp.setContentType(cType); - - try { - //write answer - String resultStr=""; - if(cType.equals("text/plain")){ - resultStr="a\nb\nc\nd"; - } - else if(cType.equals(SPARQLLanguageProcessor.QUERY_RESULT_TYPE_JSON)) { - resultStr = FileUtils.readFileToString(new File("src/test/resources/sparql-json-response.json"), "UTF-8"); - } - resp.getOutputStream().write(resultStr.getBytes()); - resp.getOutputStream().close(); - } catch (IOException e) { - LOGGER.error("Could not close Response Output Stream"); - } - } - - public List getTimes(){ - return this.requestTimes; - } - - public Set getEncodedAuth() { - return encodedAuth; - } -} diff --git a/src/test/java/org/aksw/iguana/cc/worker/impl/CLIWorkersTests.java b/src/test/java/org/aksw/iguana/cc/worker/impl/CLIWorkersTests.java deleted file mode 100644 index 9aca35f85..000000000 --- a/src/test/java/org/aksw/iguana/cc/worker/impl/CLIWorkersTests.java +++ /dev/null @@ -1,172 +0,0 @@ -package org.aksw.iguana.cc.worker.impl; - -import org.aksw.iguana.cc.config.elements.ConnectionConfig; -import org.aksw.iguana.cc.model.QueryExecutionStats; -import org.aksw.iguana.cc.utils.FileUtils; -import org.aksw.iguana.commons.constants.COMMON; -import org.junit.After; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; -import java.util.*; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -@Ignore("CLI workers don't work right now") -public class CLIWorkersTests { - - private File f; - - @Before - public void createFile() { - String file = UUID.randomUUID().toString(); - this.f = new File(file); - } - - @After - public void deleteFile() { - this.f.delete(); - } - - @Test - public void checkMultipleProcesses() { - ConnectionConfig con = new ConnectionConfig(); - con.setEndpoint("src/test/resources/cli/echoinput.sh " + f.getAbsolutePath()); - MultipleCLIInputWorker worker = new MultipleCLIInputWorker("123/1/1", 1, con, getQueryConfig(), null, null, null, null, "init finished", "rows", "query fail", 2); - assertEquals(2, worker.processList.size()); - for (Process p : worker.processList) { - assertTrue(p.isAlive()); - } - //should run normally - assertEquals(0, worker.currentProcessId); - worker.executeQuery("test", "1"); - assertEquals(0, worker.currentProcessId); - worker.executeQuery("quit", "2"); - worker.executeQuery("test", "1"); - assertEquals(1, worker.currentProcessId); - assertEquals(2, worker.processList.size()); - - for (Process p : worker.processList) { - assertTrue(p.isAlive()); - } - worker.executeQuery("quit", "2"); - worker.executeQuery("test", "1"); - assertEquals(0, worker.currentProcessId); - } - - @Test - public void checkFileInput() throws IOException { - //check if file is created and used - ConnectionConfig con = new ConnectionConfig(); - String dir = UUID.randomUUID().toString(); - con.setEndpoint("src/test/resources/cli/echoinput.sh " + f.getAbsolutePath()); - CLIInputFileWorker worker = new CLIInputFileWorker("123/1/1", 1, con, getQueryConfig(), null, null, null, null, "init finished", "rows", "query fail", 1, dir); - worker.executeQuery("test", "1"); - assertEquals("test", FileUtils.readFile(dir + File.separator + "tmpquery.sparql")); - worker.executeQuery("SELECT whatever", "1"); - assertEquals("SELECT whatever", FileUtils.readFile(dir + File.separator + "tmpquery.sparql")); - assertEquals("tmpquery.sparql\ntmpquery.sparql\n", FileUtils.readFile(f.getAbsolutePath())); - - org.apache.commons.io.FileUtils.deleteDirectory(new File(dir)); - worker.stopSending(); - - } - - @Test - public void checkInput() throws IOException { - // check if connection stays - ConnectionConfig con = new ConnectionConfig(); - - con.setEndpoint("src/test/resources/cli/echoinput.sh " + f.getAbsolutePath()); - CLIInputWorker worker = new CLIInputWorker("123/1/1", 1, con, getQueryConfig(), null, null, null, null, "init finished", "rows", "query fail"); - worker.executeQuery("test", "1"); - worker.executeQuery("SELECT whatever", "1"); - assertEquals("test\nSELECT whatever\n", FileUtils.readFile(f.getAbsolutePath())); - Collection succeededResults = worker.popQueryResults(); - assertEquals(2, succeededResults.size()); - QueryExecutionStats succ = succeededResults.iterator().next(); - assertEquals(COMMON.QUERY_SUCCESS, succ.responseCode()); - assertEquals(3L, succ.responseCode()); - succ = succeededResults.iterator().next(); - assertEquals(COMMON.QUERY_SUCCESS, succ.responseCode()); - assertEquals(3L, succ.responseCode()); - - // check fail - worker.executeQuery("fail", "2"); - assertEquals("test\nSELECT whatever\nfail\n", FileUtils.readFile(f.getAbsolutePath())); - Collection failedResults = worker.popQueryResults(); - assertEquals(1, failedResults.size()); - QueryExecutionStats fail = failedResults.iterator().next(); - assertEquals(COMMON.QUERY_UNKNOWN_EXCEPTION, fail.responseCode()); - assertEquals(0L, fail.resultSize()); - worker.stopSending(); - - - } - - @Test - public void checkPrefix() throws IOException { - // check if connection stays - ConnectionConfig con = new ConnectionConfig(); - - con.setEndpoint("src/test/resources/cli/echoinput.sh " + f.getAbsolutePath()); - CLIInputPrefixWorker worker = new CLIInputPrefixWorker("123/1/1", 1, con, getQueryConfig(), null, null, null, null, "init finished", "rows", "query fail", 1, "prefix", "suffix"); - worker.executeQuery("test", "1"); - worker.executeQuery("SELECT whatever", "1"); - assertEquals("prefix test suffix\nprefix SELECT whatever suffix\n", FileUtils.readFile(f.getAbsolutePath())); - Collection succeededResults = worker.popQueryResults(); - assertEquals(2, succeededResults.size()); - QueryExecutionStats succ = succeededResults.iterator().next(); - assertEquals(COMMON.QUERY_SUCCESS, succ.responseCode()); - assertEquals(3L, succ.resultSize()); - succ = succeededResults.iterator().next(); - assertEquals(COMMON.QUERY_SUCCESS, succ.responseCode()); - assertEquals(3L, succ.resultSize()); - - // check fail - worker.executeQuery("fail", "2"); - assertEquals("prefix test suffix\nprefix SELECT whatever suffix\nprefix fail suffix\n", FileUtils.readFile(f.getAbsolutePath())); - Collection failedResults = worker.popQueryResults(); - assertEquals(1, failedResults.size()); - QueryExecutionStats fail = failedResults.iterator().next(); - assertEquals(COMMON.QUERY_UNKNOWN_EXCEPTION, fail.responseCode()); - assertEquals(0L, fail.resultSize()); - worker.stopSending(); - } - - @Test - public void checkCLI() throws IOException { - //check if simple cli works - // public CLIWorker(String taskID, Connection connection, String queriesFile, @Nullable Integer timeOut, @Nullable Integer timeLimit, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, Integer workerID) { - ConnectionConfig con = new ConnectionConfig(); - con.setUser("user1"); - con.setPassword("pwd"); - - con.setEndpoint("/bin/echo \"$QUERY$ $USER$:$PASSWORD$ $ENCODEDQUERY$\" > " + f.getAbsolutePath()); - CLIWorker worker = new CLIWorker("123/1/1", 1, con, getQueryConfig(), null, null, null, null); - worker.executeQuery("test ()", "1"); - String content = FileUtils.readFile(f.getAbsolutePath()); - assertEquals("test () user1:pwd test+%28%29\n", content); - - con = new ConnectionConfig(); - con.setEndpoint("/bin/echo \"$QUERY$ $USER$:$PASSWORD$ $ENCODEDQUERY$\" > " + f.getAbsolutePath() + " | /bin/printf \"HeaderDoesNotCount\na\na\""); - worker = new CLIWorker("123/1/1", 1, con, getQueryConfig(), null, null, null, null); - worker.executeQuery("test ()", "1"); - content = FileUtils.readFile(f.getAbsolutePath()); - assertEquals("test () : test+%28%29\n", content); - Collection results = worker.popQueryResults(); - assertEquals(1, results.size()); - QueryExecutionStats p = results.iterator().next(); - assertEquals(2L, p.resultSize()); - } - - private Map getQueryConfig() { - Map config = new HashMap<>(); - config.put("location", "src/test/resources/updates/empty.nt"); - return config; - } -} diff --git a/src/test/java/org/aksw/iguana/cc/worker/impl/HttpPostWorkerTest.java b/src/test/java/org/aksw/iguana/cc/worker/impl/HttpPostWorkerTest.java deleted file mode 100644 index 631319a22..000000000 --- a/src/test/java/org/aksw/iguana/cc/worker/impl/HttpPostWorkerTest.java +++ /dev/null @@ -1,51 +0,0 @@ -package org.aksw.iguana.cc.worker.impl; - -import org.aksw.iguana.cc.config.elements.ConnectionConfig; -import org.apache.http.client.methods.HttpPost; -import org.junit.Test; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; -import java.util.stream.Collectors; - -import static org.junit.Assert.assertEquals; - -public class HttpPostWorkerTest { - - private static Map getDefaultQueryConfig() { - Map queries = new HashMap<>(); - queries.put("location", "src/test/resources/workers/single-query.txt"); - return queries; - } - - @Test - public void buildRequest() throws IOException { - String query = "DELETE DATA { \"äöüÄÖÜß\" . }"; - - HttpPostWorker postWorker = new HttpPostWorker(null, 0, getConnection(), getDefaultQueryConfig(), null, null, null, null, null, null, "application/sparql"); - postWorker.buildRequest(query, null); - - HttpPost request = ((HttpPost) postWorker.request); - - assertEquals("Content-Type: text/plain; charset=UTF-8", request.getEntity().getContentType().toString()); - - String content = new BufferedReader(new InputStreamReader(request.getEntity().getContent(), StandardCharsets.UTF_8)).lines().collect(Collectors.joining("\n")); - assertEquals(query, content); - } - - private ConnectionConfig getConnection() { - String service = "http://localhost:3030"; - - ConnectionConfig con = new ConnectionConfig(); - con.setName("test"); - con.setPassword("test"); - con.setUser("abc"); - con.setEndpoint(service); - con.setUpdateEndpoint(service); - return con; - } -} \ No newline at end of file diff --git a/src/test/java/org/aksw/iguana/cc/worker/impl/RequestFactoryTest.java b/src/test/java/org/aksw/iguana/cc/worker/impl/RequestFactoryTest.java new file mode 100644 index 000000000..ef1c09d9f --- /dev/null +++ b/src/test/java/org/aksw/iguana/cc/worker/impl/RequestFactoryTest.java @@ -0,0 +1,110 @@ +package org.aksw.iguana.cc.worker.impl; + +import org.aksw.iguana.cc.mockup.MockupConnection; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URLEncoder; +import java.net.http.HttpResponse; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.concurrent.Flow; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class RequestFactoryTest { + static final class StringSubscriber implements Flow.Subscriber { + final HttpResponse.BodySubscriber wrapped; + StringSubscriber(HttpResponse.BodySubscriber wrapped) { + this.wrapped = wrapped; + } + @Override + public void onSubscribe(Flow.Subscription subscription) { + wrapped.onSubscribe(subscription); + } + @Override + public void onNext(ByteBuffer item) { wrapped.onNext(List.of(item)); } + @Override + public void onError(Throwable throwable) { wrapped.onError(throwable); } + @Override + public void onComplete() { wrapped.onComplete(); } + } + + + @ParameterizedTest + @EnumSource(SPARQLProtocolWorker.RequestFactory.RequestType.class) + public void test(SPARQLProtocolWorker.RequestFactory.RequestType type) throws URISyntaxException, IOException { + final var content = "SELECT * WHERE { ?s ?p ?o }"; + final var connection = MockupConnection.createConnectionConfig("test-conn", "", "http://localhost:8080/sparql"); + final var duration = Duration.of(2, ChronoUnit.SECONDS); + final var stream = new ByteArrayInputStream(content.getBytes()); + final var requestHeader = "application/sparql-results+json"; + + final var requestFactory = new SPARQLProtocolWorker.RequestFactory(type); + final var request = requestFactory.buildHttpRequest( + stream, + duration, + connection, + requestHeader + ); + + switch (type) { + case GET_QUERY -> assertEquals(connection.endpoint() + "?query=" + URLEncoder.encode(content, StandardCharsets.UTF_8), request.uri().toString()); + case POST_QUERY -> { + assertEquals("application/sparql-query", request.headers().firstValue("Content-Type").get()); + assertEquals("http://localhost:8080/sparql", request.uri().toString()); + assertTrue(request.bodyPublisher().isPresent()); + String body = request.bodyPublisher().map(p -> { + var bodySubscriber = HttpResponse.BodySubscribers.ofString(StandardCharsets.UTF_8); + var flowSubscriber = new StringSubscriber(bodySubscriber); + p.subscribe(flowSubscriber); + return bodySubscriber.getBody().toCompletableFuture().join(); + }).get(); + assertEquals(content, body); + } + case POST_UPDATE -> { + assertEquals("application/sparql-update", request.headers().firstValue("Content-Type").get()); + assertEquals("http://localhost:8080/sparql", request.uri().toString()); + assertTrue(request.bodyPublisher().isPresent()); + String body = request.bodyPublisher().map(p -> { + var bodySubscriber = HttpResponse.BodySubscribers.ofString(StandardCharsets.UTF_8); + var flowSubscriber = new StringSubscriber(bodySubscriber); + p.subscribe(flowSubscriber); + return bodySubscriber.getBody().toCompletableFuture().join(); + }).get(); + assertEquals(content, body); + } + case POST_URL_ENC_QUERY -> { + assertEquals("application/x-www-form-urlencoded", request.headers().firstValue("Content-Type").get()); + assertEquals("http://localhost:8080/sparql", request.uri().toString()); + assertTrue(request.bodyPublisher().isPresent()); + String body = request.bodyPublisher().map(p -> { + var bodySubscriber = HttpResponse.BodySubscribers.ofString(StandardCharsets.UTF_8); + var flowSubscriber = new StringSubscriber(bodySubscriber); + p.subscribe(flowSubscriber); + return bodySubscriber.getBody().toCompletableFuture().join(); + }).get(); + assertEquals("query=" + URLEncoder.encode(content, StandardCharsets.UTF_8), body); + } + case POST_URL_ENC_UPDATE -> { + assertEquals("application/x-www-form-urlencoded", request.headers().firstValue("Content-Type").get()); + assertEquals("http://localhost:8080/sparql", request.uri().toString()); + assertTrue(request.bodyPublisher().isPresent()); + String body = request.bodyPublisher().map(p -> { + var bodySubscriber = HttpResponse.BodySubscribers.ofString(StandardCharsets.UTF_8); + var flowSubscriber = new StringSubscriber(bodySubscriber); + p.subscribe(flowSubscriber); + return bodySubscriber.getBody().toCompletableFuture().join(); + }).get(); + assertEquals("update=" + URLEncoder.encode(content, StandardCharsets.UTF_8), body); + } + } + } +} diff --git a/src/test/java/org/aksw/iguana/cc/worker/impl/SPARQLProtocolWorkerTest.java b/src/test/java/org/aksw/iguana/cc/worker/impl/SPARQLProtocolWorkerTest.java new file mode 100644 index 000000000..31641287e --- /dev/null +++ b/src/test/java/org/aksw/iguana/cc/worker/impl/SPARQLProtocolWorkerTest.java @@ -0,0 +1,258 @@ +package org.aksw.iguana.cc.worker.impl; + +import com.github.tomakehurst.wiremock.common.Slf4jNotifier; +import com.github.tomakehurst.wiremock.core.Options; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; +import com.github.tomakehurst.wiremock.http.Fault; +import com.github.tomakehurst.wiremock.junit5.WireMockExtension; +import org.aksw.iguana.cc.config.elements.ConnectionConfig; +import org.aksw.iguana.cc.config.elements.DatasetConfig; +import org.aksw.iguana.cc.query.handler.QueryHandler; +import org.aksw.iguana.cc.worker.HttpWorker; +import org.aksw.iguana.cc.worker.ResponseBodyProcessor; +import org.junit.jupiter.api.*; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static org.junit.jupiter.api.Assertions.*; + +public class SPARQLProtocolWorkerTest { + + @RegisterExtension + public static WireMockExtension wm = WireMockExtension.newInstance() + .options(new WireMockConfiguration().useChunkedTransferEncoding(Options.ChunkedEncodingPolicy.NEVER).dynamicPort().notifier(new Slf4jNotifier(true))) + .failOnUnmatchedRequests(true) + .build(); + + private final static String QUERY = "SELECT * WHERE { ?s ?p ?o }"; + private final static int QUERY_MIXES = 1; + private static Path queryFile; + + @BeforeAll + public static void setup() throws IOException { + queryFile = Files.createTempFile("iguana-test-queries", ".tmp"); + Files.writeString(queryFile, QUERY, StandardCharsets.UTF_8); + } + + @BeforeEach + public void reset() { + wm.resetMappings(); // reset stubbing maps after each test + } + + @AfterAll + public static void cleanup() throws IOException { + Files.deleteIfExists(queryFile); + } + + public static Stream> requestFactoryData() throws IOException, URISyntaxException { + final var uri = new URI("http://localhost:" + wm.getPort() + "/ds/query"); + + final var processor = new ResponseBodyProcessor("application/sparql-results+json"); + final var format = QueryHandler.Config.Format.SEPARATOR; + final var queryHandlder = new QueryHandler(new QueryHandler.Config(queryFile.toAbsolutePath().toString(), format, null, true, QueryHandler.Config.Order.LINEAR, 0L, QueryHandler.Config.Language.SPARQL)); + final var datasetConfig = new DatasetConfig("TestDS", null); + final var connection = new ConnectionConfig("TestConn", "1", datasetConfig, uri, new ConnectionConfig.Authentication("testUser", "password"), null, null); + final var workers = new ArrayDeque>(); + int i = 0; + for (var requestType : SPARQLProtocolWorker.RequestFactory.RequestType.values()) { + final var config = new SPARQLProtocolWorker.Config( + 1, + queryHandlder, + new HttpWorker.QueryMixes(QUERY_MIXES), + connection, + Duration.parse("PT100S"), + "application/sparql-results+json", + requestType, + false + ); + workers.add(Named.of(config.requestType().name(), new SPARQLProtocolWorker(i++, processor, config))); + } + return workers.stream(); + } + + public static List completionTargets() { + final var out = new ArrayList(); + final var queryMixesAmount = List.of(1, 2, 5, 10, 100, 1000); + final var timeDurations = List.of(Duration.of(1, ChronoUnit.SECONDS), Duration.of(5, ChronoUnit.SECONDS)); + + for (var queryMixes : queryMixesAmount) { + out.add(Arguments.of(new HttpWorker.QueryMixes(queryMixes))); + } + + for (var duration : timeDurations) { + out.add(Arguments.of(new HttpWorker.TimeLimit(duration))); + } + + return out; + } + + @ParameterizedTest(name = "[{index}] requestType = {0}") + @MethodSource("requestFactoryData") + @DisplayName("Test Request Factory") + public void testRequestFactory(SPARQLProtocolWorker worker) { + switch (worker.config().requestType()) { + case GET_QUERY -> wm.stubFor(get(urlPathEqualTo("/ds/query")) + .withQueryParam("query", equalTo(QUERY)) + .withBasicAuth("testUser", "password") + .willReturn(aResponse().withStatus(200).withBody("Non-Empty-Body"))); + case POST_QUERY -> { + wm.stubFor(post(urlPathEqualTo("/ds/query")) + .withHeader("Content-Type", equalTo("application/sparql-query")) + .withHeader("Transfer-Encoding", equalTo("chunked")) + .withBasicAuth("testUser", "password") + .withRequestBody(equalTo(QUERY)) + .willReturn(aResponse().withStatus(200).withBody("Non-Empty-Body"))); + return; + } + case POST_UPDATE -> { + wm.stubFor(post(urlPathEqualTo("/ds/query")) + .withHeader("Content-Type", equalTo("application/sparql-update")) + .withHeader("Transfer-Encoding", equalTo("chunked")) + .withBasicAuth("testUser", "password") + .withRequestBody(equalTo(QUERY)) + .willReturn(aResponse().withStatus(200).withBody("Non-Empty-Body"))); + return; // TODO: wiremock behaves really weirdly when the request body is streamed + } + + case POST_URL_ENC_QUERY -> wm.stubFor(post(urlPathEqualTo("/ds/query")) + .withHeader("Content-Type", equalTo("application/x-www-form-urlencoded")) + .withBasicAuth("testUser", "password") + .withRequestBody(equalTo("query=" + URLEncoder.encode(QUERY, StandardCharsets.UTF_8))) + .willReturn(aResponse().withStatus(200).withBody("Non-Empty-Body"))); + case POST_URL_ENC_UPDATE -> wm.stubFor(post(urlPathEqualTo("/ds/query")) + .withHeader("Content-Type", equalTo("application/x-www-form-urlencoded")) + .withBasicAuth("testUser", "password") + .withRequestBody(equalTo("update=" + URLEncoder.encode(QUERY, StandardCharsets.UTF_8))) + .willReturn(aResponse().withStatus(200).withBody("Non-Empty-Body"))); + } + + final HttpWorker.Result result = worker.start().join(); + + assertEquals(result.executionStats().size(), QUERY_MIXES, "Worker should have executed only 1 query"); + assertNull(result.executionStats().get(0).error().orElse(null), "Worker threw an exception, during execution"); + assertEquals(200, result.executionStats().get(0).httpStatusCode().get(), "Worker returned wrong status code"); + assertNotEquals(0, result.executionStats().get(0).responseBodyHash().getAsLong(), "Worker didn't return a response body hash"); + assertEquals("Non-Empty-Body".getBytes(StandardCharsets.UTF_8).length, result.executionStats().get(0).contentLength().getAsLong(), "Worker returned wrong content length"); + assertNotEquals(Duration.ZERO, result.executionStats().get(0).duration(), "Worker returned zero duration"); + } + + @DisplayName("Test Malformed Response Processing") + @ParameterizedTest(name = "[{index}] fault = {0}") + @EnumSource(Fault.class) + public void testMalformedResponseProcessing(Fault fault) throws IOException, URISyntaxException { + SPARQLProtocolWorker worker = (SPARQLProtocolWorker) requestFactoryData().toList().get(0).getPayload(); + wm.stubFor(get(urlPathEqualTo("/ds/query")) + .willReturn(aResponse().withFault(fault))); + final HttpWorker.Result result = worker.start().join(); + assertEquals(1, result.executionStats().size()); + assertNotNull(result.executionStats().get(0).error().orElse(null)); + } + + @Test + public void testBadHttpCodeResponse() throws IOException, URISyntaxException { + SPARQLProtocolWorker worker = (SPARQLProtocolWorker) requestFactoryData().toList().get(0).getPayload(); + wm.stubFor(get(urlPathEqualTo("/ds/query")) + .willReturn(aResponse().withStatus(404))); + final HttpWorker.Result result = worker.start().join(); + assertEquals(1, result.executionStats().size()); + assertTrue(result.executionStats().get(0).httpError()); + } + + @ParameterizedTest + @MethodSource("completionTargets") + public void testCompletionTargets(HttpWorker.CompletionTarget target) throws URISyntaxException, IOException { + final var uri = new URI("http://localhost:" + wm.getPort() + "/ds/query"); + final var processor = new ResponseBodyProcessor("application/sparql-results+json"); + final var queryHandlder = new QueryHandler(new QueryHandler.Config(queryFile.toAbsolutePath().toString(), QueryHandler.Config.Format.SEPARATOR, null, true, QueryHandler.Config.Order.LINEAR, 0L, QueryHandler.Config.Language.SPARQL)); + final var datasetConfig = new DatasetConfig("TestDS", null); + final var connection = new ConnectionConfig("TestConn", "1", datasetConfig, uri, new ConnectionConfig.Authentication("testUser", "password"), null, null); + + final var config = new SPARQLProtocolWorker.Config( + 1, + queryHandlder, + target, + connection, + Duration.parse("PT20S"), + "application/sparql-results+json", + SPARQLProtocolWorker.RequestFactory.RequestType.POST_URL_ENC_QUERY, + false + ); + + SPARQLProtocolWorker worker = new SPARQLProtocolWorker(0, processor, config); + wm.stubFor(post(urlPathEqualTo("/ds/query")) + .withHeader("Content-Type", equalTo("application/x-www-form-urlencoded")) + .withBasicAuth("testUser", "password") + .withRequestBody(equalTo("query=" + URLEncoder.encode(QUERY, StandardCharsets.UTF_8))) + .willReturn(aResponse().withStatus(200).withBody("Non-Empty-Body"))); + + final HttpWorker.Result result = worker.start().join(); + + for (var stat : result.executionStats()) { + assertTrue(stat.successful()); + assertTrue(stat.error().isEmpty()); + assertEquals(200, stat.httpStatusCode().orElseThrow()); + assertTrue(stat.contentLength().orElseThrow() > 0); + assertTrue(stat.duration().compareTo(Duration.ZERO) > 0); + } + + if (target instanceof HttpWorker.TimeLimit) { + Duration totalDuration = result.executionStats().stream() + .map(HttpWorker.ExecutionStats::duration) + .reduce(Duration::plus) + .get(); + + assertTrue(totalDuration.compareTo(((HttpWorker.TimeLimit) target).duration()) <= 0); + } else { + assertEquals(((HttpWorker.QueryMixes) target).number(), result.executionStats().size()); + } + } + + @Test + public void testTimeLimitExecutionCutoff() throws URISyntaxException, IOException { + final var uri = new URI("http://localhost:" + wm.getPort() + "/ds/query"); + + final var processor = new ResponseBodyProcessor("application/sparql-results+json"); + final var queryHandlder = new QueryHandler(new QueryHandler.Config(queryFile.toAbsolutePath().toString(), QueryHandler.Config.Format.SEPARATOR, null, true, QueryHandler.Config.Order.LINEAR, 0L, QueryHandler.Config.Language.SPARQL)); + final var datasetConfig = new DatasetConfig("TestDS", null); + final var connection = new ConnectionConfig("TestConn", "1", datasetConfig, uri, new ConnectionConfig.Authentication("testUser", "password"), null, null); + + final var config = new SPARQLProtocolWorker.Config( + 1, + queryHandlder, + new HttpWorker.TimeLimit(Duration.of(2, ChronoUnit.SECONDS)), + connection, + Duration.parse("PT20S"), + "application/sparql-results+json", + SPARQLProtocolWorker.RequestFactory.RequestType.POST_URL_ENC_QUERY, + false + ); + + SPARQLProtocolWorker worker = new SPARQLProtocolWorker(0, processor, config); + wm.stubFor(post(urlPathEqualTo("/ds/query")) + .withHeader("Content-Type", equalTo("application/x-www-form-urlencoded")) + .withBasicAuth("testUser", "password") + .withRequestBody(equalTo("query=" + URLEncoder.encode(QUERY, StandardCharsets.UTF_8))) + .willReturn(aResponse().withStatus(200).withBody("Non-Empty-Body").withFixedDelay(1000))); + + final HttpWorker.Result result = worker.start().join(); + assertEquals(1, result.executionStats().size()); // because of the delay, only one query should be executed + } +} \ No newline at end of file diff --git a/src/test/java/org/aksw/iguana/commons/factory/AnnotatedFactorizedObject.java b/src/test/java/org/aksw/iguana/commons/factory/AnnotatedFactorizedObject.java deleted file mode 100644 index 5913230c2..000000000 --- a/src/test/java/org/aksw/iguana/commons/factory/AnnotatedFactorizedObject.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.aksw.iguana.commons.factory; - -import org.aksw.iguana.commons.annotation.Nullable; -import org.aksw.iguana.commons.annotation.ParameterNames; -import org.aksw.iguana.commons.annotation.Shorthand; - -@Shorthand(value = "facto") -public class AnnotatedFactorizedObject extends FactorizedObject { - public AnnotatedFactorizedObject(String[] args, String[] args2) { - this.setArgs(args); - this.setArgs2(args2); - } - - @ParameterNames(names={"a","b","c"}) - public AnnotatedFactorizedObject(String a, String b, String c) { - this.setArgs(new String[] {a, b, c}); - } - - @ParameterNames(names={"a","b"}) - public AnnotatedFactorizedObject(String a, @Nullable String b) { - this.setArgs(new String[] {a, b==null?"wasNull":b}); - } - - public AnnotatedFactorizedObject() { - args = new String[] {"a3", "b3"}; - } - -} diff --git a/src/test/java/org/aksw/iguana/commons/factory/FactorizedObject.java b/src/test/java/org/aksw/iguana/commons/factory/FactorizedObject.java deleted file mode 100644 index e6f954a60..000000000 --- a/src/test/java/org/aksw/iguana/commons/factory/FactorizedObject.java +++ /dev/null @@ -1,56 +0,0 @@ -package org.aksw.iguana.commons.factory; - -import org.aksw.iguana.commons.annotation.Nullable; - -public class FactorizedObject { - - protected String[] args; - protected String[] args2; - - public FactorizedObject(String[] args, String[] args2) { - this.setArgs(args); - this.setArgs2(args2); - } - - public FactorizedObject(String a, String b, String c) { - this.setArgs(new String[] {a, b, c}); - } - - public FactorizedObject(String a, @Nullable String b) { - this.setArgs(new String[] {a, b==null?"wasNull":b}); - } - - - public FactorizedObject() { - args = new String[] {"a3", "b3"}; - } - - /** - * @return the args - */ - public String[] getArgs() { - return args; - } - - /** - * @param args the args to set - */ - public void setArgs(String[] args) { - this.args = args; - } - - /** - * @return the args2 - */ - public String[] getArgs2() { - return args2; - } - - /** - * @param args2 the args2 to set - */ - public void setArgs2(String[] args2) { - this.args2 = args2; - } - -} diff --git a/src/test/java/org/aksw/iguana/commons/factory/TypedFactoryTest.java b/src/test/java/org/aksw/iguana/commons/factory/TypedFactoryTest.java deleted file mode 100644 index 3e27832dd..000000000 --- a/src/test/java/org/aksw/iguana/commons/factory/TypedFactoryTest.java +++ /dev/null @@ -1,152 +0,0 @@ -package org.aksw.iguana.commons.factory; - -import org.junit.Test; - -import java.util.HashMap; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -public class TypedFactoryTest { - - @Test - public void argumentClassesTest() { - String[] args = new String[]{"a1", "b1"}; - String[] args2 = new String[]{"a2", "b2"}; - TypedFactory factory = new TypedFactory<>(); - FactorizedObject testObject = factory.create("org.aksw.iguana.commons.factory.FactorizedObject", - new Object[]{args, args2}, new Class[]{String[].class, String[].class}); - assertEquals(args[0], testObject.getArgs()[0]); - assertEquals(args[1], testObject.getArgs()[1]); - assertEquals(args2[0], testObject.getArgs2()[0]); - assertEquals(args2[1], testObject.getArgs2()[1]); - - } - - - @Test - public void noConstructor() { - TypedFactory factory = new TypedFactory<>(); - HashMap map = new HashMap<>(); - map.put("nope", "nope"); - assertNull(factory.create("org.aksw.iguana.commons.factory.FactorizedObject", map)); - assertNull(factory.create("org.aksw.iguana.commons.factory.FactorizedObject", new Object[]{"nope"})); - assertNull(factory.create("org.aksw.iguana.commons.factory.FactorizedObject", new Object[]{"nope"}, new Class[]{String.class})); - assertNull(factory.createAnnotated("org.aksw.iguana.commons.factory.AnnotatedFactorizedObject", map)); - - map.clear(); - map.put("a", 123); - map.put("b", true); - assertNull(factory.create("org.aksw.iguana.commons.factory.FactorizedObject", map)); - assertNull(factory.createAnnotated("org.aksw.iguana.commons.factory.AnnotatedFactorizedObject", map)); - - } - - @Test - public void nullConstructorClass() { - TypedFactory factory = new TypedFactory<>(); - FactorizedObject testObject = factory.create("org.aksw.iguana.commons.factory.FactorizedObject", new Object[]{"a", "b", "c"}, null); - assertEquals("a", testObject.getArgs()[0]); - assertEquals("b", testObject.getArgs()[1]); - assertEquals("c", testObject.getArgs()[2]); - testObject = factory.create("org.aksw.iguana.commons.factory.FactorizedObject", null, null); - assertEquals("a3", testObject.getArgs()[0]); - assertEquals("b3", testObject.getArgs()[1]); - } - - @Test - public void nullClass() { - TypedFactory factory = new TypedFactory<>(); - assertNull(factory.create(null, new HashMap<>())); - assertNull(factory.create(null, new Object[]{})); - assertNull(factory.create(null, new Object[]{}, new Class[]{})); - - } - - @Test - public void classNameNotFoundTest() { - TypedFactory factory = new TypedFactory<>(); - assertNull(factory.create("thisClassShouldNotExist", new HashMap<>())); - assertNull(factory.create("thisClassShouldNotExist", new Object[]{})); - assertNull(factory.create("thisClassShouldNotExist", new Object[]{}, new Class[]{})); - assertNull(factory.createAnnotated("thisClassShouldNotExist", new HashMap<>())); - } - - @Test - public void argumentStringsTest() { - - TypedFactory factory = new TypedFactory<>(); - FactorizedObject testObject = factory.create("org.aksw.iguana.commons.factory.FactorizedObject", (Object[]) null); - assertEquals("a3", testObject.getArgs()[0]); - assertEquals("b3", testObject.getArgs()[1]); - } - - - @Test - public void mapCreationTestParameterNames() { - - TypedFactory factory = new TypedFactory<>(); - Map arguments = new HashMap<>(); - arguments.put("a", "a4"); - arguments.put("b", "b4"); - arguments.put("c", "c4"); - FactorizedObject testObject = factory.createAnnotated("org.aksw.iguana.commons.factory.AnnotatedFactorizedObject", arguments); - assertEquals("a4", testObject.getArgs()[0]); - assertEquals("b4", testObject.getArgs()[1]); - assertEquals("c4", testObject.getArgs()[2]); - arguments.clear(); - arguments.put("a", "a5"); - testObject = factory.createAnnotated("org.aksw.iguana.commons.factory.AnnotatedFactorizedObject", arguments); - assertEquals("a5", testObject.getArgs()[0]); - assertEquals("wasNull", testObject.getArgs()[1]); - } - - @Test - public void testNullable() { - - TypedFactory factory = new TypedFactory<>(); - Map arguments = new HashMap<>(); - arguments.put("a", "a4"); - arguments.put("b", "b4"); - FactorizedObject testObject = factory.create("org.aksw.iguana.commons.factory.FactorizedObject", arguments); - assertEquals("a4", testObject.getArgs()[0]); - assertEquals("b4", testObject.getArgs()[1]); - arguments.remove("b"); - testObject = factory.create("org.aksw.iguana.commons.factory.FactorizedObject", arguments); - assertEquals("a4", testObject.getArgs()[0]); - assertEquals("wasNull", testObject.getArgs()[1]); - - } - - @Test - public void mapCreationTest() { - - TypedFactory factory = new TypedFactory<>(); - Map arguments = new HashMap<>(); - arguments.put("a", "a4"); - arguments.put("b", "b4"); - arguments.put("c", "c4"); - FactorizedObject testObject = factory.create("org.aksw.iguana.commons.factory.FactorizedObject", arguments); - assertEquals("a4", testObject.getArgs()[0]); - assertEquals("b4", testObject.getArgs()[1]); - assertEquals("c4", testObject.getArgs()[2]); - - } - - @Test - public void shortHandAnnotationTest() { - - TypedFactory factory = new TypedFactory<>(); - Map arguments = new HashMap<>(); - arguments.put("a", "a4"); - arguments.put("b", "b4"); - arguments.put("c", "c4"); - AnnotatedFactorizedObject testObject = factory.create("facto", arguments); - assertEquals("a4", testObject.getArgs()[0]); - assertEquals("b4", testObject.getArgs()[1]); - assertEquals("c4", testObject.getArgs()[2]); - - } - -} diff --git a/src/test/java/org/aksw/iguana/commons/io/BigByteArrayInputStreamTest.java b/src/test/java/org/aksw/iguana/commons/io/BigByteArrayInputStreamTest.java new file mode 100644 index 000000000..cb68b1b82 --- /dev/null +++ b/src/test/java/org/aksw/iguana/commons/io/BigByteArrayInputStreamTest.java @@ -0,0 +1,224 @@ +package org.aksw.iguana.commons.io; + +import com.google.common.primitives.Bytes; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.*; + +@Disabled("This test takes a lot of time and resources.") +class BigByteArrayInputStreamTest { + + private static final int MAX_SINGLE_BUFFER_SIZE = Integer.MAX_VALUE - 8; + private static Random rng = new Random(); + + /** + * Creates a random 2d-array buffer with the given size. + * + * @param size number of bytes + * @param maxSingleBufferSize maximum size of a single array + * @return 2d-array buffer + */ + public static byte[][] getBigRandomBuffer(long size, int maxSingleBufferSize) { + if (size < 1) + return new byte[0][0]; + final var bufferField = new byte[(int) ((size - 1) / maxSingleBufferSize) + 1][]; + for (int i = 0; i < bufferField.length; i++) { + final var bufferSize = (size > maxSingleBufferSize) ? maxSingleBufferSize : (int) size; + bufferField[i] = new byte[bufferSize]; + rng.nextBytes(bufferField[i]); + size -= bufferSize; + } + return bufferField; + } + + @Test + @DisplayName("Test illegal arguments") + public void testIllegalArguments() throws IOException { + final var bbaos = new BigByteArrayOutputStream(100); + final var data = 1; + bbaos.write(data); + final var bbais = new BigByteArrayInputStream(bbaos); + + assertThrows(NullPointerException.class, () -> bbais.readNBytes(null, 0, 1)); + assertThrows(IndexOutOfBoundsException.class, () -> bbais.readNBytes(new byte[1], -1, 1)); + assertThrows(IndexOutOfBoundsException.class, () -> bbais.readNBytes(new byte[1], 0, -1)); + assertThrows(IndexOutOfBoundsException.class, () -> bbais.readNBytes(new byte[1], 0, 2)); + assertThrows(IndexOutOfBoundsException.class, () -> bbais.readNBytes(new byte[1], 1, 1)); + assertThrows(IndexOutOfBoundsException.class, () -> bbais.readNBytes(new byte[1], 2, 0)); + assertThrows(NullPointerException.class, () -> bbais.read(null, 0, 1)); + assertThrows(IndexOutOfBoundsException.class, () -> bbais.read(new byte[1], -1, 1)); + assertThrows(IndexOutOfBoundsException.class, () -> bbais.read(new byte[1], 0, -1)); + assertThrows(IndexOutOfBoundsException.class, () -> bbais.read(new byte[1], 0, 2)); + assertThrows(IndexOutOfBoundsException.class, () -> bbais.read(new byte[1], 1, 1)); + assertThrows(IndexOutOfBoundsException.class, () -> bbais.read(new byte[1], 2, 0)); + + assertThrows(NullPointerException.class, () -> new BigByteArrayInputStream((byte[]) null)); + assertThrows(NullPointerException.class, () -> new BigByteArrayInputStream((BigByteArrayOutputStream) null)); + } + + @Test + @DisplayName("Test read method with big data") + public void testBigRead() throws IOException { + final var bbaos = new BigByteArrayOutputStream(); + final var buffer = getBigRandomBuffer(((long) MAX_SINGLE_BUFFER_SIZE) + 1000L, MAX_SINGLE_BUFFER_SIZE - 1); + bbaos.write(buffer); + final var bbais = new BigByteArrayInputStream(bbaos); + + assertArrayEquals(buffer[0], bbais.readNBytes(MAX_SINGLE_BUFFER_SIZE - 1)); + assertArrayEquals(buffer[1], bbais.readNBytes(MAX_SINGLE_BUFFER_SIZE - 1)); + } + + @Test + @DisplayName("Test read method with small data") + public void testSmallRead() throws IOException { + final var bbaos = new BigByteArrayOutputStream(100); + final var data = 1; + bbaos.write(data); + final var bbais = new BigByteArrayInputStream(bbaos); + assertEquals(data, bbais.read()); + assertEquals(-1, bbais.read()); + } + + @Test + @DisplayName("Test allBytes() method throws exception") + public void testReadAllBytesException() throws IOException { + final var bbais = new BigByteArrayInputStream(new byte[]{ 1,2,3,4 }); + assertThrows(IOException.class, () -> bbais.readAllBytes()); + } + + @Test + @DisplayName("Test readNBytes(len) method") + public void testReadMethods1() throws IOException { + final var bbaos = new BigByteArrayOutputStream(); + final var buffer = getBigRandomBuffer(1000, MAX_SINGLE_BUFFER_SIZE); + bbaos.write(buffer); + final var bbais = new BigByteArrayInputStream(bbaos); + + assertArrayEquals(Arrays.copyOfRange(buffer[0], 0, 500), bbais.readNBytes(500)); + assertArrayEquals(Arrays.copyOfRange(buffer[0], 500, 1000), bbais.readNBytes(510)); + assertArrayEquals(new byte[0], bbais.readNBytes(1)); + assertEquals(-1, bbais.read()); + } + + @Test + @DisplayName("Test readNBytes(buffer, off, len) method") + public void testReadMethods2() throws IOException { + final var bbaos = new BigByteArrayOutputStream(); + final var data = getBigRandomBuffer(210, MAX_SINGLE_BUFFER_SIZE); + bbaos.write(data); + final var bbais = new BigByteArrayInputStream(bbaos); + + final var buffer = new byte[100]; + assertEquals(100, bbais.readNBytes(buffer, 0, 100)); + assertArrayEquals(Arrays.copyOfRange(data[0], 0, 100), buffer); + assertEquals(50, bbais.readNBytes(buffer, 0, 50)); + assertEquals(50, bbais.readNBytes(buffer, 50, 50)); + assertArrayEquals(Arrays.copyOfRange(data[0], 100, 200), buffer); + assertEquals(10, bbais.readNBytes(buffer, 0, 100)); + assertArrayEquals(Arrays.copyOfRange(data[0], 200, 210), Arrays.copyOfRange(buffer, 0, 10)); + assertEquals(0, bbais.readNBytes(buffer, 0, 100)); + } + + @Test + @DisplayName("Test read(buffer, off, len) method") + public void testReadMethods3() throws IOException { + final var bbaos = new BigByteArrayOutputStream(); + final var data = getBigRandomBuffer(210, MAX_SINGLE_BUFFER_SIZE); + bbaos.write(data); + final var bbais = new BigByteArrayInputStream(bbaos); + + final var buffer = new byte[100]; + assertEquals(100, bbais.read(buffer, 0, 100)); + assertArrayEquals(Arrays.copyOfRange(data[0], 0, 100), buffer); + assertEquals(50, bbais.read(buffer, 0, 50)); + assertEquals(50, bbais.read(buffer, 50, 50)); + assertArrayEquals(Arrays.copyOfRange(data[0], 100, 200), buffer); + assertEquals(10, bbais.read(buffer, 0, 100)); + assertArrayEquals(Arrays.copyOfRange(data[0], 200, 210), Arrays.copyOfRange(buffer, 0, 10)); + assertEquals(-1, bbais.read(buffer, 0, 100)); + } + + @Test + @DisplayName("Test read(buffer) method") + public void testReadMethods4() throws IOException { + final var bbaos = new BigByteArrayOutputStream(); + final var data = getBigRandomBuffer(110, MAX_SINGLE_BUFFER_SIZE); + bbaos.write(data); + final var bbais = new BigByteArrayInputStream(bbaos); + + assertEquals(0, bbais.read(new byte[0])); + final var buffer = new byte[100]; + assertEquals(100, bbais.read(buffer)); + assertArrayEquals(Arrays.copyOfRange(data[0], 0, 100), buffer); + assertEquals(10, bbais.read(buffer)); + assertArrayEquals(Arrays.copyOfRange(data[0], 100, 110), Arrays.copyOfRange(buffer, 0 , 10)); + assertEquals(-1, bbais.read(buffer)); + } + + @Test + @DisplayName("Test read() method") + public void testReadMethods5() throws IOException { + final var bbaos = new BigByteArrayOutputStream(); + final var data = "test".getBytes(StandardCharsets.UTF_8); + bbaos.write(data); + final var bbais = new BigByteArrayInputStream(bbaos); + + List buffer = new ArrayList<>(); + byte currentByte; + while ((currentByte = (byte) bbais.read()) != -1) { + buffer.add(currentByte); + } + assertEquals("test", new String(Bytes.toArray(buffer), StandardCharsets.UTF_8)); + } + + + @Test + @DisplayName("Test bbaos is closed after reading") + public void testBbaosIsClosed() throws IOException { + final var bbaos = new BigByteArrayOutputStream(); + bbaos.write(new byte[] { 1, 2, 3, 4 }); + final var bbais = new BigByteArrayInputStream(bbaos); + assertEquals(1, bbais.read()); + assertEquals(2, bbais.read()); + assertEquals(3, bbais.read()); + assertEquals(4, bbais.read()); + assertEquals(-1, bbais.read()); + assertThrows(IOException.class, () -> bbaos.write("test".getBytes())); + } + + @Test + @DisplayName("Test skip() method with small data") + public void testSmallSkip() throws IOException { + final var bigBuffer = getBigRandomBuffer(400, MAX_SINGLE_BUFFER_SIZE); + final var bbaos = new BigByteArrayOutputStream(); + bbaos.write(bigBuffer); + final var bbais = new BigByteArrayInputStream(bbaos); + assertEquals(100, bbais.skip(100)); + assertArrayEquals(Arrays.copyOfRange(bigBuffer[0], 100, 200), bbais.readNBytes(100)); + assertEquals(200, bbais.skip(200)); + assertEquals(-1, bbais.read()); + assertEquals(0, bbais.skip(100)); + } + + @Test + @DisplayName("Test skip() method with big data") + public void testBigSkip() throws IOException { + final var bigBuffer = getBigRandomBuffer(((long) MAX_SINGLE_BUFFER_SIZE) * 2L, MAX_SINGLE_BUFFER_SIZE); + final var bbaos = new BigByteArrayOutputStream(); + bbaos.write(bigBuffer); + final var bbais = new BigByteArrayInputStream(bbaos); + assertEquals((MAX_SINGLE_BUFFER_SIZE * 2L) - 4, bbais.skip((MAX_SINGLE_BUFFER_SIZE * 2L) - 4)); + assertArrayEquals(Arrays.copyOfRange(bigBuffer[1], MAX_SINGLE_BUFFER_SIZE - 4, MAX_SINGLE_BUFFER_SIZE - 2), bbais.readNBytes(2)); + assertEquals(2, bbais.skip(200)); + assertEquals(-1, bbais.read()); + } +} \ No newline at end of file diff --git a/src/test/java/org/aksw/iguana/commons/io/BigByteArrayOutputStreamTest.java b/src/test/java/org/aksw/iguana/commons/io/BigByteArrayOutputStreamTest.java new file mode 100644 index 000000000..5b49c0541 --- /dev/null +++ b/src/test/java/org/aksw/iguana/commons/io/BigByteArrayOutputStreamTest.java @@ -0,0 +1,310 @@ +package org.aksw.iguana.commons.io; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Named; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import java.util.function.Supplier; + +import static org.junit.jupiter.api.Assertions.*; + +@Disabled("This test takes a lot of time and resources.") +class BigByteArrayOutputStreamTest { + final static Random rng = new Random(0); + + public static List data() { + final long maxSize = Integer.MAX_VALUE - 8; + + final Supplier sup1 = () -> getBigRandomBuffer(10L, (int) maxSize); + final Supplier sup2 = () -> getBigRandomBuffer(maxSize * 2L, (int) maxSize); + + return List.of( + Arguments.of(Named.of(String.valueOf(10), sup1), 10, new int[] { 10 }), + Arguments.of(Named.of(String.valueOf(10), sup1), maxSize * 2L, new int[] {(int) maxSize, (int) maxSize}), // small data, high initial capacity + Arguments.of(Named.of(String.valueOf(maxSize * 2L), sup2), maxSize * 2L, new int[] {(int) maxSize, (int) maxSize}), + Arguments.of(Named.of(String.valueOf(maxSize * 2L), sup2), 0, new int[] {(int) maxSize, (int) maxSize}) + ); + } + + /** + * Creates a random 2d-array buffer with the given size. + * + * @param size number of bytes + * @param maxSingleBufferSize maximum size of a single array + * @return 2d-array buffer + */ + public static byte[][] getBigRandomBuffer(long size, int maxSingleBufferSize) { + if (size < 1) + return new byte[0][0]; + final var bufferField = new byte[(int) ((size - 1) / maxSingleBufferSize) + 1][]; + for (int i = 0; i < bufferField.length; i++) { + final var bufferSize = (size > maxSingleBufferSize) ? maxSingleBufferSize : (int) size; + bufferField[i] = new byte[bufferSize]; + rng.nextBytes(bufferField[i]); + size -= bufferSize; + } + return bufferField; + } + + @Test + public void testClose() throws IOException { + final var bbaos = new BigByteArrayOutputStream(); + final var testData = "test123".getBytes(StandardCharsets.UTF_8); + bbaos.write(testData); + bbaos.close(); + assertThrows(IOException.class, () -> bbaos.clear()); + assertThrows(IOException.class, () -> bbaos.reset()); + assertThrows(IOException.class, () -> bbaos.write(1)); + assertThrows(IOException.class, () -> bbaos.write((byte) 1)); + assertThrows(IOException.class, () -> bbaos.write(new byte[][] {{1}}) ); + assertThrows(IOException.class, () -> bbaos.write(new byte[] {1}, 0, 1)); + assertThrows(IOException.class, () -> bbaos.write(new byte[] {1})); + assertThrows(IOException.class, () -> bbaos.write((new BigByteArrayOutputStream(10)))); + assertEquals(testData.length, bbaos.size()); + assertArrayEquals(new byte[][] {testData} , bbaos.toByteArray()); + assertEquals(1, bbaos.getBaos().size()); + assertArrayEquals(testData, bbaos.getBaos().get(0).toByteArray()); + } + + @Test + @DisplayName("Test basic write operations") + public void testOtherWriteMethods() throws IOException { + final byte[] buffer = getBigRandomBuffer(10, 10)[0]; + + final var b2 = new byte[] { 0, 1, 2, 3 }; + int i = ByteBuffer.wrap(b2).getInt(); + + try (final var bbaos = new BigByteArrayOutputStream()) { + assertDoesNotThrow(() -> bbaos.write(buffer[0])); + assertEquals(1, bbaos.size()); + assertEquals(buffer[0], bbaos.toByteArray()[0][0]); + + assertDoesNotThrow(() -> bbaos.write(buffer, 1, 9)); + assertEquals(10, bbaos.size()); + assertArrayEquals(buffer, bbaos.toByteArray()[0]); + + final var bbaos2 = new BigByteArrayOutputStream(1); + assertDoesNotThrow(() -> bbaos2.write(bbaos)); + assertEquals(10, bbaos2.size()); + assertArrayEquals(buffer, bbaos2.toByteArray()[0]); + + assertDoesNotThrow(() -> bbaos2.write(i)); + assertEquals(11, bbaos2.size()); + assertEquals(b2[3], bbaos2.toByteArray()[0][10]); // low order byte + } + } + + @Test + @DisplayName("Test illegal capacity arguments") + public void testNegativeCapactiy() { + assertThrows(IllegalArgumentException.class, () -> new BigByteArrayOutputStream(-1)); + assertThrows(IllegalArgumentException.class, () -> new BigByteArrayOutputStream(-1L)); + } + + @Test + @DisplayName("Test illegal write arguments") + public void testIndexOutOfBounds() throws IOException { + try (final var bbaos = new BigByteArrayOutputStream()) { + final byte[] nullBuffer = null; + final var buffer = new byte[10]; + assertThrows(IndexOutOfBoundsException.class, () -> bbaos.write(buffer, -1, 10)); + assertThrows(IndexOutOfBoundsException.class, () -> bbaos.write(buffer, 0, -1)); + assertThrows(IndexOutOfBoundsException.class, () -> bbaos.write(buffer, 0, 11)); + assertThrows(NullPointerException.class, () -> bbaos.write(nullBuffer)); + } + } + + + @Test + @DisplayName("Test default constructor") + void testDefaultConstructor() throws IOException { + try (final var bbaos = new BigByteArrayOutputStream()) { + assertEquals(0, bbaos.size()); + assertEquals(1, bbaos.getBaos().size()); + assertEquals(0, bbaos.getBaos().get(0).size()); + assertDoesNotThrow(() -> bbaos.write("test".getBytes(StandardCharsets.UTF_8))); + assertEquals(1, bbaos.getBaos().size()); + assertEquals(4, bbaos.getBaos().get(0).size()); + assertEquals(4, bbaos.size()); + } + } + + @Test + @DisplayName("Test constructor with capacity argument") + void testConstructorWithInt() throws IOException { + try (final var bbaos = new BigByteArrayOutputStream(100)) { + assertEquals(0, bbaos.size()); + assertEquals(1, bbaos.getBaos().size()); + assertEquals(0, bbaos.getBaos().get(0).size()); + assertEquals(100, bbaos.getBaos().get(0).getBuffer().length); + assertDoesNotThrow(() -> bbaos.write("test".getBytes(StandardCharsets.UTF_8))); + assertEquals(4, bbaos.size()); + assertEquals(1, bbaos.getBaos().size()); + assertEquals(4, bbaos.getBaos().get(0).size()); + assertEquals(100, bbaos.getBaos().get(0).getBuffer().length); + } + } + + @Test + @DisplayName("Test constructor with big capacity argument") + void testConstructorWithBigLong() throws IOException { + try (final var bbaos = new BigByteArrayOutputStream(((long) Integer.MAX_VALUE) + 10)) { + assertEquals(0, bbaos.size()); + assertEquals(2, bbaos.getBaos().size()); + assertEquals(0, bbaos.getBaos().get(0).size()); + assertEquals(0, bbaos.getBaos().get(1).size()); + assertNotEquals(0, bbaos.getBaos().get(0).getBuffer().length); // rough comparison + assertNotEquals(0, bbaos.getBaos().get(1).getBuffer().length); + assertDoesNotThrow(() -> bbaos.write("test".getBytes(StandardCharsets.UTF_8))); + assertEquals(4, bbaos.size()); + assertEquals(2, bbaos.getBaos().size()); + assertEquals(4, bbaos.getBaos().get(0).size()); + assertEquals(0, bbaos.getBaos().get(1).size()); + } + } + + @Test + @DisplayName("Test write method with big byte arrays") + void testBaosOverflow() throws IOException { + final var maxArraySize = Integer.MAX_VALUE - 8; + final var firstBufferSize = maxArraySize - 1; + final var secondBufferSize = 2; + try (final var bbaos = new BigByteArrayOutputStream(maxArraySize)) { + final var firstBuffer = getBigRandomBuffer(firstBufferSize, maxArraySize); + final var secondBuffer = getBigRandomBuffer(secondBufferSize, maxArraySize); + + assertEquals(0, bbaos.size()); + assertEquals(1, bbaos.getBaos().size()); + assertEquals(0, bbaos.getBaos().get(0).size()); + assertEquals(maxArraySize, bbaos.getBaos().get(0).getBuffer().length); + assertDoesNotThrow(() -> bbaos.write(firstBuffer)); + for (int i = 0; i < firstBufferSize; i++) { + assertEquals(firstBuffer[0][i], bbaos.getBaos().get(0).getBuffer()[i]); // save memory during execution of this test with this loop + } + assertEquals(firstBufferSize, bbaos.size()); + assertEquals(1, bbaos.getBaos().size()); + assertEquals(firstBufferSize, bbaos.getBaos().get(0).size()); + assertArrayEquals(firstBuffer, bbaos.toByteArray()); + + // overflow first baos + assertDoesNotThrow(() -> bbaos.write(secondBuffer)); + assertEquals(maxArraySize, bbaos.getBaos().get(1).getBuffer().length); + assertEquals(firstBufferSize + secondBufferSize, bbaos.size()); + assertEquals(2, bbaos.getBaos().size()); + assertEquals(maxArraySize, bbaos.getBaos().get(0).size()); + assertEquals(secondBufferSize - (maxArraySize - firstBufferSize), bbaos.getBaos().get(1).size()); + + // test content of first baos + for (int i = 0; i < firstBufferSize; i++) + assertEquals(firstBuffer[0][i], bbaos.getBaos().get(0).getBuffer()[i]); + for (int i = firstBufferSize; i < maxArraySize; i++) + assertEquals(secondBuffer[0][i - firstBufferSize], bbaos.getBaos().get(0).getBuffer()[i]); + + // test content of second baos + assertArrayEquals(Arrays.copyOfRange(secondBuffer[0], secondBufferSize - (maxArraySize - firstBufferSize), secondBufferSize), bbaos.getBaos().get(1).toByteArray()); + + // reset + bbaos.reset(); + assertEquals(2, bbaos.getBaos().size()); // baos won't be removed with reset + assertEquals(0, bbaos.size()); + assertEquals(0, bbaos.getBaos().get(0).size()); + assertEquals(0, bbaos.getBaos().get(1).size()); + assertEquals(maxArraySize, bbaos.getBaos().get(0).getBuffer().length); + assertEquals(maxArraySize, bbaos.getBaos().get(1).getBuffer().length); + + assertDoesNotThrow(() -> bbaos.write(firstBuffer)); + assertEquals(firstBufferSize, bbaos.size()); + assertEquals(firstBufferSize, bbaos.getBaos().get(0).size()); + for (int i = 0; i < firstBufferSize; i++) { + assertEquals(firstBuffer[0][i], bbaos.getBaos().get(0).getBuffer()[i]); + } + + assertDoesNotThrow(() -> bbaos.write(secondBuffer)); + assertEquals(2, bbaos.getBaos().size()); + assertEquals(maxArraySize, bbaos.getBaos().get(1).getBuffer().length); + assertEquals(firstBufferSize + secondBufferSize, bbaos.size()); + assertEquals(maxArraySize, bbaos.getBaos().get(0).size()); + assertEquals(secondBufferSize - (maxArraySize - firstBufferSize), bbaos.getBaos().get(1).size()); + for (int i = 0; i < firstBufferSize; i++) + assertEquals(firstBuffer[0][i], bbaos.getBaos().get(0).getBuffer()[i]); + for (int i = firstBufferSize; i < maxArraySize; i++) + assertEquals(secondBuffer[0][i - firstBufferSize], bbaos.getBaos().get(0).getBuffer()[i]); + + assertArrayEquals(Arrays.copyOfRange(secondBuffer[0], secondBufferSize - (maxArraySize - firstBufferSize), secondBufferSize), bbaos.getBaos().get(1).toByteArray()); + } + } + + @ParameterizedTest(name = "[{index}] randomBufferSize={0}, initialCapacitiy={1}, baosSizes={2}") + @MethodSource("data") + @DisplayName("Test reset method") + void testReset(Supplier bufferSup, long initialCapacitiy, int[] baosSizes) throws IOException { + final var buffer = bufferSup.get(); + try (final var bbaos = new BigByteArrayOutputStream(initialCapacitiy)) { + bbaos.write(buffer); + assertEquals(baosSizes.length, bbaos.getBaos().size()); // expected amount of baos + for (int i = 0; i < buffer.length; i++) { + assertArrayEquals(buffer[i], bbaos.getBaos().get(i).toByteArray()); // expected content + assertEquals(baosSizes[i], bbaos.getBaos().get(i).getBuffer().length); // expected baos sizes + } + assertEquals(Arrays.stream(buffer).mapToLong(x -> x.length).sum(), bbaos.size()); + + bbaos.reset(); + + assertEquals(0, bbaos.size()); + assertEquals(baosSizes.length, bbaos.getBaos().size()); // same amount of baos + for (int i = 0; i < buffer.length; i++) { + assertEquals(baosSizes[i], bbaos.getBaos().get(i).getBuffer().length); // baos sizes should be same + } + + // after clear, a new write should result same expected content and state + bbaos.write(buffer); + assertEquals(Arrays.stream(buffer).mapToLong(x -> x.length).sum(), bbaos.size()); + for (int i = 0; i < buffer.length; i++) { + assertArrayEquals(buffer[i], bbaos.getBaos().get(i).toByteArray()); // expected content + } + + // check baos sizes again after write + for (int i = 0; i < baosSizes.length; i++) { + assertEquals(baosSizes[i], bbaos.getBaos().get(i).getBuffer().length); + } + } + } + + @ParameterizedTest(name = "[{index}] randomBufferSize={0}, initialCapacitiy={1}, baosSizes={2}") + @MethodSource("data") + @DisplayName("Test clear method") + void testClear(Supplier bufferSup, long initialCapacitiy, int[] baosSizes) throws IOException { + final var buffer = bufferSup.get(); + try (final var bbaos = new BigByteArrayOutputStream(initialCapacitiy)) { + bbaos.write(buffer); + assertEquals(baosSizes.length, bbaos.getBaos().size()); // expected amount of baos + for (int i = 0; i < buffer.length; i++) { + assertArrayEquals(buffer[i], bbaos.getBaos().get(i).toByteArray()); // expected content + assertEquals(baosSizes[i], bbaos.getBaos().get(i).getBuffer().length); // expected baos sizes + } + assertEquals(Arrays.stream(buffer).mapToLong(x -> x.length).sum(), bbaos.size()); + + bbaos.clear(); + assertEquals(0, bbaos.size()); + assertEquals(1, bbaos.getBaos().size()); // deleted all baos except first one + assertEquals(baosSizes[0], bbaos.getBaos().get(0).getBuffer().length); // first baos maintained previous buffer size + + // after clear, a new write should result same expected content + bbaos.write(buffer); + for (int i = 0; i < buffer.length; i++) { + assertArrayEquals(buffer[i], bbaos.getBaos().get(i).toByteArray()); // expected content + } + assertEquals(Arrays.stream(buffer).mapToLong(x -> x.length).sum(), bbaos.size()); + } + } +} \ No newline at end of file diff --git a/src/test/java/org/aksw/iguana/commons/number/NumberUtilsTest.java b/src/test/java/org/aksw/iguana/commons/number/NumberUtilsTest.java deleted file mode 100644 index fa77c09a2..000000000 --- a/src/test/java/org/aksw/iguana/commons/number/NumberUtilsTest.java +++ /dev/null @@ -1,56 +0,0 @@ -package org.aksw.iguana.commons.number; - - -import org.aksw.iguana.commons.numbers.NumberUtils; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -import static org.junit.Assert.assertEquals; - - -@RunWith(Parameterized.class) -public class NumberUtilsTest { - - @Parameterized.Parameters - public static Collection data() { - List testConfigs = new ArrayList(); - //simple method - testConfigs.add(new Object[]{"123", Long.class, 123L}); - testConfigs.add(new Object[]{"123.0", Double.class, 123.0}); - testConfigs.add(new Object[]{"123", Double.class, 123.0}); - testConfigs.add(new Object[]{"123.A", Double.class, null}); - testConfigs.add(new Object[]{"123.A", Long.class, null}); - testConfigs.add(new Object[]{"123.0123", Double.class, 123.0123}); - testConfigs.add(new Object[]{null, Double.class, null}); - testConfigs.add(new Object[]{null, Long.class, null}); - - return testConfigs; - } - - private String number; - private Class clazz; - private Number expected; - - public NumberUtilsTest(String number, Class clazz, Number expected){ - this.number=number; - this.expected = expected; - this.clazz=clazz; - } - - @Test - public void checkForClass(){ - if(clazz == Long.class){ - assertEquals(expected, NumberUtils.getLong(number)); - } - else if(clazz == Double.class) { - assertEquals(expected, NumberUtils.getDouble(number)); - - } - } - -} diff --git a/src/test/java/org/aksw/iguana/commons/script/ScriptExecutorTest.java b/src/test/java/org/aksw/iguana/commons/script/ScriptExecutorTest.java deleted file mode 100644 index 9c6959367..000000000 --- a/src/test/java/org/aksw/iguana/commons/script/ScriptExecutorTest.java +++ /dev/null @@ -1,79 +0,0 @@ -package org.aksw.iguana.commons.script; - -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -@RunWith(Parameterized.class) -public class ScriptExecutorTest { - - private static Logger LOGGER = LoggerFactory.getLogger(ScriptExecutorTest.class); - - private String cmd; - private String[] args; - private int expectedExitCode; - private Method callbackMethod; - private Object[] callbackArgs=new Object[]{}; - - @Parameterized.Parameters - public static Collection data() { - List testConfigs = new ArrayList(); - //simple method - testConfigs.add(new Object[]{"/bin/touch", new String[]{"ShouldNotExistWhatSoEver"}, 0, "removeFile", new Object[]{"ShouldNotExistWhatSoEver"}}); - //testing if additional arguments are checked - testConfigs.add(new Object[]{"/bin/echo test", new String[]{"123", "456"}, 0, "emptyCallback", new Object[]{}}); - //should fail as file not exist - testConfigs.add(new Object[]{"scriptThatShouldNotExist", new String[]{}, -1, "emptyCallback", new Object[]{}}); - //should fail with 1 - - - return testConfigs; - } - - - - public ScriptExecutorTest(String cmd, String[] args, int expectedExitCode, String callbackMethodName, Object[] callbackArgs) throws NoSuchMethodException { - this.cmd=cmd; - this.args=args; - this.expectedExitCode=expectedExitCode; - this.callbackArgs = callbackArgs; - Class[] classes = new Class[callbackArgs.length]; - for(int i=0;i