From 6b0a9ee675fa4150edde2a4d99d45b8e45c4288e Mon Sep 17 00:00:00 2001 From: Jey Puget Gil Date: Thu, 7 Nov 2024 10:02:39 +0100 Subject: [PATCH] Added metrics exporter to QuaQue --- .../vcity/converg/QuadsLoaderApplication.java | 2 +- quads-query/pom.xml | 18 ++++++ .../jpugetgil/converg/MetricsSingleton.java | 58 +++++++++++++++++++ .../jpugetgil/converg/QuadsQueryApp.java | 29 +++++++++- .../converg/SPARQLtoSQLTranslator.java | 29 ++++++---- .../converg/VersioningQueryExecution.java | 6 ++ 6 files changed, 129 insertions(+), 13 deletions(-) create mode 100644 quads-query/src/main/java/fr/cnrs/liris/jpugetgil/converg/MetricsSingleton.java diff --git a/quads-loader/src/main/java/fr/vcity/converg/QuadsLoaderApplication.java b/quads-loader/src/main/java/fr/vcity/converg/QuadsLoaderApplication.java index 0d3029e..c9c359a 100644 --- a/quads-loader/src/main/java/fr/vcity/converg/QuadsLoaderApplication.java +++ b/quads-loader/src/main/java/fr/vcity/converg/QuadsLoaderApplication.java @@ -29,7 +29,7 @@ public static void main(String[] args) throws IOException, InterruptedException .register(); System.out.println( - "HTTPServer listening on port http://localhost:" + server.getPort() + "/metrics"); + "HTTPServer listening on port:" + server.getPort() + "/metrics"); while (true) { Thread.sleep(1000); diff --git a/quads-query/pom.xml b/quads-query/pom.xml index 6294b47..e024a3e 100644 --- a/quads-query/pom.xml +++ b/quads-query/pom.xml @@ -15,6 +15,7 @@ UTF-8 21 21 + 1.3.2 @@ -88,6 +89,23 @@ guava 33.1.0-jre + + + + io.prometheus + prometheus-metrics-core + ${prometheus.project.version} + + + io.prometheus + prometheus-metrics-instrumentation-jvm + ${prometheus.project.version} + + + io.prometheus + prometheus-metrics-exporter-httpserver + ${prometheus.project.version} + diff --git a/quads-query/src/main/java/fr/cnrs/liris/jpugetgil/converg/MetricsSingleton.java b/quads-query/src/main/java/fr/cnrs/liris/jpugetgil/converg/MetricsSingleton.java new file mode 100644 index 0000000..7cdc0f6 --- /dev/null +++ b/quads-query/src/main/java/fr/cnrs/liris/jpugetgil/converg/MetricsSingleton.java @@ -0,0 +1,58 @@ +package fr.cnrs.liris.jpugetgil.converg; + +import io.prometheus.metrics.core.metrics.Counter; +import io.prometheus.metrics.core.metrics.Summary; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Objects; + +public class MetricsSingleton { + + private static final Logger log = LoggerFactory.getLogger(MetricsSingleton.class); + + private static MetricsSingleton metricsSingleton; + + public Counter selectQueryCounter; + + public Summary queryTranslationDuration; + + public Summary queryExecutionDuration; + + /** + * Constructor method in order to create db connection & statement + */ + private MetricsSingleton() { + selectQueryCounter = Counter.builder() + .name("select_query_counter") + .help("Number of SELECT queries asked to be translated") + .register(); + + queryTranslationDuration = Summary.builder() + .name("query_translation_duration") + .labelNames("query") + .help("Duration of the query translation") + .register(); + + queryExecutionDuration = Summary.builder() + .name("query_execution_duration") + .labelNames("query") + .help("Duration of the query execution") + .register(); + } + + /** + * Create the instance of {@link MetricsSingleton} object if it is not created yet and guarantee that there is only one single instance is created for this class. + * + * @return MetricsSingleton created single instance + */ + public static MetricsSingleton getInstance() { + log.info("MetricsSingleton.getInstance()"); + + if (Objects.isNull(metricsSingleton)) { + metricsSingleton = new MetricsSingleton(); + } + + return metricsSingleton; + } +} \ No newline at end of file diff --git a/quads-query/src/main/java/fr/cnrs/liris/jpugetgil/converg/QuadsQueryApp.java b/quads-query/src/main/java/fr/cnrs/liris/jpugetgil/converg/QuadsQueryApp.java index bd33138..9b1622e 100644 --- a/quads-query/src/main/java/fr/cnrs/liris/jpugetgil/converg/QuadsQueryApp.java +++ b/quads-query/src/main/java/fr/cnrs/liris/jpugetgil/converg/QuadsQueryApp.java @@ -1,5 +1,9 @@ package fr.cnrs.liris.jpugetgil.converg; +import io.prometheus.metrics.core.metrics.Counter; +import io.prometheus.metrics.exporter.httpserver.HTTPServer; +import io.prometheus.metrics.instrumentation.jvm.JvmMetrics; +import io.prometheus.metrics.model.snapshots.Unit; import org.apache.jena.fuseki.main.FusekiServer; import org.apache.jena.fuseki.main.sys.FusekiModules; import org.apache.jena.fuseki.server.Operation; @@ -9,13 +13,15 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; + /** * Query main class */ public class QuadsQueryApp { private static final Logger log = LoggerFactory.getLogger(QuadsQueryApp.class); - public static void main(String[] args) { + public static void main(String[] args) throws InterruptedException, IOException { log.info("Building Fuseki server..."); DatasetGraph dsg = DatasetFactory.createGeneral().asDatasetGraph(); Dataset ds = DatasetFactory.wrap(dsg); @@ -29,5 +35,26 @@ public static void main(String[] args) { .build(); log.info("Fuseki server is built, starting..."); server.start(); + + JvmMetrics.builder().register(); + + HTTPServer exporterServer = HTTPServer.builder() + .port(9400) + .buildAndStart(); + + Counter counter = + Counter.builder() + .name("uptime_seconds_total") + .help("total number of seconds since this application was started") + .unit(Unit.SECONDS) + .register(); + + System.out.println( + "HTTPServer listening on port: " + exporterServer.getPort() + "/metrics"); + + while (true) { + Thread.sleep(1000); + counter.inc(); + } } } diff --git a/quads-query/src/main/java/fr/cnrs/liris/jpugetgil/converg/SPARQLtoSQLTranslator.java b/quads-query/src/main/java/fr/cnrs/liris/jpugetgil/converg/SPARQLtoSQLTranslator.java index a0a9a2f..d0cc598 100644 --- a/quads-query/src/main/java/fr/cnrs/liris/jpugetgil/converg/SPARQLtoSQLTranslator.java +++ b/quads-query/src/main/java/fr/cnrs/liris/jpugetgil/converg/SPARQLtoSQLTranslator.java @@ -7,28 +7,24 @@ import fr.cnrs.liris.jpugetgil.converg.sql.SQLContext; import fr.cnrs.liris.jpugetgil.converg.sql.SQLQuery; import fr.cnrs.liris.jpugetgil.converg.sql.operator.*; -import org.apache.jena.datatypes.xsd.XSDDatatype; +import io.prometheus.metrics.core.metrics.Counter; +import io.prometheus.metrics.core.metrics.Summary; +import io.prometheus.metrics.model.snapshots.Unit; import org.apache.jena.graph.Node; -import org.apache.jena.graph.NodeFactory; import org.apache.jena.graph.Triple; import org.apache.jena.query.Dataset; import org.apache.jena.query.Query; import org.apache.jena.query.ResultSet; -import org.apache.jena.sparql.ARQException; import org.apache.jena.sparql.ARQNotImplemented; import org.apache.jena.sparql.algebra.Algebra; import org.apache.jena.sparql.algebra.Op; import org.apache.jena.sparql.algebra.op.*; import org.apache.jena.sparql.core.Var; import org.apache.jena.sparql.engine.ResultSetStream; -import org.apache.jena.sparql.engine.binding.Binding; -import org.apache.jena.sparql.engine.binding.BindingBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.sql.ResultSetMetaData; import java.sql.SQLException; -import java.sql.Types; import java.util.*; /** @@ -40,6 +36,12 @@ public class SPARQLtoSQLTranslator extends SPARQLLanguageTranslator { private final JdbcConnection jdbcConnection; + private final Summary queryTranslationDuration = MetricsSingleton.getInstance().queryTranslationDuration; + + private final Summary queryExecutionDuration = MetricsSingleton.getInstance().queryExecutionDuration; + + private final Counter selectQueryCounter = MetricsSingleton.getInstance().selectQueryCounter; + /** * Constructor of the SPARQLtoSQLTranslator * @@ -58,19 +60,24 @@ public SPARQLtoSQLTranslator(boolean condensedMode) { public ResultSet translateAndExecSelect(Query query) { Op op = Algebra.compile(query); - Long start = System.nanoTime(); + Long startTranslation = System.nanoTime(); SQLQuery qu = buildSPARQLContext(op) .finalizeQuery(); + Long endTranslation = System.nanoTime(); - Long end = System.nanoTime(); - - log.info("[Measure] (Query translation duration): {} ns for query: {};", end - start, query); + queryTranslationDuration + .labelValues(String.valueOf(selectQueryCounter.get())) + .observe(Unit.nanosToSeconds(endTranslation - startTranslation)); + log.info("[Measure] (Query translation duration): {} ns for query: {};", endTranslation - startTranslation, query); log.info("Query result: {};", qu.getSql()); try { Long startExec = System.nanoTime(); java.sql.ResultSet rs = jdbcConnection.executeSQL(qu.getSql()); Long endExec = System.nanoTime(); + queryExecutionDuration + .labelValues(String.valueOf(selectQueryCounter.get())) + .observe(Unit.nanosToSeconds(endExec - startExec)); log.info("[Measure] (Query execution duration): {} ns for query: {};", endExec - startExec, query); BindingIterator bindingIterator = new BindingIterator(rs); diff --git a/quads-query/src/main/java/fr/cnrs/liris/jpugetgil/converg/VersioningQueryExecution.java b/quads-query/src/main/java/fr/cnrs/liris/jpugetgil/converg/VersioningQueryExecution.java index 30c9051..eedae21 100644 --- a/quads-query/src/main/java/fr/cnrs/liris/jpugetgil/converg/VersioningQueryExecution.java +++ b/quads-query/src/main/java/fr/cnrs/liris/jpugetgil/converg/VersioningQueryExecution.java @@ -1,5 +1,6 @@ package fr.cnrs.liris.jpugetgil.converg; +import io.prometheus.metrics.core.metrics.Counter; import org.apache.jena.atlas.json.JsonArray; import org.apache.jena.atlas.json.JsonObject; import org.apache.jena.graph.Triple; @@ -23,6 +24,8 @@ public class VersioningQueryExecution implements QueryExecution { private final SPARQLLanguageTranslator translator; + private final Counter selectQueryCounter = MetricsSingleton.getInstance().selectQueryCounter; + private static final String TARGET_LANG = System.getenv("TARGET_LANG") == null ? "SQL" : getSupportedTargetLanguage(System.getenv("TARGET_LANG")); @@ -71,6 +74,9 @@ public String getQueryString() { @Override public org.apache.jena.query.ResultSet execSelect() { + // Increment the counter for the number of SELECT queries + selectQueryCounter.inc(); + return translator.translateAndExecSelect(query); }