Added exporter to QuaDer

quads-loader/pom.xml

@@ -25,6 +25,7 @@
             ${project.basedir}/target/site/jacoco/jacoco.xml
         </sonar.coverage.jacoco.xmlReportPaths>
         <sonar.language>java</sonar.language>
+        <prometheus.project.version>1.3.2</prometheus.project.version>
     </properties>
     <dependencies>
         <dependency>
@@ -57,19 +58,6 @@
             <scope>test</scope>
         </dependency>
 
-        <!--   Benchmark dependencies     -->
-        <dependency>
-            <groupId>org.openjdk.jmh</groupId>
-            <artifactId>jmh-core</artifactId>
-            <version>${jmh.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.openjdk.jmh</groupId>
-            <artifactId>jmh-generator-annprocess</artifactId>
-            <version>${jmh.version}</version>
-            <scope>provided</scope>
-        </dependency>
-
         <!--   RDF dependencies     -->
         <dependency>
             <groupId>org.apache.jena</groupId>
@@ -84,6 +72,23 @@
             <artifactId>jacoco-maven-plugin</artifactId>
             <version>${jacoco.version}</version>
         </dependency>
+
+        <!-- Prometheus exporter -->
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>prometheus-metrics-core</artifactId>
+            <version>${prometheus.project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>prometheus-metrics-instrumentation-jvm</artifactId>
+            <version>${prometheus.project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>prometheus-metrics-exporter-httpserver</artifactId>
+            <version>${prometheus.project.version}</version>
+        </dependency>
     </dependencies>
     <build>
         <pluginManagement>

quads-loader/src/main/java/fr/vcity/converg/QuadsLoaderApplication.java

@@ -1,12 +1,39 @@
 package fr.vcity.converg;
 
+import io.prometheus.metrics.core.metrics.Counter;
+import io.prometheus.metrics.instrumentation.jvm.JvmMetrics;
+import io.prometheus.metrics.exporter.httpserver.HTTPServer;
+import io.prometheus.metrics.model.snapshots.Unit;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 
+import java.io.IOException;
+
 @SpringBootApplication
 public class QuadsLoaderApplication {
 
-    public static void main(String[] args) {
+    public static void main(String[] args) throws IOException, InterruptedException {
         SpringApplication.run(QuadsLoaderApplication.class, args);
+
+        JvmMetrics.builder().register();
+
+        HTTPServer server = HTTPServer.builder()
+                .port(9400)
+                .buildAndStart();
+
+        Counter counter =
+                Counter.builder()
+                        .name("uptime_seconds_total")
+                        .help("total number of seconds since this application was started")
+                        .unit(Unit.SECONDS)
+                        .register();
+
+        System.out.println(
+                "HTTPServer listening on port http://localhost:" + server.getPort() + "/metrics");
+
+        while (true) {
+            Thread.sleep(1000);
+            counter.inc();
+        }
     }
 }

quads-loader/src/main/java/fr/vcity/converg/services/QuadImportService.java

@@ -3,6 +3,9 @@
 import fr.vcity.converg.dao.ResourceOrLiteral;
 import fr.vcity.converg.dao.Version;
 import fr.vcity.converg.repository.*;
+import io.prometheus.metrics.core.metrics.Counter;
+import io.prometheus.metrics.core.metrics.Summary;
+import io.prometheus.metrics.model.snapshots.Unit;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.jena.graph.Triple;
@@ -70,6 +73,36 @@ public record QuadValueType(TripleValueType tripleValueType, String namedGraph,
     Set<String> namedGraphs = new HashSet<>();
     List<TripleValueType> tripleValueTypes = new ArrayList<>();
 
+    Counter counterVersionTotal =
+            Counter.builder()
+                    .name("version_count_total")
+                    .help("number of version that have been imported in the database")
+                    .register();
+
+    Summary summaryVersionBatchingDuration =
+            Summary.builder()
+                    .name("file_at_version_batching_duration_seconds")
+                    .help("duration of the file batching in seconds at a certain version")
+                    .unit(Unit.SECONDS)
+                    .labelNames("filename", "version")
+                    .register();
+
+    Summary summaryVersionCatalogDuration =
+            Summary.builder()
+                    .name("file_at_version_catalog_duration_seconds")
+                    .help("duration of the file catalog in seconds at a certain version")
+                    .unit(Unit.SECONDS)
+                    .labelNames("filename", "version")
+                    .register();
+
+    Summary summaryVersionCondensingDuration =
+            Summary.builder()
+                    .name("file_at_version_condensing_duration_seconds")
+                    .help("duration of the file condensing in seconds at a certain version")
+                    .unit(Unit.SECONDS)
+                    .labelNames("filename", "version")
+                    .register();
+
     public QuadImportService(
             IFlatModelQuadRepository flatModelQuadRepository,
             IFlatModelTripleRepository flatModelTripleRepository,
@@ -96,6 +129,9 @@ public QuadImportService(
         this.metadataIsInVersion = rdfResourceRepository.save("https://github.com/VCityTeam/ConVer-G/Version#is-in-version", null);
         this.metadataIsVersionOf = rdfResourceRepository.save("https://github.com/VCityTeam/ConVer-G/Version#is-version-of", null);
         this.defaultGraphURI = rdfResourceRepository.save("https://github.com/VCityTeam/ConVer-G/Named-Graph#default-graph", null);
+
+        // Prometheus metrics
+        counterVersionTotal.inc(this.versionRepository.count());
     }
 
     /**
@@ -107,30 +143,39 @@ public QuadImportService(
     public Integer importModel(MultipartFile file) throws RiotException {
         LocalDateTime startTransactionTime = LocalDateTime.now();
         Version version = versionRepository.save(file.getOriginalFilename(), startTransactionTime);
+        counterVersionTotal.inc();
 
         log.info("Current file: {}", file.getOriginalFilename());
 
-
         try (InputStream inputStream = file.getInputStream()) {
             Long start = System.nanoTime();
 
             Long startBatching = System.nanoTime();
             getQuadsStreamRDF(inputStream, file.getOriginalFilename(), version.getIndexVersion())
                     .finish();
             Long endBatching = System.nanoTime();
+            summaryVersionBatchingDuration
+                    .labelValues(file.getOriginalFilename(), version.getIndexVersion().toString())
+                    .observe(Unit.nanosToSeconds(endBatching - startBatching));
             log.info("[Measure] (Batching): {} ns for file: {};", endBatching - startBatching, file.getOriginalFilename());
 
             log.info("Saving quads to catalog");
             Long startCatalog = System.nanoTime();
             rdfResourceRepository.flatModelQuadsToCatalog();
             Long endCatalog = System.nanoTime();
+            summaryVersionCatalogDuration
+                    .labelValues(file.getOriginalFilename(), version.getIndexVersion().toString())
+                    .observe(Unit.nanosToSeconds(endCatalog - startCatalog));
             log.info("[Measure] (Catalog): {} ns for file: {};", endCatalog - startCatalog, file.getOriginalFilename());
 
             log.info("Condensing quads to catalog");
             Long startCondensing = System.nanoTime();
             rdfVersionedQuadRepository.condenseModel();
             rdfVersionedQuadRepository.updateValidityVersionedQuad();
             Long endCondensing = System.nanoTime();
+            summaryVersionCondensingDuration
+                    .labelValues(file.getOriginalFilename(), version.getIndexVersion().toString())
+                    .observe(Unit.nanosToSeconds(endCondensing - startCondensing));
             log.info("[Measure] (Condensing): {} ns for file: {};", endCondensing - startCondensing, file.getOriginalFilename());
 
             flatModelQuadRepository.deleteAll();