bug fixes and stability

README.md

@@ -6,21 +6,21 @@ In our server, we add additional bash command to run it continuously, but we did
 To run this prototype, the prerequisite is that you have a JDK 8+ and Maven installed in your computer.
 Additionally, the config.properties is build for local Jena fuseki installation. Make sure that: 
 * the `config.properties` is available (and adjust it if necessary; especially with regards to the triplestore/fuseki installation)
-* you have an empty repo called 'sepses' in your fuseki installation
+* you have an empty repo called 'sepses' in your fuseki/virtuoso installation
 
 The following steps are required to run the engine: 
 * run `mvn clean` to build the required jar files from the `lib` folder
 * run `mvn install -DskipTests=true` to build the application
     * optinally, you can also run the tests (without the `-DskipTests=true`) to check the extracted data against a set of SHACL constraints to make sure that the conversion for each source is correctly defined
     * however, in several cases it yields outOfMemory error, so I marked it as optional.
-* run `java -jar target/cyber-kb-1.0.2-jar-with-dependencies.jar`
+* run `java -jar target/cyber-kb-1.0.3-jar-with-dependencies.jar`
 
 By default, the application will run conversion for for all registered resources (CAPEC, CWE, CPE, CVE+CVSS)
 
 The prototype will then 
 * (i) generate the RDF graph from these sources and create necessary linking
 * (ii) store the data in the triplestore
 
-We have tried and tested it in OSX (Intel i7@3,1GHz, OSX Mojave, 16GB RAM) and Ubuntu (Intel Xeon E5-2620@2GHz, Ubuntu 18.04, 6GB RAM) machines
+We have tried and tested it in OSX (Intel i7@3,1GHz, OSX Mojave, 16GB RAM)
 
 @SEPSES team

config.properties

@@ -2,9 +2,11 @@
 InputDir=input
 OutputDir=output
 #================SPARQL Endpoint=========================
-SparqlEndpoint=http://localhost:3030/sepses
-Triplestore=fuseki
-UseAuth=false
+#SparqlEndpoint=http://localhost:3030/sepses
+#Triplestore=fuseki
+SparqlEndpoint=http://localhost:8890/sparql
+Triplestore=virtuoso
+UseAuth=true
 EndpointUser=dba
 EndpointPass=dba
 #================CAPEC Config============================

pom.xml

@@ -4,7 +4,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>ac.at.tuwien.ifs.sepses</groupId>
     <artifactId>cyber-kg-converter</artifactId>
-    <version>1.0.2</version>
+    <version>1.0.3</version>
     <build>
         <plugins>
             <!-- generate tbox assessor -->

src/main/java/ac/at/tuwien/ifs/sepses/helper/Statistics.java

@@ -0,0 +1,20 @@
+package ac.at.tuwien.ifs.sepses.helper;
+
+public class Statistics {
+
+    public static void generateCVEStatistics() {
+        // instances in the new data
+        // - new instances
+        // - modified instances
+        // - deleted instances
+
+        // other statistics
+        // - avg number of links to CWE
+        // - avg number of links to CPE
+        // - avg number of links to LogicalTest
+    }
+
+    public static void runQuery(String endpoint, String namegraph, String query) {
+
+    }
+}

src/main/java/ac/at/tuwien/ifs/sepses/helper/Utility.java

@@ -2,6 +2,7 @@
 
 import ac.at.tuwien.ifs.sepses.storage.Storage;
 import ac.at.tuwien.ifs.sepses.storage.impl.FusekiStorage;
+import ac.at.tuwien.ifs.sepses.storage.impl.VirtuosoStorage;
 import ac.at.tuwien.ifs.sepses.vocab.*;
 import org.apache.jena.query.*;
 import org.apache.jena.rdf.model.*;
@@ -33,8 +34,8 @@ public class Utility {
      * @return true if existing catalog id the same with the new catalog id
      * @throws IOException
      */
-    public static boolean checkIsUpToDate(Model metaModel, String endpoint, String graph,
-            Resource catalogResource) throws IOException {
+    public static boolean checkIsUpToDate(Model metaModel, String endpoint, String graph, Resource catalogResource)
+            throws IOException {
 
         ParameterizedSparqlString queryString1 =
                 new ParameterizedSparqlString("select ?s from ?graph where { ?s a ?catalog }");
@@ -184,6 +185,9 @@ public static Storage getStorage(Properties properties) {
         if (triplestore.equalsIgnoreCase("fuseki")) {
             log.info("Fuseki triplestore is selected");
             return FusekiStorage.getInstance();
+        } else if (triplestore.equalsIgnoreCase("virtuoso")) {
+            log.info("Virtuoso triplestore is selected");
+            return VirtuosoStorage.getInstance();
         } else {
             log.error("Triplestore type is not supported !!!");
             return null;

src/main/java/ac/at/tuwien/ifs/sepses/parser/impl/CAPECParser.java

@@ -1,11 +1,11 @@
 package ac.at.tuwien.ifs.sepses.parser.impl;
 
 import ac.at.tuwien.ifs.sepses.helper.DownloadUnzip;
+import ac.at.tuwien.ifs.sepses.helper.Utility;
+import ac.at.tuwien.ifs.sepses.helper.XMLParser;
 import ac.at.tuwien.ifs.sepses.parser.Parser;
 import ac.at.tuwien.ifs.sepses.parser.tool.Linker;
-import ac.at.tuwien.ifs.sepses.helper.XMLParser;
 import ac.at.tuwien.ifs.sepses.storage.Storage;
-import ac.at.tuwien.ifs.sepses.helper.Utility;
 import ac.at.tuwien.ifs.sepses.vocab.CAPEC;
 import org.apache.jena.rdf.model.Model;
 import org.apache.jena.rdf.model.ModelFactory;
@@ -113,9 +113,8 @@ public static void main(String[] args) throws Exception {
 
         // Step 4 - Checking whether CAPEC is up-to-date ...
         log.info("Checking updates from " + sparqlEndpoint + " using graphname " + namegraph);
-        Boolean cat = Utility
-                .checkIsUpToDate(XMLParser.Parse(capecXML, rmlMetaModel), sparqlEndpoint, namegraph,
-                        CAPEC.ATTACK_PATTERN_CATALOG);
+        Boolean cat = Utility.checkIsUpToDate(XMLParser.Parse(capecXML, rmlMetaModel), sparqlEndpoint, namegraph,
+                CAPEC.ATTACK_PATTERN_CATALOG);
         if (cat) {
             log.info("CAPEC is up-to-date...! ");
             model = ModelFactory.createDefaultModel();

src/main/java/ac/at/tuwien/ifs/sepses/parser/impl/CPEParser.java

@@ -1,18 +1,20 @@
 package ac.at.tuwien.ifs.sepses.parser.impl;
 
 import ac.at.tuwien.ifs.sepses.helper.DownloadUnzip;
+import ac.at.tuwien.ifs.sepses.helper.Utility;
+import ac.at.tuwien.ifs.sepses.helper.XMLParser;
 import ac.at.tuwien.ifs.sepses.parser.Parser;
 import ac.at.tuwien.ifs.sepses.parser.tool.CPETool;
-import ac.at.tuwien.ifs.sepses.helper.XMLParser;
 import ac.at.tuwien.ifs.sepses.storage.Storage;
-import ac.at.tuwien.ifs.sepses.helper.Utility;
 import ac.at.tuwien.ifs.sepses.vocab.CPE;
+import org.apache.commons.io.IOUtils;
 import org.apache.jena.rdf.model.Model;
 import org.apache.jena.rdf.model.ModelFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.*;
+import java.nio.charset.Charset;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Properties;
@@ -77,6 +79,22 @@ public static void main(String[] args) throws Exception {
             model.close();
         }
 
+        // remove CPE-is-not-available-comment
+        addEmptyCPEComments();
+
+    }
+
+    private void addEmptyCPEComments() {
+        try {
+            log.info("add CPE-is-not-available-comment process");
+            InputStream is =
+                    CVEParser.class.getClassLoader().getResourceAsStream("sparql/add-cpe-comments.sparql");
+            String query = IOUtils.toString(is, Charset.forName("UTF-8"));
+            storage.executeUpdate(sparqlEndpoint, query, isUseAuth, user, pass);
+        } catch (IOException e) {
+            log.error("failed add CPE-is-not-available-comment to graph: " + namegraph, e);
+        }
+        log.info("add CPE-is-not-available-comment process finished");
     }
 
     @Override public Model getModelFromLastUpdate() throws IOException {
@@ -140,8 +158,8 @@ public static void main(String[] args) throws Exception {
         Integer cpeCount = Utility.countInstance(sparqlEndpoint, namegraph, CPE.CPE);
         log.info("existing cpe count = " + cpeCount);
 
-        boolean sameVersion = Utility
-                .checkIsEqualModifedDate(rmlMetaModel, CPEXML, sparqlEndpoint, namegraph, CPE.GENERATOR_TIME_STAMP);
+        boolean sameVersion = Utility.checkIsEqualModifedDate(rmlMetaModel, CPEXML, sparqlEndpoint, namegraph,
+                CPE.GENERATOR_TIME_STAMP);
         if (sameVersion) {
             log.info("CPE is up-to-date!!");
             model = ModelFactory.createDefaultModel();

src/main/java/ac/at/tuwien/ifs/sepses/parser/impl/CVEParser.java

@@ -8,13 +8,16 @@
 import ac.at.tuwien.ifs.sepses.parser.tool.Linker;
 import ac.at.tuwien.ifs.sepses.storage.Storage;
 import ac.at.tuwien.ifs.sepses.vocab.CVE;
+import org.apache.commons.io.IOUtils;
 import org.apache.jena.rdf.model.Model;
 import org.apache.jena.rdf.model.ModelFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -98,6 +101,7 @@ public static void main(String[] args) throws Exception {
                 storeFileInRepo(filename);
             }
             model.close();
+
         }
     }
 
@@ -114,7 +118,7 @@ public static void main(String[] args) throws Exception {
     @Override public void storeFileInRepo(String filename) {
 
         log.info("delete previous CVE metadata in the triple store");
-        CVETool.deleteCVEMeta(sparqlEndpoint, namegraph);
+        CVETool.deleteCVEMeta(storage, sparqlEndpoint, namegraph, isUseAuth, user, pass);
 
         log.info("Store data to " + sparqlEndpoint + " using graph " + namegraph);
         storage.storeData(filename, sparqlEndpoint, namegraph, isUseAuth, user, pass);
@@ -136,7 +140,9 @@ public Boolean parseTempCVE(String CVEXMLFile, String RMLFileTemp, String CyberK
             fileName = CVEXMLFile.substring(CVEXMLFile.lastIndexOf("\\") + 1);
         }
         Model CVEModelTemp = XMLParser.Parse(CVEXMLFile, RMLFileTemp);
-        ArrayList<String>[] CVEArray = CVETool.checkExistingCVE(CVEModelTemp, CyberKnowledgeEp, CVEGraphName);
+        ArrayList<String>[] CVEArray =
+                CVETool.checkExistingCVE(storage, CVEModelTemp, CyberKnowledgeEp, CVEGraphName, isUseAuth, user,
+                        pass);
         log.info("Done!");
         log.info("Found New CVE: " + CVEArray[0].size());
         log.info("Found modified CVE : " + CVEArray[1].size());

src/main/java/ac/at/tuwien/ifs/sepses/parser/impl/CWEParser.java

@@ -1,11 +1,11 @@
 package ac.at.tuwien.ifs.sepses.parser.impl;
 
 import ac.at.tuwien.ifs.sepses.helper.DownloadUnzip;
+import ac.at.tuwien.ifs.sepses.helper.Utility;
+import ac.at.tuwien.ifs.sepses.helper.XMLParser;
 import ac.at.tuwien.ifs.sepses.parser.Parser;
 import ac.at.tuwien.ifs.sepses.parser.tool.Linker;
-import ac.at.tuwien.ifs.sepses.helper.XMLParser;
 import ac.at.tuwien.ifs.sepses.storage.Storage;
-import ac.at.tuwien.ifs.sepses.helper.Utility;
 import ac.at.tuwien.ifs.sepses.vocab.CWE;
 import org.apache.jena.rdf.model.Model;
 import org.apache.jena.rdf.model.ModelFactory;

src/main/java/ac/at/tuwien/ifs/sepses/parser/tool/CVETool.java

@@ -1,13 +1,10 @@
 package ac.at.tuwien.ifs.sepses.parser.tool;
 
+import ac.at.tuwien.ifs.sepses.storage.Storage;
 import ac.at.tuwien.ifs.sepses.vocab.CVE;
 import org.apache.commons.io.IOUtils;
 import org.apache.jena.query.*;
 import org.apache.jena.rdf.model.*;
-import org.apache.jena.update.UpdateExecutionFactory;
-import org.apache.jena.update.UpdateFactory;
-import org.apache.jena.update.UpdateProcessor;
-import org.apache.jena.update.UpdateRequest;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -19,8 +16,8 @@ public class CVETool {
 
     private static final Logger log = LoggerFactory.getLogger(CVETool.class);
 
-    public static ArrayList<String>[] checkExistingCVE(Model CVEModelTemp, String sparqlEndpoint, String CVEGraphName)
-            throws IOException {
+    public static ArrayList<String>[] checkExistingCVE(Storage storage, Model CVEModelTemp, String sparqlEndpoint,
+            String CVEGraphName, Boolean isUseAuth, String user, String pass) throws IOException {
 
         //select all CVE on CVEModelTemp
         String queryTemp =
@@ -51,7 +48,7 @@ public static ArrayList<String>[] checkExistingCVE(Model CVEModelTemp, String sp
                 if (co2.equals("0^^http://www.w3.org/2001/XMLSchema#integer")) {
                     //need updates
                     CVEUpdate.add(cveId.toString());
-                    deleteCVE(sparqlEndpoint, cveRes.asResource(), CVEGraphName);
+                    deleteCVE(storage, sparqlEndpoint, cveRes.asResource(), CVEGraphName, isUseAuth, user, pass);
                 } else {
                     //leave it
                     CVELeave.add(cveId.toString());
@@ -108,15 +105,8 @@ private static String checkCVENeedUpdate(String CyberKnowledgeEp, String Id, Str
         return c;
     }
 
-    /**
-     * Delete older CVEs and replace them with the new version from incoming data
-     *
-     * @param endpoint    SPARQL endpoint
-     * @param cveInstance CVE instance
-     * @param graphName   named Graph
-     * @throws IOException
-     */
-    private static void deleteCVE(String endpoint, Resource cveInstance, String graphName) throws IOException {
+    private static void deleteCVE(Storage storage, String endpoint, Resource cveInstance, String graphName,
+            Boolean isUseAuth, String user, String pass) throws IOException {
         InputStream is = CVETool.class.getClassLoader().getResourceAsStream("sparql/deleteCVE.sparql");
         String query = IOUtils.toString(is, Charset.defaultCharset());
 
@@ -125,9 +115,7 @@ private static void deleteCVE(String endpoint, Resource cveInstance, String grap
         deleteQuery.setParam("cve", cveInstance);
         deleteQuery.setNsPrefix("cve", CVE.NS);
 
-        UpdateRequest updateRequest = UpdateFactory.create(deleteQuery.toString());
-        UpdateProcessor updateProcessor = UpdateExecutionFactory.createRemote(updateRequest, endpoint);
-        updateProcessor.execute();
+        storage.executeUpdate(endpoint, deleteQuery.toString(), isUseAuth, user, pass);
 
     }
 
@@ -151,19 +139,23 @@ public static String readMetaSHA(String CVEMeta) {
 
     public static Model generateCVEMetaTriple(String metaSHA, int year) {
         Model CVEMetaModel = ModelFactory.createDefaultModel();
-        Property metaSHA256 = CVEMetaModel.createProperty("http://w3id.org/sepses/vocab/ref/cve#metaSHA256");
-        Resource CVEMeta1 = CVEMetaModel.createResource("http://w3id.org/sepses/resource/cve/meta/cveMeta" + year);
+        Property metaSHA256 = CVE.META_SHA_256;
+        Resource CVEMeta1 = CVEMetaModel.createResource(CVE.NS_INSTANCE + "meta/cveMeta" + year);
         CVEMetaModel.add(CVEMeta1, metaSHA256, metaSHA);
         return CVEMetaModel;
 
     }
 
-    public static void deleteCVEMeta(String CyberKnowledgeEp, String Namegraph) {
-        String p = "<http://w3id.org/sepses/vocab/ref/cve#metaSHA256>";
-        String Query = "with  <" + Namegraph + "> DELETE { ?s ?p ?o }  WHERE { ?s ?p ?o.  filter (?p = " + p + ")}";
-        UpdateRequest QCVE1 = UpdateFactory.create(Query);
-        UpdateProcessor qeQCVE1 = UpdateExecutionFactory.createRemote(QCVE1, CyberKnowledgeEp);
-        qeQCVE1.execute();
+    public static void deleteCVEMeta(Storage storage, String endpoint, String namegraph, boolean isUseAuth,
+            String user, String pass) {
+        Resource graphResource = ResourceFactory.createResource(namegraph);
+        ParameterizedSparqlString query =
+                new ParameterizedSparqlString("WITH ?g DELETE { ?s ?p ?o } WHERE { ?s ?p ?o }");
+        query.setParam("p", CVE.META_SHA_256);
+        query.setParam("g", graphResource);
+        log.info(query.toString());
+
+        storage.executeUpdate(endpoint, query.toString(), isUseAuth, user, pass);
     }
 
 }

src/main/java/ac/at/tuwien/ifs/sepses/storage/Storage.java

@@ -1,7 +1,5 @@
 package ac.at.tuwien.ifs.sepses.storage;
 
-import org.apache.jena.rdf.model.Model;
-
 public interface Storage {
 
     /**
@@ -17,8 +15,8 @@ public void replaceData(String file, String endpoint, String namegraph, Boolean
             String pass);
 
     /**
-     * delete data from a named graph
+     * execute update query in selected triplestore
      */
-    public void deleteData(String endpoint, String namegraph, Boolean isUseAuth, String user, String pass);
+    public void executeUpdate(String endpoint, String query, Boolean isUseAuth, String user, String pass);
 
 }