Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feature/enable disable qts #30

Merged
merged 3 commits into from
Jun 15, 2018
Merged

feature/enable disable qts #30

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e5359c1
add constant for the new config parameter for enabling/disabling QTs
vpapako Jun 15, 2018
d635c51
read the new config parameter for enabling/disabling QTs
vpapako Jun 15, 2018
d88a2bd
filter generated tasks based on the given param by the system
vpapako Jun 15, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions src/main/java/org/hobbit/benchmark/versioning/components/VersioningBenchmarkController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@ public class VersioningBenchmarkController extends AbstractBenchmarkController {

private static final Logger LOGGER = LoggerFactory.getLogger(VersioningBenchmarkController.class);

private static final String DATA_GENERATOR_CONTAINER_IMAGE = "git.project-hobbit.eu:4567/papv/versioningdatagenerator:2.1.0";
private static final String TASK_GENERATOR_CONTAINER_IMAGE = "git.project-hobbit.eu:4567/papv/versioningtaskgenerator:2.1.0";
private static final String EVALUATION_MODULE_CONTAINER_IMAGE = "git.project-hobbit.eu:4567/papv/versioningevaluationmodule:2.1.0";
private static final String DATA_GENERATOR_CONTAINER_IMAGE = "git.project-hobbit.eu:4567/papv/versioningdatagenerator:dev";
private static final String TASK_GENERATOR_CONTAINER_IMAGE = "git.project-hobbit.eu:4567/papv/versioningtaskgenerator:dev";
private static final String EVALUATION_MODULE_CONTAINER_IMAGE = "git.project-hobbit.eu:4567/papv/versioningevaluationmodule:dev";

private static final String PREFIX = "http://w3id.org/hobbit/versioning-benchmark/vocab#";

Expand Down Expand Up @@ -62,6 +62,7 @@ public void init() throws Exception {
int insRatio = (Integer) getPropertyOrDefault(PREFIX + "versionInsertionRatio", 5);
int delRatio = (Integer) getPropertyOrDefault(PREFIX + "versionDeletionRatio", 3);
String dataForm = (String) getPropertyOrDefault(PREFIX + "generatedDataForm", "ic");
String enabledQueries = (String) getPropertyOrDefault(PREFIX + "enableDisableQT", "QT1=1;QT2=1;QT3=1;QT4=1;QT5=1;QT6=1;QT7=1;QT8=1");

loadingTimes = new long[numOfVersions];
triplesToBeAdded = new AtomicIntegerArray(numOfVersions);
Expand All @@ -76,8 +77,10 @@ public void init() throws Exception {
VersioningConstants.NUMBER_OF_VERSIONS + "=" + numOfVersions,
VersioningConstants.VERSION_INSERTION_RATIO + "=" + insRatio,
VersioningConstants.VERSION_DELETION_RATIO + "=" + delRatio,
VersioningConstants.SENT_DATA_FORM + "=" + dataForm
VersioningConstants.SENT_DATA_FORM + "=" + dataForm,
VersioningConstants.ENABLED_QUERY_TYPES + "=" + enabledQueries
};
LOGGER.info(enabledQueries);

// evaluation module environmental values
evalModuleEnvVariables = new String[] {
Expand Down
196 changes: 113 additions & 83 deletions src/main/java/org/hobbit/benchmark/versioning/components/VersioningDataGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.StringReader;
import java.lang.reflect.Constructor;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
Expand All @@ -18,6 +19,7 @@
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Semaphore;
Expand All @@ -27,10 +29,8 @@
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.RegexFileFilter;
import org.apache.commons.lang3.SerializationUtils;
import org.apache.jena.query.Query;
import org.apache.jena.query.QueryExecution;
import org.apache.jena.query.QueryExecutionFactory;
import org.apache.jena.query.QueryFactory;
import org.apache.jena.query.ResultSet;
import org.apache.jena.query.ResultSetFactory;
import org.apache.jena.query.ResultSetFormatter;
Expand Down Expand Up @@ -83,6 +83,7 @@ public class VersioningDataGenerator extends AbstractDataGenerator {
private int versionDeletionRatio;
private String serializationFormat;
private String sentDataForm;
private String enabledQueryTypesParam;
private String generatedDatasetPath = "/versioning/data";
private String initialVersionDataPath = generatedDatasetPath + File.separator + "v0";
private String ontologiesPath = "/versioning/ontologies";
Expand All @@ -94,6 +95,8 @@ public class VersioningDataGenerator extends AbstractDataGenerator {
private int[] triplesExpectedToBeLoaded;
private int[] cwsToBeLoaded;

private Properties enabledQueryTypes = new Properties();

private AtomicInteger numberOfmessages = new AtomicInteger(0);

private Configuration configuration = new Configuration();
Expand Down Expand Up @@ -147,6 +150,9 @@ public void init() throws Exception {
triplesExpectedToBeLoaded = new int[numberOfVersions];
cwsToBeLoaded = new int[numberOfVersions];

// load the enabled queries
enabledQueryTypes.load(new StringReader(enabledQueryTypesParam.replaceAll(";", "\n")));

// Given the above input, update configuration files that are necessary for data generation
reInitializeSPBProperties();

Expand Down Expand Up @@ -320,17 +326,18 @@ public void run() {
}

public void initFromEnv() {
LOGGER.info("Getting Data Generator's properites from the environment...");
LOGGER.info("Getting data generator configuration parameters...");

Map<String, String> env = System.getenv();
// Assume that in v0Size the 40362 triples of DBpedia initial dataset
// plus the 8135 ontologies triples are included
v0SizeInTriples = (Integer) getFromEnv(env, VersioningConstants.V0_SIZE_IN_TRIPLES, 0) - 48497 ;
numberOfVersions = (Integer) getFromEnv(env, VersioningConstants.NUMBER_OF_VERSIONS, 0);
subGeneratorSeed = (Integer) getFromEnv(env, VersioningConstants.DATA_GENERATOR_SEED, 0) + getGeneratorId();
versionInsertionRatio = (Integer) getFromEnv(env, VersioningConstants.VERSION_INSERTION_RATIO, 5);
versionDeletionRatio = (Integer) getFromEnv(env, VersioningConstants.VERSION_DELETION_RATIO, 3);
versionInsertionRatio = (Integer) getFromEnv(env, VersioningConstants.VERSION_INSERTION_RATIO, 0);
versionDeletionRatio = (Integer) getFromEnv(env, VersioningConstants.VERSION_DELETION_RATIO, 0);
sentDataForm = (String) getFromEnv(env, VersioningConstants.SENT_DATA_FORM, "");
enabledQueryTypesParam = (String) getFromEnv(env, VersioningConstants.ENABLED_QUERY_TYPES, "");
}

/*
Expand Down Expand Up @@ -588,27 +595,34 @@ public void buildSPRQLQueries() {
String queryString;

// QT1
queryString = compileMustacheTemplate(1, queryIndex++, 0);
tasks.add(new Task(1, 1, Integer.toString(taskId++), queryString, null));
try {
FileUtils.writeStringToFile(new File(queriesPath + "versioningQuery1.1.1.sparql"), queryString);
} catch (IOException e) {
LOGGER.error("Exception caught during saving of generated task : ", e);
if(enabledQueryTypes.getProperty("QT1").equals("1")) {
queryString = compileMustacheTemplate(1, queryIndex, 0);
tasks.add(new Task(1, 1, Integer.toString(taskId++), queryString, null));
try {
FileUtils.writeStringToFile(new File(queriesPath + "versioningQuery1.1.1.sparql"), queryString);
} catch (IOException e) {
LOGGER.error("Exception caught during saving of generated task : ", e);
}
}
queryIndex++;

// QT2
querySubstParamCount = 5;
for (int querySubType = 1; querySubType <= Statistics.VERSIONING_SUB_QUERIES_COUNT; querySubType++) {
for(int querySubstParam = 1; querySubstParam <= querySubstParamCount; querySubstParam++) {
queryString = compileMustacheTemplate(2, queryIndex, querySubstParam);
tasks.add(new Task(2, querySubType, Integer.toString(taskId++), queryString, null));
try {
FileUtils.writeStringToFile(new File(queriesPath + "versioningQuery2." + querySubType + "." + querySubstParam + ".sparql"), queryString);
} catch (IOException e) {
LOGGER.error("Exception caught during saving of generated task : ", e);
if(enabledQueryTypes.getProperty("QT2").equals("1")) {
querySubstParamCount = 5;
for (int querySubType = 1; querySubType <= Statistics.VERSIONING_SUB_QUERIES_COUNT; querySubType++) {
for(int querySubstParam = 1; querySubstParam <= querySubstParamCount; querySubstParam++) {
queryString = compileMustacheTemplate(2, queryIndex, querySubstParam);
tasks.add(new Task(2, querySubType, Integer.toString(taskId++), queryString, null));
try {
FileUtils.writeStringToFile(new File(queriesPath + "versioningQuery2." + querySubType + "." + querySubstParam + ".sparql"), queryString);
} catch (IOException e) {
LOGGER.error("Exception caught during saving of generated task : ", e);
}
}
queryIndex++;
}
queryIndex++;
} else {
queryIndex += Statistics.VERSIONING_SUB_QUERIES_COUNT;
}

// if there is only one version return
Expand All @@ -617,33 +631,39 @@ public void buildSPRQLQueries() {

// QT3
// if the total number of versions is lower than 1 there are no historical versions
querySubstParamCount = 3;
for(int querySubstParam = 1; querySubstParam <= querySubstParamCount && querySubstParam < numberOfVersions ; querySubstParam++) {
queryString = compileMustacheTemplate(3, queryIndex, querySubstParam);
tasks.add(new Task(3, 1, Integer.toString(taskId++), queryString, null));
try {
FileUtils.writeStringToFile(new File(queriesPath + "versioningQuery3.1." + querySubstParam + ".sparql"), queryString);
} catch (IOException e) {
LOGGER.error("Exception caught during saving of generated task : ", e);
if(enabledQueryTypes.getProperty("QT3").equals("1")) {
querySubstParamCount = 3;
for(int querySubstParam = 1; querySubstParam <= querySubstParamCount && querySubstParam < numberOfVersions ; querySubstParam++) {
queryString = compileMustacheTemplate(3, queryIndex, querySubstParam);
tasks.add(new Task(3, 1, Integer.toString(taskId++), queryString, null));
try {
FileUtils.writeStringToFile(new File(queriesPath + "versioningQuery3.1." + querySubstParam + ".sparql"), queryString);
} catch (IOException e) {
LOGGER.error("Exception caught during saving of generated task : ", e);
}
}
}
}
queryIndex++;

// QT4
// if the total number of versions is lower than 1 there are no historical versions
querySubstParamCount = 3;
for (int querySubType = 1; querySubType <= Statistics.VERSIONING_SUB_QUERIES_COUNT; querySubType++) {
// if there are less than four versions take all the historical ones
for(int querySubstParam = 1; querySubstParam <= querySubstParamCount && querySubstParam < numberOfVersions; querySubstParam++) {
queryString = compileMustacheTemplate(4, queryIndex, querySubstParam);
tasks.add(new Task(4, querySubType, Integer.toString(taskId++), queryString, null));
try {
FileUtils.writeStringToFile(new File(queriesPath + "versioningQuery4." + querySubType + "." + querySubstParam + ".sparql"), queryString);
} catch (IOException e) {
LOGGER.error("Exception caught during saving of generated task : ", e);
if(enabledQueryTypes.getProperty("QT4").equals("1")) {
querySubstParamCount = 3;
for (int querySubType = 1; querySubType <= Statistics.VERSIONING_SUB_QUERIES_COUNT; querySubType++) {
// if there are less than four versions take all the historical ones
for(int querySubstParam = 1; querySubstParam <= querySubstParamCount && querySubstParam < numberOfVersions; querySubstParam++) {
queryString = compileMustacheTemplate(4, queryIndex, querySubstParam);
tasks.add(new Task(4, querySubType, Integer.toString(taskId++), queryString, null));
try {
FileUtils.writeStringToFile(new File(queriesPath + "versioningQuery4." + querySubType + "." + querySubstParam + ".sparql"), queryString);
} catch (IOException e) {
LOGGER.error("Exception caught during saving of generated task : ", e);
}
}
queryIndex++;
}
queryIndex++;
} else {
queryIndex += Statistics.VERSIONING_SUB_QUERIES_COUNT;
}

// if the total number of versions is lower than 5 read deltas for all the historical ones
Expand All @@ -654,65 +674,75 @@ public void buildSPRQLQueries() {
querySubstParamCount = 4;
}
// QT5
for(int querySubstParam = 1; querySubstParam <= querySubstParamCount; querySubstParam++) {
queryString = compileMustacheTemplate(5, queryIndex, querySubstParam);
tasks.add(new Task(5, 1, Integer.toString(taskId++), queryString, null));
try {
FileUtils.writeStringToFile(new File(queriesPath + "versioningQuery5.1." + querySubstParam + ".sparql"), queryString);
} catch (IOException e) {
LOGGER.error("Exception caught during saving of generated task : ", e);
if(enabledQueryTypes.getProperty("QT5").equals("1")) {
for(int querySubstParam = 1; querySubstParam <= querySubstParamCount; querySubstParam++) {
queryString = compileMustacheTemplate(5, queryIndex, querySubstParam);
tasks.add(new Task(5, 1, Integer.toString(taskId++), queryString, null));
try {
FileUtils.writeStringToFile(new File(queriesPath + "versioningQuery5.1." + querySubstParam + ".sparql"), queryString);
} catch (IOException e) {
LOGGER.error("Exception caught during saving of generated task : ", e);
}
}
}
queryIndex++;

// QT6
// same querySubstParamCount as QT5
for(int querySubstParam = 1; querySubstParam <= querySubstParamCount; querySubstParam++) {
queryString = compileMustacheTemplate(6, queryIndex, querySubstParam);
tasks.add(new Task(6, 1, Integer.toString(taskId++), queryString, null));
try {
FileUtils.writeStringToFile(new File(queriesPath + "versioningQuery6.1." + querySubstParam + ".sparql"), queryString);
} catch (IOException e) {
LOGGER.error("Exception caught during saving of generated task : ", e);
if(enabledQueryTypes.getProperty("QT6").equals("1")) {
for(int querySubstParam = 1; querySubstParam <= querySubstParamCount; querySubstParam++) {
queryString = compileMustacheTemplate(6, queryIndex, querySubstParam);
tasks.add(new Task(6, 1, Integer.toString(taskId++), queryString, null));
try {
FileUtils.writeStringToFile(new File(queriesPath + "versioningQuery6.1." + querySubstParam + ".sparql"), queryString);
} catch (IOException e) {
LOGGER.error("Exception caught during saving of generated task : ", e);
}
}
}
queryIndex++;

// QT7
// can not be supported when we have 2 or less total versions, as there cannot exist cross-deltas
querySubstParamCount = 3;
for(int querySubstParam = 1; querySubstParam <= querySubstParamCount && querySubstParam < numberOfVersions - 1; querySubstParam++) {
queryString = compileMustacheTemplate(7, queryIndex, querySubstParam);
tasks.add(new Task(7, 1, Integer.toString(taskId++), queryString, null));
try {
FileUtils.writeStringToFile(new File(queriesPath + "versioningQuery7.1." + querySubstParam + ".sparql"), queryString);
} catch (IOException e) {
LOGGER.error("Exception caught during saving of generated task : ", e);
}
}
queryIndex++;

// QT8
if (numberOfVersions == 2) {
querySubstParamCount = 1;
} else if(numberOfVersions == 3) {
if(enabledQueryTypes.getProperty("QT7").equals("1")) {
querySubstParamCount = 3;
} else if(numberOfVersions == 4) {
querySubstParamCount = 5;
} else {
querySubstParamCount = 6;
}
for (int querySubType = 1; querySubType <= Statistics.VERSIONING_SUB_QUERIES_COUNT; querySubType++) {
for(int querySubstParam = 1; querySubstParam <= querySubstParamCount; querySubstParam++) {
queryString = compileMustacheTemplate(8, queryIndex, querySubstParam);
tasks.add(new Task(8, querySubType, Integer.toString(taskId++), queryString, null));
for(int querySubstParam = 1; querySubstParam <= querySubstParamCount && querySubstParam < numberOfVersions - 1; querySubstParam++) {
queryString = compileMustacheTemplate(7, queryIndex, querySubstParam);
tasks.add(new Task(7, 1, Integer.toString(taskId++), queryString, null));
try {
FileUtils.writeStringToFile(new File(queriesPath + "versioningQuery8." + querySubType + "." + querySubstParam + ".sparql"), queryString);
FileUtils.writeStringToFile(new File(queriesPath + "versioningQuery7.1." + querySubstParam + ".sparql"), queryString);
} catch (IOException e) {
LOGGER.error("Exception caught during saving of generated task : ", e);
}
}
queryIndex++;
}
queryIndex++;

// QT8
if(enabledQueryTypes.getProperty("QT8").equals("1")) {
if (numberOfVersions == 2) {
querySubstParamCount = 1;
} else if(numberOfVersions == 3) {
querySubstParamCount = 3;
} else if(numberOfVersions == 4) {
querySubstParamCount = 5;
} else {
querySubstParamCount = 6;
}
for (int querySubType = 1; querySubType <= Statistics.VERSIONING_SUB_QUERIES_COUNT; querySubType++) {
for(int querySubstParam = 1; querySubstParam <= querySubstParamCount; querySubstParam++) {
queryString = compileMustacheTemplate(8, queryIndex, querySubstParam);
tasks.add(new Task(8, querySubType, Integer.toString(taskId++), queryString, null));
try {
FileUtils.writeStringToFile(new File(queriesPath + "versioningQuery8." + querySubType + "." + querySubstParam + ".sparql"), queryString);
} catch (IOException e) {
LOGGER.error("Exception caught during saving of generated task : ", e);
}
}
queryIndex++;
}
} else {
queryIndex += Statistics.VERSIONING_SUB_QUERIES_COUNT;
}
}

Expand Down
4 changes: 3 additions & 1 deletion src/main/java/org/hobbit/benchmark/versioning/properties/VersioningConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,9 @@ public final class VersioningConstants {
public static final String NUMBER_OF_VERSIONS = "number_of_versions";

public static final String SENT_DATA_FORM = "sent_data_form";


public static final String ENABLED_QUERY_TYPES = "enabled_query_types";

// =============== TASK GENERATOR CONSTANTS ===============

public static final String NUMBER_OF_TASK_GENERATORS= "task_generators_number";
Expand Down