Skip to content

Commit

Permalink
Merge pull request #12 from hobbit-project/send-to-ftp_v2
Browse files Browse the repository at this point in the history
Send to FTP
  • Loading branch information
vpapako authored Feb 9, 2018
2 parents fa01a41 + 009587d commit 40657fc
Show file tree
Hide file tree
Showing 3 changed files with 130 additions and 47 deletions.
9 changes: 0 additions & 9 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -67,22 +67,13 @@
<groupId>eu.ldbc</groupId>
<artifactId>semanticpublishing</artifactId>
<version>2.0</version>
<scope>system</scope>
<systemPath>${project.basedir}/src/main/resources/semanticpublishing-2.0.jar</systemPath>
</dependency>
<!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-core -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>2.8.8</version>
</dependency>
<dependency>
<groupId>org.virtuoso</groupId>
<artifactId>virtjdbc</artifactId>
<version>4.2</version>
<scope>system</scope>
<systemPath>${project.basedir}/src/main/resources/virtjdbc-4.2.jar</systemPath>
</dependency>
<dependency>
<groupId>org.aksw.jena-sparql-api</groupId>
<artifactId>jena-sparql-api-core</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import org.hobbit.benchmark.versioning.Task;
import org.hobbit.benchmark.versioning.properties.RDFUtils;
import org.hobbit.benchmark.versioning.properties.VersioningConstants;
import org.hobbit.benchmark.versioning.util.FTPUtils;
import org.hobbit.benchmark.versioning.util.VirtuosoSystemAdapterConstants;
import org.hobbit.core.components.AbstractDataGenerator;
import org.hobbit.core.rabbit.RabbitMQUtils;
Expand Down Expand Up @@ -601,44 +602,6 @@ public void computeExpectedAnswers() {
}
}

public void writeResults() {
String resultsPath = System.getProperty("user.dir") + File.separator + "results";
File resultsDir = new File(resultsPath);
resultsDir.mkdirs();
// skip current version materialization query
int taskId = 1;

// mind the non zero-based numbering of query types
for (int queryType = 0; queryType < Statistics.VERSIONING_QUERIES_COUNT; queryType++) {
if (Arrays.asList(1,3,7).contains(queryType)) {
for (int querySubType = 0; querySubType < Statistics.VERSIONING_SUB_QUERIES_COUNT; querySubType++) {
for (int querySubstParam = 0; querySubstParam < subsParametersAmount; querySubstParam++) {
ByteBuffer expectedResultsBuffer = ByteBuffer.wrap(tasks.get(taskId++).getExpectedAnswers());
RabbitMQUtils.readString(expectedResultsBuffer);
byte[] expectedResults = RabbitMQUtils.readString(expectedResultsBuffer).getBytes(StandardCharsets.UTF_8);
try {
FileUtils.writeByteArrayToFile(new File(resultsDir + File.separator + "versionigQuery" + (queryType + 1) + "." + (querySubType + 1) + "." + (querySubstParam + 1) + "_results.json"), expectedResults);
} catch (IOException e) {
LOGGER.error("Exception caught during saving of expected results: ", e);
}
}
}
continue;
}
for (int querySubstParam = 0; querySubstParam < subsParametersAmount; querySubstParam++) {
ByteBuffer expectedResultsBuffer = ByteBuffer.wrap(tasks.get(taskId++).getExpectedAnswers());
RabbitMQUtils.readString(expectedResultsBuffer);
byte[] expectedResults = RabbitMQUtils.readString(expectedResultsBuffer).getBytes(StandardCharsets.UTF_8);
try {
FileUtils.writeByteArrayToFile(new File(resultsDir + File.separator + "versionigQuery" + (queryType + 1) + ".1." + (querySubstParam + 1) + "_results.json"), expectedResults);
} catch (IOException e) {
LOGGER.error("Exception caught during saving of expected results : ", e);
}
if (queryType == 0) break;
}
}
}

public void buildSPRQLTasks() {
int taskId = 0;
String queryString;
Expand Down Expand Up @@ -1072,6 +1035,64 @@ public void receiveCommand(byte command, byte[] data) {
super.receiveCommand(command, data);
}

public void writeResults() {
String resultsPath = System.getProperty("user.dir") + File.separator + "results";
File resultsDir = new File(resultsPath);
resultsDir.mkdirs();
// skip current version materialization query
int taskId = 0;

// mind the non zero-based numbering of query types
for (int queryType = 0; queryType < Statistics.VERSIONING_QUERIES_COUNT; queryType++) {
if (Arrays.asList(1,3,7).contains(queryType)) {
for (int querySubType = 0; querySubType < Statistics.VERSIONING_SUB_QUERIES_COUNT; querySubType++) {
for (int querySubstParam = 0; querySubstParam < subsParametersAmount; querySubstParam++) {
ByteBuffer expectedResultsBuffer = ByteBuffer.wrap(tasks.get(taskId++).getExpectedAnswers());
expectedResultsBuffer.getInt();
expectedResultsBuffer.getInt();
byte expectedDataBytes[] = RabbitMQUtils.readByteArray(expectedResultsBuffer);
try {
FileUtils.writeByteArrayToFile(new File(resultsDir + File.separator + "versionigQuery" + (queryType + 1) + "." + (querySubType + 1) + "." + (querySubstParam + 1) + "_results.json"), expectedDataBytes);
} catch (IOException e) {
LOGGER.error("Exception caught during saving of expected results: ", e);
}
}
}
continue;
}
for (int querySubstParam = 0; querySubstParam < subsParametersAmount; querySubstParam++) {
ByteBuffer expectedResultsBuffer = ByteBuffer.wrap(tasks.get(taskId++).getExpectedAnswers());
expectedResultsBuffer.getInt();
expectedResultsBuffer.getInt();
byte expectedDataBytes[] = RabbitMQUtils.readByteArray(expectedResultsBuffer);
try {
FileUtils.writeByteArrayToFile(new File(resultsDir + File.separator + "versionigQuery" + (queryType + 1) + ".1." + (querySubstParam + 1) + "_results.json"), expectedDataBytes);
} catch (IOException e) {
LOGGER.error("Exception caught during saving of expected results : ", e);
}
if (queryType == 0) break;
}
}
}

public void sendAllToFTP() {
writeResults();
FTPUtils.sendToFtp("/versioning/data/v0/", "public/MOCHA_ESWC2018/Task3/data/changesets/c0", "nt");
FTPUtils.sendToFtp("/versioning/data/c1/", "public/MOCHA_ESWC2018/Task3/data/changesets/c1", "nt");
FTPUtils.sendToFtp("/versioning/data/c2/", "public/MOCHA_ESWC2018/Task3/data/changesets/c2", "nt");
FTPUtils.sendToFtp("/versioning/data/c3/", "public/MOCHA_ESWC2018/Task3/data/changesets/c3", "nt");
FTPUtils.sendToFtp("/versioning/data/c4/", "public/MOCHA_ESWC2018/Task3/data/changesets/c4", "nt");
FTPUtils.sendToFtp("/versioning/data/final/v0/", "public/MOCHA_ESWC2018/Task3/data/independentcopies/v0", "nt");
FTPUtils.sendToFtp("/versioning/data/final/v1/", "public/MOCHA_ESWC2018/Task3/data/independentcopies/v1", "nt");
FTPUtils.sendToFtp("/versioning/data/final/v2/", "public/MOCHA_ESWC2018/Task3/data/independentcopies/v2", "nt");
FTPUtils.sendToFtp("/versioning/data/final/v3/", "public/MOCHA_ESWC2018/Task3/data/independentcopies/v3", "nt");
FTPUtils.sendToFtp("/versioning/data/final/v4/", "public/MOCHA_ESWC2018/Task3/data/independentcopies/v4", "nt");
FTPUtils.sendToFtp("/versioning/queries/", "public/MOCHA_ESWC2018/Task3/queries", "sparql");
FTPUtils.sendToFtp("/versioning/query_templates/", "public/MOCHA_ESWC2018/Task3/query_templates", "txt");
FTPUtils.sendToFtp("/versioning/substitution_parameters/", "public/MOCHA_ESWC2018/Task3/substitution_parameters", "txt");
FTPUtils.sendToFtp("/versioning/results/", "public/MOCHA_ESWC2018/Task3/expected_results", "json");
}

@Override
public void close() throws IOException {
LOGGER.info("Closing Data Generator...");
Expand Down
71 changes: 71 additions & 0 deletions src/main/java/org/hobbit/benchmark/versioning/util/FTPUtils.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
package org.hobbit.benchmark.versioning.util;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.List;

import org.apache.commons.io.FileUtils;
import org.apache.commons.net.ftp.FTPClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class FTPUtils {

private static final Logger LOGGER = LoggerFactory.getLogger(FTPUtils.class);

public static void sendToFtp(String inputDir, String outputDir, String fileExtention) {
FTPClient client = new FTPClient();
FileInputStream fis = null;

try {
client.connect("hobbitdata.informatik.uni-leipzig.de");
LOGGER.info("connected: " + client.sendNoOp());
client.enterLocalPassiveMode();
String username = "****";
String pwd = "****";
LOGGER.info("login: " + client.login(username, pwd));

LOGGER.info(client.printWorkingDirectory());
// recursively create ftp folders if not exist
boolean dirExists = true;
String[] directories = outputDir.split("/");
for (String dir : directories ) {
if (!dir.isEmpty() ) {
if (dirExists) {
dirExists = client.changeWorkingDirectory(dir);
}
if (!dirExists) {
if (!client.makeDirectory(dir)) {
throw new IOException("Unable to create remote directory '" + dir + "'. error='" + client.getReplyString()+"'");
}
if (!client.changeWorkingDirectory(dir)) {
throw new IOException("Unable to change into newly created remote directory '" + dir + "'. error='" + client.getReplyString()+"'");
}
}
}
}

LOGGER.info(client.printWorkingDirectory());
File inputDirFile = new File(inputDir);
List<File> inputFiles = (List<File>) FileUtils.listFiles(inputDirFile, new String[] { fileExtention }, true);
for (File file : inputFiles) {
fis = new FileInputStream(file);
client.deleteFile(file.getAbsolutePath());
client.storeFile(file.getName(), fis);
}
client.logout();
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
if (fis != null) {
fis.close();
}
client.disconnect();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}

0 comments on commit 40657fc

Please sign in to comment.