diff --git a/.gitignore b/.gitignore index f4835cc7..f7f4e84b 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ .externalToolBuilders/ .classpath .factorypath +bin/ # Maven target/ diff --git a/application/pom.xml b/application/pom.xml index 20ca1440..c971a78b 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.1-SNAPSHOT smarti-application @@ -66,6 +66,14 @@ com.google.guava guava + + com.googlecode.json-simple + json-simple + + + org.mongodb + mongo-java-driver + commons-io commons-io diff --git a/application/src/main/java/io/redlink/smarti/services/I_ImporterService.java b/application/src/main/java/io/redlink/smarti/services/I_ImporterService.java new file mode 100644 index 00000000..4b62c067 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/I_ImporterService.java @@ -0,0 +1,12 @@ +package io.redlink.smarti.services; + +import io.redlink.smarti.services.importer.I_ConversationSource; +import io.redlink.smarti.services.importer.I_ConversationTarget; + +public interface I_ImporterService { + + static final String SMARTI_IMPORT_WEBSERVICE = "import"; + + void importComversation(I_ConversationSource source, I_ConversationTarget target) throws Exception; + +} diff --git a/application/src/main/java/io/redlink/smarti/services/ImporterService.java b/application/src/main/java/io/redlink/smarti/services/ImporterService.java new file mode 100644 index 00000000..0593a9d6 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/ImporterService.java @@ -0,0 +1,129 @@ +package io.redlink.smarti.services; + +import java.io.IOException; + +import java.text.ParseException; +import java.util.Date; +import java.util.Iterator; + +import org.apache.http.client.ClientProtocolException; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import io.redlink.smarti.api.StoreService; +import io.redlink.smarti.model.Conversation; +import io.redlink.smarti.model.ConversationMeta; +import io.redlink.smarti.model.Message; +import io.redlink.smarti.model.User; +import io.redlink.smarti.services.importer.I_ConversationSource; +import io.redlink.smarti.services.importer.I_ConversationTarget; + +/** + * This service provides an Smarti importer.

+ * + * Sources for import data to Smarti are: + *

  • Directly from Rocket.Chat MongoDB
    • + *
  • A Rocket.Chat MongoDB Export
    • + *
  • An E-Mailbox
    • + *
  • A FAQ website
    • + * + * @author Ruediger Kurz + */ +@Service +public class ImporterService implements I_ImporterService { + + private Logger log = LoggerFactory.getLogger(ImporterService.class); + + @Autowired + private ConversationService conversationService; + + @Autowired + private StoreService storeService; + + /** + * @see I_ImporterService#importComversation(I_ConversationSource, I_ConversationTarget) + */ + @Override + public void importComversation(I_ConversationSource source, I_ConversationTarget target) throws Exception { + + if (source != null && target != null) { + long time = new Date().getTime(); + log.info("Start export at : " + (new Date().getTime() - time) + " ms"); + String export = source.exportConversations(); + log.info("Start parsing at : " + (new Date().getTime() - time) + " ms"); + JSONObject eportedJSON = (JSONObject) new JSONParser().parse(export); + log.info("Start import at : " + (new Date().getTime() - time) + " ms"); + importJSON(source, target, eportedJSON); + log.info("Finished import at : " + (new Date().getTime() - time) + " ms"); + } + } + + /** + * @param source + * @param target + * @param export + * + * @throws IOException + * @throws ClientProtocolException + * @throws ParseException + */ + @SuppressWarnings("unchecked") + private void importJSON(I_ConversationSource source, I_ConversationTarget target, JSONObject export) throws IOException, ClientProtocolException, ParseException { + + Iterator entryIterator = ((JSONArray) export.get("export")).iterator(); + while (entryIterator.hasNext()) { + + JSONObject expEntry = entryIterator.next(); + if ("r".equals((String) expEntry.get("t"))) { + // the type of this entry is a room + + String channelId = (String) expEntry.get("_id"); + String channelName = (String) expEntry.get("name"); + + JSONArray messages = (JSONArray) expEntry.get("messages"); + Iterator messageIterator = messages.iterator(); + + Message m = null; + Conversation conversation = null; + + while (messageIterator.hasNext()) { + JSONObject message = messageIterator.next(); + Object t = message.get("t"); + if (t == null) { + conversation = storeService.getCurrentConversationByChannelId(channelId, () -> { + final Conversation newConversation = new Conversation(); + newConversation.getContext().setContextType(source.getSourceType().toString()); + newConversation.getContext().setDomain(target.getClientId()); + newConversation.getContext().setEnvironment("channel", channelName); + newConversation.getContext().setEnvironment("channel_id", channelId); + newConversation.getContext().setEnvironment("token", target.getToken()); + return newConversation; + }); + + m = new Message(); + m.setId((String) message.get("_id")); + m.setContent((String) message.get("msg")); + m.setTime(new Date((Long) message.get("_updatedAt"))); + m.setOrigin(Message.Origin.User); + + // TODO: Use a UserService to actually *store* the users + JSONObject user = (JSONObject) message.get("u"); + final User userO = new User((String) user.get("_id")); + userO.setDisplayName((String) user.get("username")); + m.setUser(userO); + + conversation = conversationService.appendMessage(conversation, m); + conversation.getMeta().setStatus(ConversationMeta.Status.Complete); + conversationService.completeConversation(conversation); + log.info(conversation.toString()); + } + } + } + } + } +} diff --git a/application/src/main/java/io/redlink/smarti/services/importer/A_ConversationSource.java b/application/src/main/java/io/redlink/smarti/services/importer/A_ConversationSource.java new file mode 100644 index 00000000..86482cd2 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/importer/A_ConversationSource.java @@ -0,0 +1,47 @@ +package io.redlink.smarti.services.importer; + +import io.redlink.smarti.webservice.pojo.I_SourceConfiguration; + +/** + * Abstract implementation of a generic data source.

    + * + * @author Ruediger Kurz + */ +public abstract class A_ConversationSource implements I_ConversationSource { + + /** Source type of this data source. */ + private E_SourceType sourceType; + + /** Configuration options for this data source. */ + I_SourceConfiguration sourceConfiguration; + + /** + * Constructor with parameters.

    + * + * @param sourceType the source type + * @param sourceConfiguration the configuration options + */ + public A_ConversationSource (E_SourceType sourceType, I_SourceConfiguration sourceConfiguration) { + + this.sourceType = sourceType; + this.sourceConfiguration = sourceConfiguration; + } + + /** + * @see I_ConversationSource#getSourceType() + */ + @Override + public E_SourceType getSourceType() { + + return this.sourceType; + } + + /** + * @see I_SourceConfiguration#getSourceConfiguration() + */ + @Override + public I_SourceConfiguration getSourceConfiguration() { + + return this.sourceConfiguration; + } +} diff --git a/application/src/main/java/io/redlink/smarti/services/importer/A_ConversationTarget.java b/application/src/main/java/io/redlink/smarti/services/importer/A_ConversationTarget.java new file mode 100644 index 00000000..297080b5 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/importer/A_ConversationTarget.java @@ -0,0 +1,22 @@ +package io.redlink.smarti.services.importer; + +public abstract class A_ConversationTarget implements I_ConversationTarget { + + private String clientId; + private String token; + + public A_ConversationTarget (String clientId, String token) { + this.clientId = clientId; + this.token = token; + } + + @Override + public String getClientId() { + return clientId; + } + + @Override + public String getToken() { + return token; + } +} diff --git a/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatJSONFile.java b/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatJSONFile.java new file mode 100644 index 00000000..153dc8f3 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatJSONFile.java @@ -0,0 +1,42 @@ +package io.redlink.smarti.services.importer; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URL; +import java.nio.charset.Charset; + +import io.redlink.smarti.webservice.pojo.RocketFileConfig; + +public class ConversationSourceRocketChatJSONFile extends A_ConversationSource { + + RocketFileConfig config; + + public ConversationSourceRocketChatJSONFile(RocketFileConfig config) { + super(E_SourceType.RocketChatJSONFile, config); + this.config = config; + } + + @Override + public String exportConversations() throws Exception { + + return readJsonFromUrl(config.getJsonFileURL()); + } + + public static String readJsonFromUrl(String url) throws IOException { + + InputStream is = new URL(url).openStream(); + try { + BufferedReader rd = new BufferedReader(new InputStreamReader(is, Charset.forName("UTF-8"))); + StringBuilder sb = new StringBuilder(); + int cp; + while ((cp = rd.read()) != -1) { + sb.append((char) cp); + } + return sb.toString(); + } finally { + is.close(); + } + } +} diff --git a/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatMongoDB.java b/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatMongoDB.java new file mode 100644 index 00000000..2e6d0202 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatMongoDB.java @@ -0,0 +1,51 @@ +package io.redlink.smarti.services.importer; + +import java.util.Arrays; +import java.util.Collections; + +import org.bson.Document; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; +import com.mongodb.MongoClient; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoCursor; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.Aggregates; +import com.mongodb.client.model.Filters; + +import io.redlink.smarti.webservice.pojo.RocketMongoConfig; + +public class ConversationSourceRocketChatMongoDB extends A_ConversationSource { + + RocketMongoConfig mongoConfig; + + public ConversationSourceRocketChatMongoDB(RocketMongoConfig config) { + + super(E_SourceType.RocketChatMongoDB, config); + mongoConfig = config; + } + + @Override + public String exportConversations() throws Exception { + + MongoClient mongoClient = new MongoClient(mongoConfig.getHost(), mongoConfig.getPort()); + + try { + MongoDatabase db = mongoClient.getDatabase(mongoConfig.getDbname()); + MongoCollection coll = db.getCollection(mongoConfig.getRoomCollection()); + // get all rooms of type 'request' and expertise 'x' joint with messages + MongoCursor iterator = coll.aggregate(Arrays.asList( + Aggregates.match(Filters.eq("t", "r")), + Aggregates.match(Filters.regex(mongoConfig.getFilterField(), mongoConfig.getFilterValue())), + Aggregates.lookup(mongoConfig.getMessageCollection(), "_id", "rid", "messages"))).iterator(); + + ObjectMapper mapper = new ObjectMapper(); + String mappString = mapper.writeValueAsString(Collections.singletonMap("export", ImmutableList.copyOf(iterator))); + + return mappString; + } finally { + mongoClient.close(); + } + } +} diff --git a/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatWebservice.java b/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatWebservice.java new file mode 100644 index 00000000..2eac664b --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatWebservice.java @@ -0,0 +1,86 @@ +package io.redlink.smarti.services.importer; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.UnsupportedCharsetException; + +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpResponse; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.ConnectionBackoffStrategy; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.HttpClientBuilder; + +import io.redlink.smarti.webservice.pojo.RocketWebserviceConfig; + +/** + * Implements the Rocket.Chat MongoDB as data source for Smarti + * + * @author Ruediger Kurz + */ +public class ConversationSourceRocketChatWebservice extends A_ConversationSource { + + RocketWebserviceConfig webserviceConfig; + + public ConversationSourceRocketChatWebservice(RocketWebserviceConfig config) { + + super(E_SourceType.RocketChatWebServie, config); + webserviceConfig = config; + } + + @Override + public String exportConversations() throws Exception { + + return getMongoRequestExport(); + } + + /** + * + * @return + * + * @throws IOException + * @throws ClientProtocolException + * @throws IOException + * @throws ClientProtocolException + * @throws UnsupportedCharsetException + */ + private String getMongoRequestExport() throws IOException, ClientProtocolException, IOException, ClientProtocolException, UnsupportedCharsetException { + + HttpPost post = new HttpPost(webserviceConfig.getRocketChatEndpoint()); + post.addHeader("Content-Type", "application/json; charset=UTF-8"); + post.addHeader("Accept-Encoding", "gzip,deflate,sdch"); + post.addHeader("Accept-Language", "en-US,en;q=0.8"); + + StringEntity entity = new StringEntity(getSourceConfiguration().asString(), "UTF-8"); + + entity.setContentType("application/json"); + post.setEntity(entity); + + try (CloseableHttpResponse response = httpClientBuilder.build().execute(post)) { + if (200 == response.getStatusLine().getStatusCode()) { + return IOUtils.toString(response.getEntity().getContent(), + Charset.defaultCharset()); + } + } + return null; + } + + /** + * A HTTP client to use the Smarti web service.

    + */ + private final HttpClientBuilder httpClientBuilder = HttpClientBuilder.create() + .setRetryHandler((exception, executionCount, context) -> executionCount < 3) + .setConnectionBackoffStrategy(new ConnectionBackoffStrategy() { + @Override + public boolean shouldBackoff(HttpResponse resp) { + return false; + } + + @Override + public boolean shouldBackoff(Throwable t) { + return t instanceof IOException; + } + }).setUserAgent("Smarti/0.0 Rocket.Chat-Endpoint/0.1"); +} diff --git a/application/src/main/java/io/redlink/smarti/services/importer/ConversationTargetSmarti.java b/application/src/main/java/io/redlink/smarti/services/importer/ConversationTargetSmarti.java new file mode 100644 index 00000000..9a20cb7d --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/importer/ConversationTargetSmarti.java @@ -0,0 +1,11 @@ +package io.redlink.smarti.services.importer; + +public class ConversationTargetSmarti extends A_ConversationTarget { + + public ConversationTargetSmarti(String clientId, String token) { + + super(clientId, token); + } + + +} diff --git a/application/src/main/java/io/redlink/smarti/services/importer/I_ConversationSource.java b/application/src/main/java/io/redlink/smarti/services/importer/I_ConversationSource.java new file mode 100644 index 00000000..3aa0db42 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/importer/I_ConversationSource.java @@ -0,0 +1,40 @@ +package io.redlink.smarti.services.importer; + +import io.redlink.smarti.webservice.pojo.I_SourceConfiguration; + +/** + * Interface for potential conversational data sources.

    + * + * @author Ruediger Kurz + * + */ +public interface I_ConversationSource { + + /** The import source type. */ + static enum E_SourceType { + RocketChatJSONFile, RocketChatMongoDB, RocketChatWebServie, Mail, FAQs + } + + /** + * Returns the type of source.

    + * + * @return the type of source + */ + E_SourceType getSourceType(); + + /** + * Returns the source specific configuration options.

    + * + * @return the configuration options + */ + I_SourceConfiguration getSourceConfiguration(); + + /** + * Executes the export action and returns the export result as JSON.

    + * + * @return the export as JSON + * + * @throws Exception if something goes wrong + */ + String exportConversations() throws Exception; +} diff --git a/application/src/main/java/io/redlink/smarti/services/importer/I_ConversationTarget.java b/application/src/main/java/io/redlink/smarti/services/importer/I_ConversationTarget.java new file mode 100644 index 00000000..48516143 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/importer/I_ConversationTarget.java @@ -0,0 +1,7 @@ +package io.redlink.smarti.services.importer; + +public interface I_ConversationTarget { + + String getClientId(); + String getToken(); +} diff --git a/application/src/main/java/io/redlink/smarti/webservice/ImporterServiceEndpoint.java b/application/src/main/java/io/redlink/smarti/webservice/ImporterServiceEndpoint.java new file mode 100644 index 00000000..37a92124 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/webservice/ImporterServiceEndpoint.java @@ -0,0 +1,129 @@ +package io.redlink.smarti.webservice; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.ResponseEntity; +import org.springframework.util.MimeTypeUtils; +import org.springframework.web.bind.annotation.CrossOrigin; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import io.redlink.smarti.services.I_ImporterService; +import io.redlink.smarti.services.ImporterService; +import io.redlink.smarti.services.importer.ConversationSourceRocketChatJSONFile; +import io.redlink.smarti.services.importer.ConversationSourceRocketChatMongoDB; +import io.redlink.smarti.services.importer.ConversationSourceRocketChatWebservice; +import io.redlink.smarti.services.importer.I_ConversationSource; +import io.redlink.smarti.services.importer.I_ConversationTarget; +import io.redlink.smarti.webservice.pojo.RocketMongoConfig; +import io.redlink.smarti.webservice.pojo.RocketFileConfig; +import io.redlink.smarti.webservice.pojo.RocketWebserviceConfig; +import io.redlink.smarti.services.importer.ConversationTargetSmarti; +import io.redlink.smarti.services.importer.I_ConversationSource.E_SourceType; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; + +@CrossOrigin +@RestController +@RequestMapping(value = I_ImporterService.SMARTI_IMPORT_WEBSERVICE, + consumes = MimeTypeUtils.APPLICATION_JSON_VALUE, + produces = MimeTypeUtils.APPLICATION_JSON_VALUE) +@Api(I_ImporterService.SMARTI_IMPORT_WEBSERVICE) +public class ImporterServiceEndpoint { + + @Autowired + private ImporterService importerService; + + private Logger log = LoggerFactory.getLogger(ImporterServiceEndpoint.class); + + /** + * This web service should be part of Rocket.Chat not of Smarti. + * + * @param payload + * + * @return the exported chat messages of help request channels + */ + @ApiOperation(value = "imports the conversation from the given source", produces=MimeTypeUtils.APPLICATION_JSON_VALUE) + @RequestMapping(value = "{clientId}/{sourceType}", method = RequestMethod.GET, + produces=MimeTypeUtils.APPLICATION_JSON_VALUE, consumes=MimeTypeUtils.APPLICATION_JSON_VALUE) + public ResponseEntity getImport( + @PathVariable(value="clientId") String clientId, + @PathVariable(value="sourceType") String sourceType, + @RequestBody String payload) { + + ObjectMapper mapper = new ObjectMapper(); + + try { + E_SourceType type = E_SourceType.valueOf(sourceType); + switch (type) { + case RocketChatJSONFile: + importRocketChatJSONFile(clientId, mapper.readValue(payload, RocketFileConfig.class)); + break; + case RocketChatMongoDB: + importRocketChatMongoDB(clientId, mapper.readValue(payload, RocketMongoConfig.class)); + break; + case RocketChatWebServie: + importRocketChatWebservice(clientId, mapper.readValue(payload, RocketWebserviceConfig.class)); + break; + default: + return ResponseEntity.ok("{status: nosource}"); + } + return ResponseEntity.ok("{status: sucess}"); + } catch (Exception e) { + return ResponseEntity.status(500).build(); + } + } + + @ApiOperation(value = "imports the conversation from the given source", produces=MimeTypeUtils.APPLICATION_JSON_VALUE) + @RequestMapping(value = "{clientId}/rocketChatJSONFile", method = RequestMethod.GET, + produces=MimeTypeUtils.APPLICATION_JSON_VALUE, consumes=MimeTypeUtils.APPLICATION_JSON_VALUE) + public ResponseEntity importRocketChatJSONFile( + @PathVariable(value="clientId") String clientId, + @RequestBody RocketFileConfig fileConfig) { + + log.warn("called import webservice: '/import/" + clientId + "/rocketChatJSONFile' with config: " + fileConfig.toString()); + return runImport (new ConversationSourceRocketChatJSONFile(fileConfig), clientId); + } + + @ApiOperation(value = "imports the conversation from the given source", produces=MimeTypeUtils.APPLICATION_JSON_VALUE) + @RequestMapping(value = "{clientId}/rocketChatMongoDB", method = RequestMethod.GET, + produces=MimeTypeUtils.APPLICATION_JSON_VALUE, consumes=MimeTypeUtils.APPLICATION_JSON_VALUE) + public ResponseEntity importRocketChatMongoDB( + @PathVariable(value="clientId") String clientId, + @RequestBody RocketMongoConfig mongoConfig) { + + log.warn("called import webservice: '/import/" + clientId + "/rocketChatJSONFile' with config: " + mongoConfig.toString()); + return runImport (new ConversationSourceRocketChatMongoDB(mongoConfig), clientId); + } + + @ApiOperation(value = "imports the conversation from the given source", produces=MimeTypeUtils.APPLICATION_JSON_VALUE) + @RequestMapping(value = "{clientId}/rocketChatWebservice", method = RequestMethod.GET, + produces=MimeTypeUtils.APPLICATION_JSON_VALUE, consumes=MimeTypeUtils.APPLICATION_JSON_VALUE) + public ResponseEntity importRocketChatWebservice( + @PathVariable(value="clientId") String clientId, + @RequestBody RocketWebserviceConfig webserviceConfig) { + + log.warn("called import webservice: '/import/" + clientId + "/rocketChatJSONFile' with config: " + webserviceConfig.toString()); + return runImport (new ConversationSourceRocketChatWebservice(webserviceConfig), clientId); + } + + private ResponseEntity runImport (I_ConversationSource source, String clientId) { + // TODO: Token not yet implemented, replace key123. + I_ConversationTarget target = new ConversationTargetSmarti(clientId, "key123"); + try { + importerService.importComversation(source, target); + return ResponseEntity.ok("{status: sucess}"); + } catch (Exception e) { + log.error("Error with message: \"{}\" occured during import: ", e.getMessage(), e); + return ResponseEntity.status(500).build(); + } + + } + +} diff --git a/application/src/main/java/io/redlink/smarti/webservice/RocketChatEndpoint.java b/application/src/main/java/io/redlink/smarti/webservice/RocketChatEndpoint.java index 15ab8539..793e6bdf 100644 --- a/application/src/main/java/io/redlink/smarti/webservice/RocketChatEndpoint.java +++ b/application/src/main/java/io/redlink/smarti/webservice/RocketChatEndpoint.java @@ -146,6 +146,11 @@ public ResponseEntity onRocketEvent(@PathVariable("clientId") String clientId private void notifyRocketChat(String callbackUrl, Conversation conversation, String token) { try (CloseableHttpClient httpClient = httpClientBuilder.build()) { + + if (StringUtils.isEmpty(callbackUrl)) { + log.warn("Webhook URL is empty."); + return; + } final HttpPost post = new HttpPost(callbackUrl); post.setEntity(new StringEntity( toJsonString(new SmartiUpdatePing(conversation.getId(), conversation.getContext().getEnvironment("channel_id"), token)), diff --git a/application/src/main/java/io/redlink/smarti/webservice/pojo/A_SourceConfiguration.java b/application/src/main/java/io/redlink/smarti/webservice/pojo/A_SourceConfiguration.java new file mode 100644 index 00000000..819df061 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/webservice/pojo/A_SourceConfiguration.java @@ -0,0 +1,24 @@ +package io.redlink.smarti.webservice.pojo; + +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class A_SourceConfiguration implements I_SourceConfiguration { + + @Override + public JSONObject asJSON() throws ParseException, JsonProcessingException { + + return (JSONObject) new JSONParser().parse(toString()); + } + + public String asString() throws JsonProcessingException { + + ObjectMapper mapper = new ObjectMapper(); + return mapper.writeValueAsString(this); + } + +} \ No newline at end of file diff --git a/application/src/main/java/io/redlink/smarti/webservice/pojo/I_SourceConfiguration.java b/application/src/main/java/io/redlink/smarti/webservice/pojo/I_SourceConfiguration.java new file mode 100644 index 00000000..e0e57479 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/webservice/pojo/I_SourceConfiguration.java @@ -0,0 +1,13 @@ +package io.redlink.smarti.webservice.pojo; + +import org.json.simple.JSONObject; +import org.json.simple.parser.ParseException; + +import com.fasterxml.jackson.core.JsonProcessingException; + +public interface I_SourceConfiguration { + + JSONObject asJSON() throws ParseException, JsonProcessingException; + + String asString() throws JsonProcessingException; +} diff --git a/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketBot.java b/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketBot.java index d73fe06e..a71cce81 100644 --- a/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketBot.java +++ b/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketBot.java @@ -42,7 +42,7 @@ public void setIdentifier(String identifier) { this.identifier = identifier; } - public static class JacksonDeserializer extends JsonDeserializer { + public static class JacksonDeserializer extends JsonDeserializer { public RocketBot deserialize(JsonParser parser, DeserializationContext context) throws IOException, JsonProcessingException { if(JsonToken.START_OBJECT.equals(parser.getCurrentToken())) { ObjectMapper mapper = new ObjectMapper(); diff --git a/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketFileConfig.java b/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketFileConfig.java new file mode 100644 index 00000000..f7cacb8a --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketFileConfig.java @@ -0,0 +1,34 @@ +package io.redlink.smarti.webservice.pojo; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ +"json_file_url" +}) +/** + * @author Ruediger Kurz (ruediger.kurz@deutschebahn.com) + * @since 03.08.2017 + */ +public class RocketFileConfig extends A_SourceConfiguration { + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("RocketFileConfig [jsonFileURL=").append(jsonFileURL).append("]"); + return builder.toString(); + } + + @JsonProperty("json_file_url") + private String jsonFileURL; + + public String getJsonFileURL() { + return jsonFileURL; + } + + public void setJsonFileURL(String jsonFileURL) { + this.jsonFileURL = jsonFileURL; + } +} diff --git a/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketMongoConfig.java b/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketMongoConfig.java new file mode 100644 index 00000000..3aebaabd --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketMongoConfig.java @@ -0,0 +1,103 @@ +package io.redlink.smarti.webservice.pojo; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +/** + * @author Ruediger Kurz (ruediger.kurz@deutschebahn.com) + * @since 03.08.2017 + */ +public class RocketMongoConfig extends A_SourceConfiguration { + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("RocketMongoConfig [dbname=").append(dbname).append(", filterField=").append(filterField) + .append(", filterValue=").append(filterValue).append(", host=").append(host) + .append(", messageCollection=").append(messageCollection).append(", port=").append(port) + .append(", roomCollection=").append(roomCollection).append("]"); + return builder.toString(); + } + + @JsonProperty("dbname") + private String dbname; + + @JsonProperty("filter_field") + private String filterField; + + @JsonProperty("filter_value") + private String filterValue; + + @JsonProperty("host") + private String host; + + @JsonProperty("message_collection") + private String messageCollection; + + @JsonProperty("port") + private int port; + + @JsonProperty("room_collection") + private String roomCollection; + + public RocketMongoConfig() { + } + + public String getDbname() { + return dbname; + } + + public String getFilterField() { + return filterField; + } + + public String getFilterValue() { + return filterValue; + } + + public String getHost() { + return host; + } + + public String getMessageCollection() { + return messageCollection; + } + + public int getPort() { + return port; + } + + public String getRoomCollection() { + return roomCollection; + } + + public void setDbname(String dbname) { + this.dbname = dbname; + } + + public void setFilterField(String filterField) { + this.filterField = filterField; + } + + public void setFilterValue(String filterValue) { + this.filterValue = filterValue; + } + + public void setHost(String host) { + this.host = host; + } + + public void setMessageCollection(String messageCollection) { + this.messageCollection = messageCollection; + } + + public void setPort(int port) { + this.port = port; + } + + public void setRoomCollection(String roomCollection) { + this.roomCollection = roomCollection; + } + +} diff --git a/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketWebserviceConfig.java b/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketWebserviceConfig.java new file mode 100644 index 00000000..74e72ca7 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketWebserviceConfig.java @@ -0,0 +1,78 @@ +package io.redlink.smarti.webservice.pojo; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + + +@JsonInclude(JsonInclude.Include.NON_NULL) +/** + * @author Ruediger Kurz (ruediger.kurz@deutschebahn.com) + * @since 03.08.2017 + */ +public class RocketWebserviceConfig extends A_SourceConfiguration { + + @JsonProperty("rc_endpoint") + private String rocketChatEndpoint; + + @JsonProperty("room_collection") + private String roomCollection; + + @JsonProperty("message_collection") + private String messageCollection; + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("RocketWebserviceConfig [rocketChatEndpoint=").append(rocketChatEndpoint) + .append(", roomCollection=").append(roomCollection).append(", messageCollection=") + .append(messageCollection).append(", filterField=").append(filterField).append(", filterValue=") + .append(filterValue).append("]"); + return builder.toString(); + } + + @JsonProperty("filter_field") + private String filterField; + + @JsonProperty("filter_value") + private String filterValue; + + public String getRocketChatEndpoint() { + return rocketChatEndpoint; + } + + public void setRocketChatEndpoint(String rocketChatEndpoint) { + this.rocketChatEndpoint = rocketChatEndpoint; + } + + public String getRoomCollection() { + return roomCollection; + } + + public void setRoomCollection(String roomCollection) { + this.roomCollection = roomCollection; + } + + public String getMessageCollection() { + return messageCollection; + } + + public void setMessageCollection(String messageCollection) { + this.messageCollection = messageCollection; + } + + public String getFilterField() { + return filterField; + } + + public void setFilterField(String filterField) { + this.filterField = filterField; + } + + public String getFilterValue() { + return filterValue; + } + + public void setFilterValue(String filterValue) { + this.filterValue = filterValue; + } +} diff --git a/application/src/main/test/io/redlink/smarti/webservice/ConversationImporterTest.java b/application/src/main/test/io/redlink/smarti/webservice/ConversationImporterTest.java new file mode 100644 index 00000000..24bff699 --- /dev/null +++ b/application/src/main/test/io/redlink/smarti/webservice/ConversationImporterTest.java @@ -0,0 +1,124 @@ +package io.redlink.smarti.webservice; + +import java.io.IOException; + +import org.apache.http.client.ClientProtocolException; +import org.apache.http.entity.StringEntity; +import org.apache.http.localserver.LocalTestServer; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import io.redlink.smarti.services.ImporterService; +import io.redlink.smarti.services.importer.ConversationSourceRocketChatJSONFile; +import io.redlink.smarti.services.importer.ConversationSourceRocketChatMongoDB; +import io.redlink.smarti.services.importer.ConversationSourceRocketChatWebservice; +import io.redlink.smarti.services.importer.ConversationTargetSmarti; +import io.redlink.smarti.services.importer.I_ConversationSource; +import io.redlink.smarti.services.importer.I_ConversationSource.E_SourceType; +import io.redlink.smarti.services.importer.I_ConversationTarget; +import io.redlink.smarti.webservice.pojo.RocketFileConfig; +import io.redlink.smarti.webservice.pojo.RocketMongoConfig; +import io.redlink.smarti.webservice.pojo.RocketWebserviceConfig; + +/** + * + * @author Ruediger Kurz (ruediger.kurz@deutschebahn.com) + * @since 03.08.2017 + */ +public class ConversationImporterTest { + + /** The database name of Rocket.Chat. */ + private static final String ROCKETCHAT_MONGO_DB_NAME = "rocketchat"; + + /** The name of the Rocket.Chat collection that holds the chat messages. */ + private static final String ROCKETCHAT_MESSAGE_COLLECTION = "rocketchat_message"; + + /** The name of the Rocket.Chat collection that holds the rooms. */ + private static final String ROCKETCHAT_ROOM_COLLECTION = "rocketchat_room"; + + /** The host name of the Rocket.Chat MongoDB server. */ + private static final String ROCKETCHAT_MONGO_HOST = "localhost"; + + /** The port number of the Rocket.Chat MongoDB. */ + private static final int ROCKETCHAT_MONGO_PORT = 27017; + + /** The name of a room field/property that is used to filter the rooms that should be exported from Rocket.Chat. */ + private static final String ROCKETCHAT_FILTER_ROOM_FIELD_NAME = "expertise"; + + /** The value of the room field that must match; a regex can be used. */ + private static final String ROCKETCHAT_FILTER_ROOM_FIELD_VALUE = "assistify"; + + /** The absolute path to the Rocket.Chat JSON export. */ + private static final String FILE_IMPORT_SOURCE = "file:///Users/rudiger/projects/smarti/application/src/test/resources/export-sapsus-faqs.json"; + + /** The id of the client that is used to identify the knowledge domain in Smarti. */ + private static final String SMARTI_CLIENT_ID = "test.assistify.de"; + + private LocalTestServer server = new LocalTestServer(null, null); + + @Before + public void setUp() throws Exception { + server.start(); + server.register("*", (httpRequest, httpResponse, httpContext) -> + httpResponse.setEntity(new StringEntity("foobar"))); + } + + @After + public void tearDown() throws Exception { + server.stop(); + } + + @Test + public void testJSONFile() throws ClientProtocolException, IOException, Exception { + runImport(E_SourceType.RocketChatJSONFile); + } + + private void runImport(E_SourceType type) throws ClientProtocolException, IOException, Exception { + + I_ConversationTarget target = new ConversationTargetSmarti(SMARTI_CLIENT_ID, "key123"); + I_ConversationSource source = null; + + switch (type) { + + case RocketChatJSONFile: + RocketFileConfig fileConfig = new RocketFileConfig(); + fileConfig.setJsonFileURL(FILE_IMPORT_SOURCE); + source = new ConversationSourceRocketChatJSONFile (fileConfig); + break; + + case RocketChatMongoDB: + RocketMongoConfig mongoConfig = new RocketMongoConfig(); + mongoConfig.setHost(ROCKETCHAT_MONGO_HOST); + mongoConfig.setPort(ROCKETCHAT_MONGO_PORT); + mongoConfig.setDbname(ROCKETCHAT_MONGO_DB_NAME); + mongoConfig.setRoomCollection(ROCKETCHAT_ROOM_COLLECTION); + mongoConfig.setMessageCollection(ROCKETCHAT_MESSAGE_COLLECTION); + mongoConfig.setFilterField(ROCKETCHAT_FILTER_ROOM_FIELD_NAME); + mongoConfig.setFilterValue(ROCKETCHAT_FILTER_ROOM_FIELD_VALUE); + source = new ConversationSourceRocketChatMongoDB(mongoConfig); + break; + + case RocketChatWebServie: + RocketWebserviceConfig webserviceConfig = new RocketWebserviceConfig(); + webserviceConfig.setRocketChatEndpoint("magic"); + webserviceConfig.setRoomCollection(ROCKETCHAT_ROOM_COLLECTION); + webserviceConfig.setMessageCollection(ROCKETCHAT_MESSAGE_COLLECTION); + webserviceConfig.setFilterField(ROCKETCHAT_FILTER_ROOM_FIELD_NAME); + webserviceConfig.setFilterValue(ROCKETCHAT_FILTER_ROOM_FIELD_VALUE); + source = new ConversationSourceRocketChatWebservice(webserviceConfig); + break; + + case FAQs: + break; + + case Mail: + break; + + default: + break; + } + + new ImporterService().importComversation(source, target); + } +} diff --git a/application/src/main/test/io/redlink/smarti/webservice/RocketChatEndpointProxyTest.java b/application/src/main/test/io/redlink/smarti/webservice/RocketChatEndpointProxyTest.java index 870d2f1d..b306f249 100644 --- a/application/src/main/test/io/redlink/smarti/webservice/RocketChatEndpointProxyTest.java +++ b/application/src/main/test/io/redlink/smarti/webservice/RocketChatEndpointProxyTest.java @@ -49,7 +49,5 @@ public void test() throws IOException { String responseString = IOUtils.toString(response.getEntity().getContent(), Charset.defaultCharset()); Assert.assertEquals("foobar", responseString); } - } - } diff --git a/core/pom.xml b/core/pom.xml index 8d4ae2c8..002057ee 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.1-SNAPSHOT smarti-core diff --git a/data/pom.xml b/data/pom.xml index 905afbe2..f81d4a69 100644 --- a/data/pom.xml +++ b/data/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.1-SNAPSHOT data diff --git a/data/solrcore-crawl-systel/pom.xml b/data/solrcore-crawl-systel/pom.xml index d46d3853..f253df1c 100644 --- a/data/solrcore-crawl-systel/pom.xml +++ b/data/solrcore-crawl-systel/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.1-SNAPSHOT ../../ diff --git a/data/solrcore-wikipedia-de/pom.xml b/data/solrcore-wikipedia-de/pom.xml index 8958f303..651bb3d5 100644 --- a/data/solrcore-wikipedia-de/pom.xml +++ b/data/solrcore-wikipedia-de/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.1-SNAPSHOT ../../ diff --git a/dist/pom.xml b/dist/pom.xml index f2b0ba6c..345521b3 100644 --- a/dist/pom.xml +++ b/dist/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.1-SNAPSHOT smarti-dist diff --git a/integration/rocket-chat/pom.xml b/integration/rocket-chat/pom.xml index 725c3265..93b6e6fe 100644 --- a/integration/rocket-chat/pom.xml +++ b/integration/rocket-chat/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.1-SNAPSHOT ../.. diff --git a/lib/hasso-vocabulary-extractors/pom.xml b/lib/hasso-vocabulary-extractors/pom.xml index 0d6e9152..5b490b3c 100644 --- a/lib/hasso-vocabulary-extractors/pom.xml +++ b/lib/hasso-vocabulary-extractors/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.1-SNAPSHOT ../../ diff --git a/lib/keyword-interestingterms/pom.xml b/lib/keyword-interestingterms/pom.xml index fed6d028..8efac099 100644 --- a/lib/keyword-interestingterms/pom.xml +++ b/lib/keyword-interestingterms/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.1-SNAPSHOT ../../ diff --git a/lib/keyword-solr-mlt/pom.xml b/lib/keyword-solr-mlt/pom.xml index 77659941..876c9a2e 100644 --- a/lib/keyword-solr-mlt/pom.xml +++ b/lib/keyword-solr-mlt/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.1-SNAPSHOT ../../ diff --git a/lib/ner/pom.xml b/lib/ner/pom.xml index 260fab19..0065da0a 100644 --- a/lib/ner/pom.xml +++ b/lib/ner/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.1-SNAPSHOT ../../ diff --git a/lib/pom.xml b/lib/pom.xml index bc162e22..d577f2a5 100644 --- a/lib/pom.xml +++ b/lib/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.1-SNAPSHOT lib diff --git a/lib/pos/pom.xml b/lib/pos/pom.xml index 62d9ff54..b8c517bf 100644 --- a/lib/pos/pom.xml +++ b/lib/pos/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.1-SNAPSHOT ../../ diff --git a/lib/query-conversation/pom.xml b/lib/query-conversation/pom.xml index df1b72cc..3e90dea1 100644 --- a/lib/query-conversation/pom.xml +++ b/lib/query-conversation/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.1-SNAPSHOT ../../ diff --git a/lib/query-dbsearch/pom.xml b/lib/query-dbsearch/pom.xml index 2ada6ba1..0a3c265c 100644 --- a/lib/query-dbsearch/pom.xml +++ b/lib/query-dbsearch/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.1-SNAPSHOT ../../ diff --git a/lib/token-processor/pom.xml b/lib/token-processor/pom.xml index 6a10794c..90d14b21 100644 --- a/lib/token-processor/pom.xml +++ b/lib/token-processor/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.1-SNAPSHOT ../../ diff --git a/pom.xml b/pom.xml index 1591d798..ceb80890 100644 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.1-SNAPSHOT pom smarti @@ -275,7 +275,7 @@ 1.8 - +