Added conversation import web service

.gitignore

@@ -10,6 +10,7 @@
 .externalToolBuilders/
 .classpath
 .factorypath
+bin/
 
 # Maven
 target/

application/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <groupId>io.redlink.smarti</groupId>
         <artifactId>smarti</artifactId>
-        <version>0.2.0-SNAPSHOT</version>
+        <version>0.2.1-SNAPSHOT</version>
     </parent>
 
     <artifactId>smarti-application</artifactId>
@@ -66,6 +66,14 @@
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>
         </dependency>
+		<dependency>
+		    <groupId>com.googlecode.json-simple</groupId>
+		    <artifactId>json-simple</artifactId>
+		</dependency>
+	    <dependency>
+	        <groupId>org.mongodb</groupId>
+	        <artifactId>mongo-java-driver</artifactId>
+	    </dependency>
         <dependency>
             <groupId>commons-io</groupId>
             <artifactId>commons-io</artifactId>

application/src/main/java/io/redlink/smarti/services/I_ImporterService.java

@@ -0,0 +1,18 @@
+package io.redlink.smarti.services;
+
+import io.redlink.smarti.services.importer.I_ConversationSource;
+import io.redlink.smarti.services.importer.I_ConversationTarget;
+
+public interface I_ImporterService {
+
+	static final String SMARTI_IMPORT_WEBSERVICE = "import";
+
+	/** The postfix to convert the Rocket.Chat query result to a valid JSON. */
+	 static final String JSON_RESULT_WRAPPER_END = "]}";
+
+	/** The prefix to convert the Rocket.Chat query result to a valid JSON. */
+	 static final String JSON_RESULT_WRAPPER_START = "{\"export\": [";
+
+	void importComversation(I_ConversationSource source, I_ConversationTarget target) throws Exception;
+
+}

application/src/main/java/io/redlink/smarti/services/ImporterService.java

@@ -0,0 +1,150 @@
+package io.redlink.smarti.services;
+
+import static org.hamcrest.CoreMatchers.instanceOf;
+
+import java.io.IOException;
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Iterator;
+
+import org.apache.http.client.ClientProtocolException;
+import org.bson.types.ObjectId;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import io.redlink.smarti.services.importer.I_ConversationSource;
+import io.redlink.smarti.services.importer.I_ConversationTarget;
+import io.redlink.smarti.webservice.ConversationWebservice;
+import io.redlink.smarti.webservice.RocketChatEndpoint;
+import io.redlink.smarti.webservice.pojo.RocketEvent;
+
+/**
+ * This service provides an Smarti importer.<p>
+ * 
+ * Sources for import data to Smarti are:
+ * <li>Directly from Rocket.Chat MongoDB</li>
+ * <li>A Rocket.Chat MongoDB Export</li>
+ * <li>An E-Mailbox</li>
+ * <li>A FAQ website</li>  
+ * 
+ * @author Ruediger Kurz
+ */
+@Service
+public class ImporterService implements I_ImporterService {
+
+	private Logger log = LoggerFactory.getLogger(ImporterService.class);
+
+	/** A simple ISO date formatter. */
+	private static final DateFormat ISODateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
+
+    @Autowired
+    private ConversationWebservice conversationWebservice;
+
+    @Autowired
+    private RocketChatEndpoint rcEndpoint;
+
+    public ImporterService() {
+    		rcEndpoint = new RocketChatEndpoint(null, 0, null);
+    }
+
+    long time;
+
+	/**
+	 * @see I_ImporterService#importComversation(I_ConversationSource, I_ConversationTarget)
+	 */
+	@Override
+	public void importComversation(I_ConversationSource source, I_ConversationTarget target) throws Exception {
+
+
+		if (source != null && target != null) {
+			time = new Date().getTime();
+			log.info("Start export at    : " + (new Date().getTime() - time) + " ms");
+			String export = source.exportConversations();
+			log.info("Start parsing at   : " + (new Date().getTime() - time) + " ms");
+			JSONObject eportedJSON = (JSONObject) new JSONParser().parse(export);
+			log.info("Start import at    : " + (new Date().getTime() - time) + " ms");
+			importJSON(eportedJSON, target);
+			log.info("Finished import at : " + (new Date().getTime() - time) + " ms");
+		}
+	}
+
+	/**
+	 * 
+	 * @param obj
+	 * @throws IOException
+	 * @throws ClientProtocolException
+	 * @throws ParseException 
+	 */
+	private void importJSON(JSONObject export, I_ConversationTarget target) throws IOException, ClientProtocolException, ParseException {
+
+		Iterator<JSONObject> entryIterator = ((JSONArray) export.get("export")).iterator();
+		while (entryIterator.hasNext()) {
+
+			JSONObject expEntry = entryIterator.next();
+			if ("r".equals((String) expEntry.get("t"))) {
+				// the type of this entry is a room
+
+				String channelId = (String) expEntry.get("_id");
+				String channelName = (String) expEntry.get("name");
+				RocketEvent rocketEvent = null;
+				boolean newConversation = false;
+
+				JSONArray messages = (JSONArray) expEntry.get("messages");
+				Iterator<JSONObject> messageIterator = messages.iterator();
+
+				while (messageIterator.hasNext()) {
+					JSONObject message = messageIterator.next();
+					Object t = message.get("t");
+					if (t == null) {
+
+						rocketEvent = new RocketEvent((String) message.get("_id"));
+						rocketEvent.setBot(null);
+						rocketEvent.setToken(target.getToken());
+						rocketEvent.setChannelId(channelId);
+						rocketEvent.setChannelName(channelName);
+						rocketEvent.setText((String) message.get("msg"));
+
+						JSONObject user = (JSONObject) message.get("u");
+						rocketEvent.setUserId((String) user.get("_id"));
+						rocketEvent.setUserId((String) user.get("username"));
+
+						Date timestamp;
+						String _updatedAt;
+						Object updatedAt = message.get("_updatedAt");
+						if (updatedAt instanceof JSONObject) {
+							JSONObject jso = (JSONObject) updatedAt;
+							String dateAsString = (jso.get("$date")).toString();
+							timestamp = new Date(new Long(dateAsString));
+							_updatedAt = ISODateFormatter.format(timestamp);
+						} else {
+							_updatedAt = (String) updatedAt;
+							timestamp = ISODateFormatter.parse(_updatedAt);
+						}
+						rocketEvent.setTimestamp(timestamp);
+						rocketEvent.setCallbackUrl((String) message.get("webhook_url"));
+
+						rcEndpoint.onRocketEvent(target.getClientId(), rocketEvent);
+						log.info(rocketEvent.toString());
+						newConversation = true;
+					}
+				}
+				if (newConversation && rocketEvent != null) {
+					ResponseEntity<?> reE = rcEndpoint.getConversation(target.getClientId(), rocketEvent.getChannelId());
+					String conversationID = String.valueOf(reE.getBody());
+					conversationWebservice.complete(new ObjectId(conversationID));
+				}
+				newConversation = false;
+			}
+		}
+	}
+}

application/src/main/java/io/redlink/smarti/services/importer/A_ConversationSource.java

@@ -0,0 +1,47 @@
+package io.redlink.smarti.services.importer;
+
+import io.redlink.smarti.webservice.pojo.I_SourceConfiguration;
+
+/**
+ * Abstract implementation of a generic data source.<p>
+ * 
+ * @author Ruediger Kurz
+ */
+public abstract class A_ConversationSource implements I_ConversationSource {
+
+	/** Source type of this data source. */
+	private E_SourceType sourceType;
+
+	/** Configuration options for this data source. */
+	I_SourceConfiguration sourceConfiguration;
+
+	/**
+	 * Constructor with parameters.<p>
+	 * 
+	 * @param sourceType the source type
+	 * @param sourceConfiguration the configuration options
+	 */
+	public A_ConversationSource (E_SourceType sourceType, I_SourceConfiguration sourceConfiguration) {
+
+		this.sourceType = sourceType;
+		this.sourceConfiguration = sourceConfiguration;
+	}
+
+	/**
+	 * @see I_ConversationSource#getSourceType()
+	 */
+	@Override
+	public E_SourceType getSourceType() {
+
+		return this.sourceType;
+	}
+
+	/**
+	 * @see I_SourceConfiguration#getSourceConfiguration()
+	 */
+	@Override
+	public I_SourceConfiguration getSourceConfiguration() {
+
+		return this.sourceConfiguration;
+	}
+}

application/src/main/java/io/redlink/smarti/services/importer/A_ConversationTarget.java

@@ -0,0 +1,22 @@
+package io.redlink.smarti.services.importer;
+
+public abstract class A_ConversationTarget implements I_ConversationTarget {
+
+	private String clientId;
+	private String token;
+
+	public A_ConversationTarget (String clientId, String token)  {
+		this.clientId = clientId;
+		this.token = token;
+	}
+
+	@Override
+	public String getClientId() {
+		return clientId;
+	}
+
+	@Override
+	public String getToken() {
+		return token;
+	}
+}

application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatJSONFile.java

@@ -0,0 +1,42 @@
+package io.redlink.smarti.services.importer;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.nio.charset.Charset;
+
+import io.redlink.smarti.webservice.pojo.RocketFileConfig;
+
+public class ConversationSourceRocketChatJSONFile extends A_ConversationSource {
+
+	RocketFileConfig config;
+
+	public ConversationSourceRocketChatJSONFile(RocketFileConfig config) {
+		super(E_SourceType.RocketChatJSONFile, config);
+		this.config = config;
+	}
+
+	@Override
+	public String exportConversations() throws Exception {
+
+		return readJsonFromUrl(config.getJsonFileURL());
+	}
+
+	public static String readJsonFromUrl(String url) throws IOException {
+
+		InputStream is = new URL(url).openStream();
+		try {
+			BufferedReader rd = new BufferedReader(new InputStreamReader(is, Charset.forName("UTF-8")));
+			StringBuilder sb = new StringBuilder();
+			int cp;
+			while ((cp = rd.read()) != -1) {
+				sb.append((char) cp);
+			}
+			return sb.toString();
+		} finally {
+			is.close();
+		}
+	}
+}

application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatMongoDB.java

@@ -0,0 +1,60 @@
+package io.redlink.smarti.services.importer;
+
+import java.util.Arrays;
+
+import org.bson.Document;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoCursor;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.model.Aggregates;
+import com.mongodb.client.model.Filters;
+
+import io.redlink.smarti.services.I_ImporterService;
+import io.redlink.smarti.webservice.pojo.RocketMongoConfig;
+
+public class ConversationSourceRocketChatMongoDB extends A_ConversationSource {
+
+	RocketMongoConfig mongoConfig;
+
+	public ConversationSourceRocketChatMongoDB(RocketMongoConfig config) {
+
+		super(E_SourceType.RocketChatMongoDB, config);
+		mongoConfig = config;
+	}
+
+	@Override
+	public String exportConversations() throws Exception {
+
+		MongoClient mongoClient = new MongoClient(mongoConfig.getHost(), mongoConfig.getPort());
+
+		try {
+			MongoDatabase db = mongoClient.getDatabase(mongoConfig.getDbname());
+			MongoCollection<Document> coll = db.getCollection(mongoConfig.getRoomCollection());
+			// get all rooms of type 'request' and expertise 'x' joint with messages 
+			MongoCursor<Document> iterator = coll.aggregate(Arrays.asList(
+					Aggregates.match(Filters.eq("t", "r")),
+					Aggregates.match(Filters.regex(mongoConfig.getFilterField(), mongoConfig.getFilterValue())),
+					Aggregates.lookup(mongoConfig.getMessageCollection(), "_id", "rid", "messages"))).iterator();
+
+			StringBuffer input = new StringBuffer();
+			input.append(I_ImporterService.JSON_RESULT_WRAPPER_START);
+
+			while (iterator.hasNext()) {
+				Document doc = iterator.next();
+				String jsonDoc = doc.toJson();
+				input.append(jsonDoc);
+				System.out.println(jsonDoc);
+				if (iterator.hasNext()) {
+					input.append(", ");
+				}
+			}
+			input.append(I_ImporterService.JSON_RESULT_WRAPPER_END);
+
+			return input.toString();
+		} finally {
+			mongoClient.close();
+		}
+	}
+}