From dc9791444b2a3ce11f630629599257a2c7835358 Mon Sep 17 00:00:00 2001 From: Thomas Kurz Date: Wed, 26 Jul 2017 12:12:55 +0200 Subject: [PATCH 1/7] [maven-release-plugin] prepare release smarti-0.2.0-RC1 --- application/pom.xml | 2 +- core/pom.xml | 2 +- data/pom.xml | 2 +- data/solrcore-crawl-systel/pom.xml | 2 +- data/solrcore-wikipedia-de/pom.xml | 2 +- dist/pom.xml | 2 +- integration/rocket-chat/pom.xml | 2 +- lib/hasso-vocabulary-extractors/pom.xml | 2 +- lib/keyword-interestingterms/pom.xml | 2 +- lib/keyword-solr-mlt/pom.xml | 2 +- lib/ner/pom.xml | 2 +- lib/pom.xml | 2 +- lib/pos/pom.xml | 2 +- lib/query-conversation/pom.xml | 2 +- lib/query-dbsearch/pom.xml | 2 +- lib/token-processor/pom.xml | 2 +- pom.xml | 6 +++--- 17 files changed, 19 insertions(+), 19 deletions(-) diff --git a/application/pom.xml b/application/pom.xml index 20ca1440..b0204c70 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.0-RC1 smarti-application diff --git a/core/pom.xml b/core/pom.xml index 8d4ae2c8..3173a4c5 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.0-RC1 smarti-core diff --git a/data/pom.xml b/data/pom.xml index 905afbe2..ab19bc34 100644 --- a/data/pom.xml +++ b/data/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.0-RC1 data diff --git a/data/solrcore-crawl-systel/pom.xml b/data/solrcore-crawl-systel/pom.xml index d46d3853..7cd96775 100644 --- a/data/solrcore-crawl-systel/pom.xml +++ b/data/solrcore-crawl-systel/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.0-RC1 ../../ diff --git a/data/solrcore-wikipedia-de/pom.xml b/data/solrcore-wikipedia-de/pom.xml index 8958f303..02015569 100644 --- a/data/solrcore-wikipedia-de/pom.xml +++ b/data/solrcore-wikipedia-de/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.0-RC1 ../../ diff --git a/dist/pom.xml b/dist/pom.xml index f2b0ba6c..a274116d 100644 --- a/dist/pom.xml +++ b/dist/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.0-RC1 smarti-dist diff --git a/integration/rocket-chat/pom.xml b/integration/rocket-chat/pom.xml index 725c3265..f97ad0fb 100644 --- a/integration/rocket-chat/pom.xml +++ b/integration/rocket-chat/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.0-RC1 ../.. diff --git a/lib/hasso-vocabulary-extractors/pom.xml b/lib/hasso-vocabulary-extractors/pom.xml index 0d6e9152..4fe799d3 100644 --- a/lib/hasso-vocabulary-extractors/pom.xml +++ b/lib/hasso-vocabulary-extractors/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.0-RC1 ../../ diff --git a/lib/keyword-interestingterms/pom.xml b/lib/keyword-interestingterms/pom.xml index fed6d028..b626d6b0 100644 --- a/lib/keyword-interestingterms/pom.xml +++ b/lib/keyword-interestingterms/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.0-RC1 ../../ diff --git a/lib/keyword-solr-mlt/pom.xml b/lib/keyword-solr-mlt/pom.xml index 77659941..4d88ab5c 100644 --- a/lib/keyword-solr-mlt/pom.xml +++ b/lib/keyword-solr-mlt/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.0-RC1 ../../ diff --git a/lib/ner/pom.xml b/lib/ner/pom.xml index 260fab19..2d40c539 100644 --- a/lib/ner/pom.xml +++ b/lib/ner/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.0-RC1 ../../ diff --git a/lib/pom.xml b/lib/pom.xml index bc162e22..fe21c470 100644 --- a/lib/pom.xml +++ b/lib/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.0-RC1 lib diff --git a/lib/pos/pom.xml b/lib/pos/pom.xml index 62d9ff54..d3477871 100644 --- a/lib/pos/pom.xml +++ b/lib/pos/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.0-RC1 ../../ diff --git a/lib/query-conversation/pom.xml b/lib/query-conversation/pom.xml index df1b72cc..8b68d38a 100644 --- a/lib/query-conversation/pom.xml +++ b/lib/query-conversation/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.0-RC1 ../../ diff --git a/lib/query-dbsearch/pom.xml b/lib/query-dbsearch/pom.xml index 2ada6ba1..9b317da1 100644 --- a/lib/query-dbsearch/pom.xml +++ b/lib/query-dbsearch/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.0-RC1 ../../ diff --git a/lib/token-processor/pom.xml b/lib/token-processor/pom.xml index 6a10794c..b0b1cab8 100644 --- a/lib/token-processor/pom.xml +++ b/lib/token-processor/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.0-RC1 ../../ diff --git a/pom.xml b/pom.xml index 1591d798..d934c0ad 100644 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,7 @@ io.redlink.smarti smarti - 0.2.0-SNAPSHOT + 0.2.0-RC1 pom smarti @@ -109,7 +109,7 @@ scm:git:https://github.com/redlink-gmbh/smarti.git scm:git:git@github.com:redlink-gmbh/smarti.git https://github.com/redlink-gmbh/smarti/src - HEAD + smarti-0.2.0-RC1 github @@ -275,7 +275,7 @@ 1.8 - + From 66d49692a61bd2b1f346b7aa48f3f8ef849db86b Mon Sep 17 00:00:00 2001 From: Thomas Kurz Date: Wed, 26 Jul 2017 12:12:56 +0200 Subject: [PATCH 2/7] [maven-release-plugin] prepare for next development iteration --- application/pom.xml | 2 +- core/pom.xml | 2 +- data/pom.xml | 2 +- data/solrcore-crawl-systel/pom.xml | 2 +- data/solrcore-wikipedia-de/pom.xml | 2 +- dist/pom.xml | 2 +- integration/rocket-chat/pom.xml | 2 +- lib/hasso-vocabulary-extractors/pom.xml | 2 +- lib/keyword-interestingterms/pom.xml | 2 +- lib/keyword-solr-mlt/pom.xml | 2 +- lib/ner/pom.xml | 2 +- lib/pom.xml | 2 +- lib/pos/pom.xml | 2 +- lib/query-conversation/pom.xml | 2 +- lib/query-dbsearch/pom.xml | 2 +- lib/token-processor/pom.xml | 2 +- pom.xml | 4 ++-- 17 files changed, 18 insertions(+), 18 deletions(-) diff --git a/application/pom.xml b/application/pom.xml index b0204c70..1a26d7e5 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC1 + 0.2.0-RC2-SNAPSHOT smarti-application diff --git a/core/pom.xml b/core/pom.xml index 3173a4c5..478253df 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC1 + 0.2.0-RC2-SNAPSHOT smarti-core diff --git a/data/pom.xml b/data/pom.xml index ab19bc34..3517c066 100644 --- a/data/pom.xml +++ b/data/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC1 + 0.2.0-RC2-SNAPSHOT data diff --git a/data/solrcore-crawl-systel/pom.xml b/data/solrcore-crawl-systel/pom.xml index 7cd96775..ee63242c 100644 --- a/data/solrcore-crawl-systel/pom.xml +++ b/data/solrcore-crawl-systel/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC1 + 0.2.0-RC2-SNAPSHOT ../../ diff --git a/data/solrcore-wikipedia-de/pom.xml b/data/solrcore-wikipedia-de/pom.xml index 02015569..31e14086 100644 --- a/data/solrcore-wikipedia-de/pom.xml +++ b/data/solrcore-wikipedia-de/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC1 + 0.2.0-RC2-SNAPSHOT ../../ diff --git a/dist/pom.xml b/dist/pom.xml index a274116d..7e761c71 100644 --- a/dist/pom.xml +++ b/dist/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC1 + 0.2.0-RC2-SNAPSHOT smarti-dist diff --git a/integration/rocket-chat/pom.xml b/integration/rocket-chat/pom.xml index f97ad0fb..a169bec6 100644 --- a/integration/rocket-chat/pom.xml +++ b/integration/rocket-chat/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC1 + 0.2.0-RC2-SNAPSHOT ../.. diff --git a/lib/hasso-vocabulary-extractors/pom.xml b/lib/hasso-vocabulary-extractors/pom.xml index 4fe799d3..0f30b0ef 100644 --- a/lib/hasso-vocabulary-extractors/pom.xml +++ b/lib/hasso-vocabulary-extractors/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC1 + 0.2.0-RC2-SNAPSHOT ../../ diff --git a/lib/keyword-interestingterms/pom.xml b/lib/keyword-interestingterms/pom.xml index b626d6b0..dbe1fab5 100644 --- a/lib/keyword-interestingterms/pom.xml +++ b/lib/keyword-interestingterms/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC1 + 0.2.0-RC2-SNAPSHOT ../../ diff --git a/lib/keyword-solr-mlt/pom.xml b/lib/keyword-solr-mlt/pom.xml index 4d88ab5c..04028fed 100644 --- a/lib/keyword-solr-mlt/pom.xml +++ b/lib/keyword-solr-mlt/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC1 + 0.2.0-RC2-SNAPSHOT ../../ diff --git a/lib/ner/pom.xml b/lib/ner/pom.xml index 2d40c539..edc13f74 100644 --- a/lib/ner/pom.xml +++ b/lib/ner/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC1 + 0.2.0-RC2-SNAPSHOT ../../ diff --git a/lib/pom.xml b/lib/pom.xml index fe21c470..e05fb092 100644 --- a/lib/pom.xml +++ b/lib/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC1 + 0.2.0-RC2-SNAPSHOT lib diff --git a/lib/pos/pom.xml b/lib/pos/pom.xml index d3477871..a361ef9c 100644 --- a/lib/pos/pom.xml +++ b/lib/pos/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC1 + 0.2.0-RC2-SNAPSHOT ../../ diff --git a/lib/query-conversation/pom.xml b/lib/query-conversation/pom.xml index 8b68d38a..628a9fc7 100644 --- a/lib/query-conversation/pom.xml +++ b/lib/query-conversation/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC1 + 0.2.0-RC2-SNAPSHOT ../../ diff --git a/lib/query-dbsearch/pom.xml b/lib/query-dbsearch/pom.xml index 9b317da1..f013d7d8 100644 --- a/lib/query-dbsearch/pom.xml +++ b/lib/query-dbsearch/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC1 + 0.2.0-RC2-SNAPSHOT ../../ diff --git a/lib/token-processor/pom.xml b/lib/token-processor/pom.xml index b0b1cab8..dbfca2fd 100644 --- a/lib/token-processor/pom.xml +++ b/lib/token-processor/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC1 + 0.2.0-RC2-SNAPSHOT ../../ diff --git a/pom.xml b/pom.xml index d934c0ad..b96de726 100644 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,7 @@ io.redlink.smarti smarti - 0.2.0-RC1 + 0.2.0-RC2-SNAPSHOT pom smarti @@ -109,7 +109,7 @@ scm:git:https://github.com/redlink-gmbh/smarti.git scm:git:git@github.com:redlink-gmbh/smarti.git https://github.com/redlink-gmbh/smarti/src - smarti-0.2.0-RC1 + HEAD github From 81036722aef29309fea87e26a3b8028456b2a5f4 Mon Sep 17 00:00:00 2001 From: Thomas Kurz Date: Fri, 28 Jul 2017 16:40:43 +0200 Subject: [PATCH 3/7] [maven-release-plugin] prepare release smarti-0.2.0-RC2 --- application/pom.xml | 2 +- core/pom.xml | 2 +- data/pom.xml | 2 +- data/solrcore-crawl-systel/pom.xml | 2 +- data/solrcore-wikipedia-de/pom.xml | 2 +- dist/pom.xml | 2 +- integration/rocket-chat/pom.xml | 2 +- lib/hasso-vocabulary-extractors/pom.xml | 2 +- lib/keyword-interestingterms/pom.xml | 2 +- lib/keyword-solr-mlt/pom.xml | 2 +- lib/ner/pom.xml | 2 +- lib/pom.xml | 2 +- lib/pos/pom.xml | 2 +- lib/query-conversation/pom.xml | 2 +- lib/query-dbsearch/pom.xml | 2 +- lib/token-processor/pom.xml | 2 +- pom.xml | 4 ++-- 17 files changed, 18 insertions(+), 18 deletions(-) diff --git a/application/pom.xml b/application/pom.xml index 1a26d7e5..271489dd 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2-SNAPSHOT + 0.2.0-RC2 smarti-application diff --git a/core/pom.xml b/core/pom.xml index 478253df..3aa36472 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2-SNAPSHOT + 0.2.0-RC2 smarti-core diff --git a/data/pom.xml b/data/pom.xml index 3517c066..b8c75c14 100644 --- a/data/pom.xml +++ b/data/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2-SNAPSHOT + 0.2.0-RC2 data diff --git a/data/solrcore-crawl-systel/pom.xml b/data/solrcore-crawl-systel/pom.xml index ee63242c..65e5a75a 100644 --- a/data/solrcore-crawl-systel/pom.xml +++ b/data/solrcore-crawl-systel/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2-SNAPSHOT + 0.2.0-RC2 ../../ diff --git a/data/solrcore-wikipedia-de/pom.xml b/data/solrcore-wikipedia-de/pom.xml index 31e14086..cee20ae7 100644 --- a/data/solrcore-wikipedia-de/pom.xml +++ b/data/solrcore-wikipedia-de/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2-SNAPSHOT + 0.2.0-RC2 ../../ diff --git a/dist/pom.xml b/dist/pom.xml index 7e761c71..0476e0d3 100644 --- a/dist/pom.xml +++ b/dist/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2-SNAPSHOT + 0.2.0-RC2 smarti-dist diff --git a/integration/rocket-chat/pom.xml b/integration/rocket-chat/pom.xml index a169bec6..b38fde55 100644 --- a/integration/rocket-chat/pom.xml +++ b/integration/rocket-chat/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2-SNAPSHOT + 0.2.0-RC2 ../.. diff --git a/lib/hasso-vocabulary-extractors/pom.xml b/lib/hasso-vocabulary-extractors/pom.xml index 0f30b0ef..77178fac 100644 --- a/lib/hasso-vocabulary-extractors/pom.xml +++ b/lib/hasso-vocabulary-extractors/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2-SNAPSHOT + 0.2.0-RC2 ../../ diff --git a/lib/keyword-interestingterms/pom.xml b/lib/keyword-interestingterms/pom.xml index dbe1fab5..43b11768 100644 --- a/lib/keyword-interestingterms/pom.xml +++ b/lib/keyword-interestingterms/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2-SNAPSHOT + 0.2.0-RC2 ../../ diff --git a/lib/keyword-solr-mlt/pom.xml b/lib/keyword-solr-mlt/pom.xml index 04028fed..2a4f806b 100644 --- a/lib/keyword-solr-mlt/pom.xml +++ b/lib/keyword-solr-mlt/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2-SNAPSHOT + 0.2.0-RC2 ../../ diff --git a/lib/ner/pom.xml b/lib/ner/pom.xml index edc13f74..16daabe0 100644 --- a/lib/ner/pom.xml +++ b/lib/ner/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2-SNAPSHOT + 0.2.0-RC2 ../../ diff --git a/lib/pom.xml b/lib/pom.xml index e05fb092..35cfbd5f 100644 --- a/lib/pom.xml +++ b/lib/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2-SNAPSHOT + 0.2.0-RC2 lib diff --git a/lib/pos/pom.xml b/lib/pos/pom.xml index a361ef9c..9ab7e549 100644 --- a/lib/pos/pom.xml +++ b/lib/pos/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2-SNAPSHOT + 0.2.0-RC2 ../../ diff --git a/lib/query-conversation/pom.xml b/lib/query-conversation/pom.xml index 628a9fc7..1cc50be8 100644 --- a/lib/query-conversation/pom.xml +++ b/lib/query-conversation/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2-SNAPSHOT + 0.2.0-RC2 ../../ diff --git a/lib/query-dbsearch/pom.xml b/lib/query-dbsearch/pom.xml index f013d7d8..2908b351 100644 --- a/lib/query-dbsearch/pom.xml +++ b/lib/query-dbsearch/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2-SNAPSHOT + 0.2.0-RC2 ../../ diff --git a/lib/token-processor/pom.xml b/lib/token-processor/pom.xml index dbfca2fd..33c570cf 100644 --- a/lib/token-processor/pom.xml +++ b/lib/token-processor/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2-SNAPSHOT + 0.2.0-RC2 ../../ diff --git a/pom.xml b/pom.xml index b96de726..85d561dc 100644 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,7 @@ io.redlink.smarti smarti - 0.2.0-RC2-SNAPSHOT + 0.2.0-RC2 pom smarti @@ -109,7 +109,7 @@ scm:git:https://github.com/redlink-gmbh/smarti.git scm:git:git@github.com:redlink-gmbh/smarti.git https://github.com/redlink-gmbh/smarti/src - HEAD + smarti-0.2.0-RC2 github From 1a48dec542f678349e3007817baa85768f4a5f95 Mon Sep 17 00:00:00 2001 From: ruKurz Date: Fri, 4 Aug 2017 15:50:05 +0200 Subject: [PATCH 4/7] #36 - Add a web service that provides conversation imports - initial check-in --- .gitignore | 1 + application/pom.xml | 8 + .../smarti/services/I_ImporterService.java | 18 +++ .../smarti/services/ImporterService.java | 150 ++++++++++++++++++ .../importer/A_ConversationSource.java | 47 ++++++ .../importer/A_ConversationTarget.java | 22 +++ .../ConversationSourceRocketChatJSONFile.java | 42 +++++ .../ConversationSourceRocketChatMongoDB.java | 60 +++++++ ...onversationSourceRocketChatWebservice.java | 79 +++++++++ .../importer/ConversationTargetSmarti.java | 11 ++ .../importer/I_ConversationSource.java | 40 +++++ .../importer/I_ConversationTarget.java | 7 + .../webservice/ImporterServiceEndpoint.java | 129 +++++++++++++++ .../smarti/webservice/RocketChatEndpoint.java | 5 + .../pojo/A_SourceConfiguration.java | 33 ++++ .../pojo/I_SourceConfiguration.java | 9 ++ .../webservice/pojo/RocketFileConfig.java | 27 ++++ .../webservice/pojo/RocketMongoConfig.java | 103 ++++++++++++ .../pojo/RocketWebserviceConfig.java | 76 +++++++++ .../webservice/ConversationImporterTest.java | 124 +++++++++++++++ .../RocketChatEndpointProxyTest.java | 2 - 21 files changed, 991 insertions(+), 2 deletions(-) create mode 100644 application/src/main/java/io/redlink/smarti/services/I_ImporterService.java create mode 100644 application/src/main/java/io/redlink/smarti/services/ImporterService.java create mode 100644 application/src/main/java/io/redlink/smarti/services/importer/A_ConversationSource.java create mode 100644 application/src/main/java/io/redlink/smarti/services/importer/A_ConversationTarget.java create mode 100644 application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatJSONFile.java create mode 100644 application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatMongoDB.java create mode 100644 application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatWebservice.java create mode 100644 application/src/main/java/io/redlink/smarti/services/importer/ConversationTargetSmarti.java create mode 100644 application/src/main/java/io/redlink/smarti/services/importer/I_ConversationSource.java create mode 100644 application/src/main/java/io/redlink/smarti/services/importer/I_ConversationTarget.java create mode 100644 application/src/main/java/io/redlink/smarti/webservice/ImporterServiceEndpoint.java create mode 100644 application/src/main/java/io/redlink/smarti/webservice/pojo/A_SourceConfiguration.java create mode 100644 application/src/main/java/io/redlink/smarti/webservice/pojo/I_SourceConfiguration.java create mode 100644 application/src/main/java/io/redlink/smarti/webservice/pojo/RocketFileConfig.java create mode 100644 application/src/main/java/io/redlink/smarti/webservice/pojo/RocketMongoConfig.java create mode 100644 application/src/main/java/io/redlink/smarti/webservice/pojo/RocketWebserviceConfig.java create mode 100644 application/src/main/test/io/redlink/smarti/webservice/ConversationImporterTest.java diff --git a/.gitignore b/.gitignore index f4835cc7..f7f4e84b 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ .externalToolBuilders/ .classpath .factorypath +bin/ # Maven target/ diff --git a/application/pom.xml b/application/pom.xml index 271489dd..ef1b1fd0 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -66,6 +66,14 @@ com.google.guava guava + + com.googlecode.json-simple + json-simple + + + org.mongodb + mongo-java-driver + commons-io commons-io diff --git a/application/src/main/java/io/redlink/smarti/services/I_ImporterService.java b/application/src/main/java/io/redlink/smarti/services/I_ImporterService.java new file mode 100644 index 00000000..4b693de9 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/I_ImporterService.java @@ -0,0 +1,18 @@ +package io.redlink.smarti.services; + +import io.redlink.smarti.services.importer.I_ConversationSource; +import io.redlink.smarti.services.importer.I_ConversationTarget; + +public interface I_ImporterService { + + static final String SMARTI_IMPORT_WEBSERVICE = "import"; + + /** The postfix to convert the Rocket.Chat query result to a valid JSON. */ + static final String JSON_RESULT_WRAPPER_END = "]}"; + + /** The prefix to convert the Rocket.Chat query result to a valid JSON. */ + static final String JSON_RESULT_WRAPPER_START = "{\"export\": ["; + + void importComversation(I_ConversationSource source, I_ConversationTarget target) throws Exception; + +} diff --git a/application/src/main/java/io/redlink/smarti/services/ImporterService.java b/application/src/main/java/io/redlink/smarti/services/ImporterService.java new file mode 100644 index 00000000..1810dbfe --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/ImporterService.java @@ -0,0 +1,150 @@ +package io.redlink.smarti.services; + +import static org.hamcrest.CoreMatchers.instanceOf; + +import java.io.IOException; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Iterator; + +import org.apache.http.client.ClientProtocolException; +import org.bson.types.ObjectId; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import io.redlink.smarti.services.importer.I_ConversationSource; +import io.redlink.smarti.services.importer.I_ConversationTarget; +import io.redlink.smarti.webservice.ConversationWebservice; +import io.redlink.smarti.webservice.RocketChatEndpoint; +import io.redlink.smarti.webservice.pojo.RocketEvent; + +/** + * This service provides an Smarti importer.

+ * + * Sources for import data to Smarti are: + *

  • Directly from Rocket.Chat MongoDB
    • + *
  • A Rocket.Chat MongoDB Export
    • + *
  • An E-Mailbox
    • + *
  • A FAQ website
    • + * + * @author Ruediger Kurz + */ +@Service +public class ImporterService implements I_ImporterService { + + private Logger log = LoggerFactory.getLogger(ImporterService.class); + + /** A simple ISO date formatter. */ + private static final DateFormat ISODateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + + @Autowired + private ConversationWebservice conversationWebservice; + + @Autowired + private RocketChatEndpoint rcEndpoint; + + public ImporterService() { + rcEndpoint = new RocketChatEndpoint(null, 0, null); + } + + long time; + + /** + * @see I_ImporterService#importComversation(I_ConversationSource, I_ConversationTarget) + */ + @Override + public void importComversation(I_ConversationSource source, I_ConversationTarget target) throws Exception { + + + if (source != null && target != null) { + time = new Date().getTime(); + log.info("Start export at : " + (new Date().getTime() - time) + " ms"); + String export = source.exportConversations(); + log.info("Start parsing at : " + (new Date().getTime() - time) + " ms"); + JSONObject eportedJSON = (JSONObject) new JSONParser().parse(export); + log.info("Start import at : " + (new Date().getTime() - time) + " ms"); + importJSON(eportedJSON, target); + log.info("Finished import at : " + (new Date().getTime() - time) + " ms"); + } + } + + /** + * + * @param obj + * @throws IOException + * @throws ClientProtocolException + * @throws ParseException + */ + private void importJSON(JSONObject export, I_ConversationTarget target) throws IOException, ClientProtocolException, ParseException { + + Iterator entryIterator = ((JSONArray) export.get("export")).iterator(); + while (entryIterator.hasNext()) { + + JSONObject expEntry = entryIterator.next(); + if ("r".equals((String) expEntry.get("t"))) { + // the type of this entry is a room + + String channelId = (String) expEntry.get("_id"); + String channelName = (String) expEntry.get("name"); + RocketEvent rocketEvent = null; + boolean newConversation = false; + + JSONArray messages = (JSONArray) expEntry.get("messages"); + Iterator messageIterator = messages.iterator(); + + while (messageIterator.hasNext()) { + JSONObject message = messageIterator.next(); + Object t = message.get("t"); + if (t == null) { + + rocketEvent = new RocketEvent((String) message.get("_id")); + rocketEvent.setBot(null); + rocketEvent.setToken(target.getToken()); + rocketEvent.setChannelId(channelId); + rocketEvent.setChannelName(channelName); + rocketEvent.setText((String) message.get("msg")); + + JSONObject user = (JSONObject) message.get("u"); + rocketEvent.setUserId((String) user.get("_id")); + rocketEvent.setUserId((String) user.get("username")); + + Date timestamp; + String _updatedAt; + Object updatedAt = message.get("_updatedAt"); + if (updatedAt instanceof JSONObject) { + JSONObject jso = (JSONObject) updatedAt; + String dateAsString = (jso.get("$date")).toString(); + timestamp = new Date(new Long(dateAsString)); + _updatedAt = ISODateFormatter.format(timestamp); + } else { + _updatedAt = (String) updatedAt; + timestamp = ISODateFormatter.parse(_updatedAt); + } + rocketEvent.setTimestamp(timestamp); + rocketEvent.setCallbackUrl((String) message.get("webhook_url")); + + rcEndpoint.onRocketEvent(target.getClientId(), rocketEvent); + log.info(rocketEvent.toString()); + newConversation = true; + } + } + if (newConversation && rocketEvent != null) { + ResponseEntity reE = rcEndpoint.getConversation(target.getClientId(), rocketEvent.getChannelId()); + String conversationID = String.valueOf(reE.getBody()); + conversationWebservice.complete(new ObjectId(conversationID)); + } + newConversation = false; + } + } + } +} diff --git a/application/src/main/java/io/redlink/smarti/services/importer/A_ConversationSource.java b/application/src/main/java/io/redlink/smarti/services/importer/A_ConversationSource.java new file mode 100644 index 00000000..86482cd2 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/importer/A_ConversationSource.java @@ -0,0 +1,47 @@ +package io.redlink.smarti.services.importer; + +import io.redlink.smarti.webservice.pojo.I_SourceConfiguration; + +/** + * Abstract implementation of a generic data source.

    + * + * @author Ruediger Kurz + */ +public abstract class A_ConversationSource implements I_ConversationSource { + + /** Source type of this data source. */ + private E_SourceType sourceType; + + /** Configuration options for this data source. */ + I_SourceConfiguration sourceConfiguration; + + /** + * Constructor with parameters.

    + * + * @param sourceType the source type + * @param sourceConfiguration the configuration options + */ + public A_ConversationSource (E_SourceType sourceType, I_SourceConfiguration sourceConfiguration) { + + this.sourceType = sourceType; + this.sourceConfiguration = sourceConfiguration; + } + + /** + * @see I_ConversationSource#getSourceType() + */ + @Override + public E_SourceType getSourceType() { + + return this.sourceType; + } + + /** + * @see I_SourceConfiguration#getSourceConfiguration() + */ + @Override + public I_SourceConfiguration getSourceConfiguration() { + + return this.sourceConfiguration; + } +} diff --git a/application/src/main/java/io/redlink/smarti/services/importer/A_ConversationTarget.java b/application/src/main/java/io/redlink/smarti/services/importer/A_ConversationTarget.java new file mode 100644 index 00000000..297080b5 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/importer/A_ConversationTarget.java @@ -0,0 +1,22 @@ +package io.redlink.smarti.services.importer; + +public abstract class A_ConversationTarget implements I_ConversationTarget { + + private String clientId; + private String token; + + public A_ConversationTarget (String clientId, String token) { + this.clientId = clientId; + this.token = token; + } + + @Override + public String getClientId() { + return clientId; + } + + @Override + public String getToken() { + return token; + } +} diff --git a/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatJSONFile.java b/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatJSONFile.java new file mode 100644 index 00000000..153dc8f3 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatJSONFile.java @@ -0,0 +1,42 @@ +package io.redlink.smarti.services.importer; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URL; +import java.nio.charset.Charset; + +import io.redlink.smarti.webservice.pojo.RocketFileConfig; + +public class ConversationSourceRocketChatJSONFile extends A_ConversationSource { + + RocketFileConfig config; + + public ConversationSourceRocketChatJSONFile(RocketFileConfig config) { + super(E_SourceType.RocketChatJSONFile, config); + this.config = config; + } + + @Override + public String exportConversations() throws Exception { + + return readJsonFromUrl(config.getJsonFileURL()); + } + + public static String readJsonFromUrl(String url) throws IOException { + + InputStream is = new URL(url).openStream(); + try { + BufferedReader rd = new BufferedReader(new InputStreamReader(is, Charset.forName("UTF-8"))); + StringBuilder sb = new StringBuilder(); + int cp; + while ((cp = rd.read()) != -1) { + sb.append((char) cp); + } + return sb.toString(); + } finally { + is.close(); + } + } +} diff --git a/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatMongoDB.java b/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatMongoDB.java new file mode 100644 index 00000000..8d4743f2 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatMongoDB.java @@ -0,0 +1,60 @@ +package io.redlink.smarti.services.importer; + +import java.util.Arrays; + +import org.bson.Document; + +import com.mongodb.MongoClient; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoCursor; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.Aggregates; +import com.mongodb.client.model.Filters; + +import io.redlink.smarti.services.I_ImporterService; +import io.redlink.smarti.webservice.pojo.RocketMongoConfig; + +public class ConversationSourceRocketChatMongoDB extends A_ConversationSource { + + RocketMongoConfig mongoConfig; + + public ConversationSourceRocketChatMongoDB(RocketMongoConfig config) { + + super(E_SourceType.RocketChatMongoDB, config); + mongoConfig = config; + } + + @Override + public String exportConversations() throws Exception { + + MongoClient mongoClient = new MongoClient(mongoConfig.getHost(), mongoConfig.getPort()); + + try { + MongoDatabase db = mongoClient.getDatabase(mongoConfig.getDbname()); + MongoCollection coll = db.getCollection(mongoConfig.getRoomCollection()); + // get all rooms of type 'request' and expertise 'x' joint with messages + MongoCursor iterator = coll.aggregate(Arrays.asList( + Aggregates.match(Filters.eq("t", "r")), + Aggregates.match(Filters.regex(mongoConfig.getFilterField(), mongoConfig.getFilterValue())), + Aggregates.lookup(mongoConfig.getMessageCollection(), "_id", "rid", "messages"))).iterator(); + + StringBuffer input = new StringBuffer(); + input.append(I_ImporterService.JSON_RESULT_WRAPPER_START); + + while (iterator.hasNext()) { + Document doc = iterator.next(); + String jsonDoc = doc.toJson(); + input.append(jsonDoc); + System.out.println(jsonDoc); + if (iterator.hasNext()) { + input.append(", "); + } + } + input.append(I_ImporterService.JSON_RESULT_WRAPPER_END); + + return input.toString(); + } finally { + mongoClient.close(); + } + } +} diff --git a/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatWebservice.java b/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatWebservice.java new file mode 100644 index 00000000..179f9992 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatWebservice.java @@ -0,0 +1,79 @@ +package io.redlink.smarti.services.importer; + +import java.io.IOException; +import java.nio.charset.Charset; + +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpResponse; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.ConnectionBackoffStrategy; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.HttpClientBuilder; + +import io.redlink.smarti.webservice.pojo.RocketWebserviceConfig; + +/** + * Implements the Rocket.Chat MongoDB as data source for Smarti + * + * @author Ruediger Kurz + */ +public class ConversationSourceRocketChatWebservice extends A_ConversationSource { + + RocketWebserviceConfig webserviceConfig; + + public ConversationSourceRocketChatWebservice(RocketWebserviceConfig config) { + + super(E_SourceType.RocketChatWebServie, config); + webserviceConfig = config; + } + + @Override + public String exportConversations() throws Exception { + + return getMongoRequestExport(); + } + + /** + * + * @param payload + * @return + */ + private String getMongoRequestExport() throws IOException, ClientProtocolException { + + HttpPost post = new HttpPost(webserviceConfig.getRocketChatEndpoint()); + post.addHeader("Content-Type", "application/json; charset=UTF-8"); + post.addHeader("Accept-Encoding", "gzip,deflate,sdch"); + post.addHeader("Accept-Language", "en-US,en;q=0.8"); + + StringEntity entity = new StringEntity(getSourceConfiguration().asJSON().toJSONString(), "UTF-8"); + + entity.setContentType("application/json"); + post.setEntity(entity); + + try (CloseableHttpResponse response = httpClientBuilder.build().execute(post)) { + if (200 == response.getStatusLine().getStatusCode()) { + return IOUtils.toString(response.getEntity().getContent(), + Charset.defaultCharset()); + } + } + return null; + } + + + /** A HTTP client to use the Smarti web service. */ + private final HttpClientBuilder httpClientBuilder = HttpClientBuilder.create() + .setRetryHandler((exception, executionCount, context) -> executionCount < 3) + .setConnectionBackoffStrategy(new ConnectionBackoffStrategy() { + @Override + public boolean shouldBackoff(HttpResponse resp) { + return false; + } + + @Override + public boolean shouldBackoff(Throwable t) { + return t instanceof IOException; + } + }).setUserAgent("Smarti/0.0 Rocket.Chat-Endpoint/0.1"); +} diff --git a/application/src/main/java/io/redlink/smarti/services/importer/ConversationTargetSmarti.java b/application/src/main/java/io/redlink/smarti/services/importer/ConversationTargetSmarti.java new file mode 100644 index 00000000..9a20cb7d --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/importer/ConversationTargetSmarti.java @@ -0,0 +1,11 @@ +package io.redlink.smarti.services.importer; + +public class ConversationTargetSmarti extends A_ConversationTarget { + + public ConversationTargetSmarti(String clientId, String token) { + + super(clientId, token); + } + + +} diff --git a/application/src/main/java/io/redlink/smarti/services/importer/I_ConversationSource.java b/application/src/main/java/io/redlink/smarti/services/importer/I_ConversationSource.java new file mode 100644 index 00000000..3aa0db42 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/importer/I_ConversationSource.java @@ -0,0 +1,40 @@ +package io.redlink.smarti.services.importer; + +import io.redlink.smarti.webservice.pojo.I_SourceConfiguration; + +/** + * Interface for potential conversational data sources.

    + * + * @author Ruediger Kurz + * + */ +public interface I_ConversationSource { + + /** The import source type. */ + static enum E_SourceType { + RocketChatJSONFile, RocketChatMongoDB, RocketChatWebServie, Mail, FAQs + } + + /** + * Returns the type of source.

    + * + * @return the type of source + */ + E_SourceType getSourceType(); + + /** + * Returns the source specific configuration options.

    + * + * @return the configuration options + */ + I_SourceConfiguration getSourceConfiguration(); + + /** + * Executes the export action and returns the export result as JSON.

    + * + * @return the export as JSON + * + * @throws Exception if something goes wrong + */ + String exportConversations() throws Exception; +} diff --git a/application/src/main/java/io/redlink/smarti/services/importer/I_ConversationTarget.java b/application/src/main/java/io/redlink/smarti/services/importer/I_ConversationTarget.java new file mode 100644 index 00000000..48516143 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/importer/I_ConversationTarget.java @@ -0,0 +1,7 @@ +package io.redlink.smarti.services.importer; + +public interface I_ConversationTarget { + + String getClientId(); + String getToken(); +} diff --git a/application/src/main/java/io/redlink/smarti/webservice/ImporterServiceEndpoint.java b/application/src/main/java/io/redlink/smarti/webservice/ImporterServiceEndpoint.java new file mode 100644 index 00000000..37a92124 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/webservice/ImporterServiceEndpoint.java @@ -0,0 +1,129 @@ +package io.redlink.smarti.webservice; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.ResponseEntity; +import org.springframework.util.MimeTypeUtils; +import org.springframework.web.bind.annotation.CrossOrigin; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import io.redlink.smarti.services.I_ImporterService; +import io.redlink.smarti.services.ImporterService; +import io.redlink.smarti.services.importer.ConversationSourceRocketChatJSONFile; +import io.redlink.smarti.services.importer.ConversationSourceRocketChatMongoDB; +import io.redlink.smarti.services.importer.ConversationSourceRocketChatWebservice; +import io.redlink.smarti.services.importer.I_ConversationSource; +import io.redlink.smarti.services.importer.I_ConversationTarget; +import io.redlink.smarti.webservice.pojo.RocketMongoConfig; +import io.redlink.smarti.webservice.pojo.RocketFileConfig; +import io.redlink.smarti.webservice.pojo.RocketWebserviceConfig; +import io.redlink.smarti.services.importer.ConversationTargetSmarti; +import io.redlink.smarti.services.importer.I_ConversationSource.E_SourceType; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; + +@CrossOrigin +@RestController +@RequestMapping(value = I_ImporterService.SMARTI_IMPORT_WEBSERVICE, + consumes = MimeTypeUtils.APPLICATION_JSON_VALUE, + produces = MimeTypeUtils.APPLICATION_JSON_VALUE) +@Api(I_ImporterService.SMARTI_IMPORT_WEBSERVICE) +public class ImporterServiceEndpoint { + + @Autowired + private ImporterService importerService; + + private Logger log = LoggerFactory.getLogger(ImporterServiceEndpoint.class); + + /** + * This web service should be part of Rocket.Chat not of Smarti. + * + * @param payload + * + * @return the exported chat messages of help request channels + */ + @ApiOperation(value = "imports the conversation from the given source", produces=MimeTypeUtils.APPLICATION_JSON_VALUE) + @RequestMapping(value = "{clientId}/{sourceType}", method = RequestMethod.GET, + produces=MimeTypeUtils.APPLICATION_JSON_VALUE, consumes=MimeTypeUtils.APPLICATION_JSON_VALUE) + public ResponseEntity getImport( + @PathVariable(value="clientId") String clientId, + @PathVariable(value="sourceType") String sourceType, + @RequestBody String payload) { + + ObjectMapper mapper = new ObjectMapper(); + + try { + E_SourceType type = E_SourceType.valueOf(sourceType); + switch (type) { + case RocketChatJSONFile: + importRocketChatJSONFile(clientId, mapper.readValue(payload, RocketFileConfig.class)); + break; + case RocketChatMongoDB: + importRocketChatMongoDB(clientId, mapper.readValue(payload, RocketMongoConfig.class)); + break; + case RocketChatWebServie: + importRocketChatWebservice(clientId, mapper.readValue(payload, RocketWebserviceConfig.class)); + break; + default: + return ResponseEntity.ok("{status: nosource}"); + } + return ResponseEntity.ok("{status: sucess}"); + } catch (Exception e) { + return ResponseEntity.status(500).build(); + } + } + + @ApiOperation(value = "imports the conversation from the given source", produces=MimeTypeUtils.APPLICATION_JSON_VALUE) + @RequestMapping(value = "{clientId}/rocketChatJSONFile", method = RequestMethod.GET, + produces=MimeTypeUtils.APPLICATION_JSON_VALUE, consumes=MimeTypeUtils.APPLICATION_JSON_VALUE) + public ResponseEntity importRocketChatJSONFile( + @PathVariable(value="clientId") String clientId, + @RequestBody RocketFileConfig fileConfig) { + + log.warn("called import webservice: '/import/" + clientId + "/rocketChatJSONFile' with config: " + fileConfig.toString()); + return runImport (new ConversationSourceRocketChatJSONFile(fileConfig), clientId); + } + + @ApiOperation(value = "imports the conversation from the given source", produces=MimeTypeUtils.APPLICATION_JSON_VALUE) + @RequestMapping(value = "{clientId}/rocketChatMongoDB", method = RequestMethod.GET, + produces=MimeTypeUtils.APPLICATION_JSON_VALUE, consumes=MimeTypeUtils.APPLICATION_JSON_VALUE) + public ResponseEntity importRocketChatMongoDB( + @PathVariable(value="clientId") String clientId, + @RequestBody RocketMongoConfig mongoConfig) { + + log.warn("called import webservice: '/import/" + clientId + "/rocketChatJSONFile' with config: " + mongoConfig.toString()); + return runImport (new ConversationSourceRocketChatMongoDB(mongoConfig), clientId); + } + + @ApiOperation(value = "imports the conversation from the given source", produces=MimeTypeUtils.APPLICATION_JSON_VALUE) + @RequestMapping(value = "{clientId}/rocketChatWebservice", method = RequestMethod.GET, + produces=MimeTypeUtils.APPLICATION_JSON_VALUE, consumes=MimeTypeUtils.APPLICATION_JSON_VALUE) + public ResponseEntity importRocketChatWebservice( + @PathVariable(value="clientId") String clientId, + @RequestBody RocketWebserviceConfig webserviceConfig) { + + log.warn("called import webservice: '/import/" + clientId + "/rocketChatJSONFile' with config: " + webserviceConfig.toString()); + return runImport (new ConversationSourceRocketChatWebservice(webserviceConfig), clientId); + } + + private ResponseEntity runImport (I_ConversationSource source, String clientId) { + // TODO: Token not yet implemented, replace key123. + I_ConversationTarget target = new ConversationTargetSmarti(clientId, "key123"); + try { + importerService.importComversation(source, target); + return ResponseEntity.ok("{status: sucess}"); + } catch (Exception e) { + log.error("Error with message: \"{}\" occured during import: ", e.getMessage(), e); + return ResponseEntity.status(500).build(); + } + + } + +} diff --git a/application/src/main/java/io/redlink/smarti/webservice/RocketChatEndpoint.java b/application/src/main/java/io/redlink/smarti/webservice/RocketChatEndpoint.java index 15ab8539..793e6bdf 100644 --- a/application/src/main/java/io/redlink/smarti/webservice/RocketChatEndpoint.java +++ b/application/src/main/java/io/redlink/smarti/webservice/RocketChatEndpoint.java @@ -146,6 +146,11 @@ public ResponseEntity onRocketEvent(@PathVariable("clientId") String clientId private void notifyRocketChat(String callbackUrl, Conversation conversation, String token) { try (CloseableHttpClient httpClient = httpClientBuilder.build()) { + + if (StringUtils.isEmpty(callbackUrl)) { + log.warn("Webhook URL is empty."); + return; + } final HttpPost post = new HttpPost(callbackUrl); post.setEntity(new StringEntity( toJsonString(new SmartiUpdatePing(conversation.getId(), conversation.getContext().getEnvironment("channel_id"), token)), diff --git a/application/src/main/java/io/redlink/smarti/webservice/pojo/A_SourceConfiguration.java b/application/src/main/java/io/redlink/smarti/webservice/pojo/A_SourceConfiguration.java new file mode 100644 index 00000000..14261a67 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/webservice/pojo/A_SourceConfiguration.java @@ -0,0 +1,33 @@ +package io.redlink.smarti.webservice.pojo; + +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class A_SourceConfiguration implements I_SourceConfiguration { + + @Override + public JSONObject asJSON() { + + try { + return (JSONObject) new JSONParser().parse(new ObjectMapper().writeValueAsString(this)); + } catch (JsonProcessingException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (ParseException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return null; + } + + @Override + public String toString() { + + return asJSON().toJSONString(); + } + +} diff --git a/application/src/main/java/io/redlink/smarti/webservice/pojo/I_SourceConfiguration.java b/application/src/main/java/io/redlink/smarti/webservice/pojo/I_SourceConfiguration.java new file mode 100644 index 00000000..ef3bbba1 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/webservice/pojo/I_SourceConfiguration.java @@ -0,0 +1,9 @@ +package io.redlink.smarti.webservice.pojo; + +import org.json.simple.JSONObject; + +public interface I_SourceConfiguration { + + JSONObject asJSON(); + +} diff --git a/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketFileConfig.java b/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketFileConfig.java new file mode 100644 index 00000000..c9d9b65c --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketFileConfig.java @@ -0,0 +1,27 @@ +package io.redlink.smarti.webservice.pojo; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ +"json_file_url" +}) +/** + * @author Ruediger Kurz (ruediger.kurz@deutschebahn.com) + * @since 03.08.2017 + */ +public class RocketFileConfig extends A_SourceConfiguration { + + @JsonProperty("json_file_url") + private String jsonFileURL; + + public String getJsonFileURL() { + return jsonFileURL; + } + + public void setJsonFileURL(String jsonFileURL) { + this.jsonFileURL = jsonFileURL; + } +} diff --git a/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketMongoConfig.java b/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketMongoConfig.java new file mode 100644 index 00000000..b1dcb79a --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketMongoConfig.java @@ -0,0 +1,103 @@ +package io.redlink.smarti.webservice.pojo; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ +"host", +"port", +"dbname", +"room_collection", +"message_collection", +"filter_field", +"filter_value" +}) +/** + * @author Ruediger Kurz (ruediger.kurz@deutschebahn.com) + * @since 03.08.2017 + */ +public class RocketMongoConfig extends A_SourceConfiguration { + + @JsonProperty("dbname") + private String dbname; + + @JsonProperty("filter_field") + private String filterField; + + @JsonProperty("filter_value") + private String filterValue; + + @JsonProperty("host") + private String host; + + @JsonProperty("message_collection") + private String messageCollection; + + @JsonProperty("port") + private int port; + + @JsonProperty("room_collection") + private String roomCollection; + + public RocketMongoConfig() { + } + + public String getDbname() { + return dbname; + } + + public String getFilterField() { + return filterField; + } + + public String getFilterValue() { + return filterValue; + } + + public String getHost() { + return host; + } + + public String getMessageCollection() { + return messageCollection; + } + + public int getPort() { + return port; + } + + public String getRoomCollection() { + return roomCollection; + } + + public void setDbname(String dbname) { + this.dbname = dbname; + } + + public void setFilterField(String filterField) { + this.filterField = filterField; + } + + public void setFilterValue(String filterValue) { + this.filterValue = filterValue; + } + + public void setHost(String host) { + this.host = host; + } + + public void setMessageCollection(String messageCollection) { + this.messageCollection = messageCollection; + } + + public void setPort(int port) { + this.port = port; + } + + public void setRoomCollection(String roomCollection) { + this.roomCollection = roomCollection; + } + +} diff --git a/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketWebserviceConfig.java b/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketWebserviceConfig.java new file mode 100644 index 00000000..14e79185 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketWebserviceConfig.java @@ -0,0 +1,76 @@ +package io.redlink.smarti.webservice.pojo; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ +"rc_endpoint", +"room_collection", +"message_collection", +"filter_field", +"filter_value" +}) +/** + * @author Ruediger Kurz (ruediger.kurz@deutschebahn.com) + * @since 03.08.2017 + */ +public class RocketWebserviceConfig extends A_SourceConfiguration { + + @JsonProperty("rc_endpoint") + private String rocketChatEndpoint; + + @JsonProperty("room_collection") + private String roomCollection; + + @JsonProperty("message_collection") + private String messageCollection; + + @JsonProperty("filter_field") + private String filterField; + + @JsonProperty("filter_value") + private String filterValue; + + public String getRocketChatEndpoint() { + return rocketChatEndpoint; + } + + public void setRocketChatEndpoint(String rocketChatEndpoint) { + this.rocketChatEndpoint = rocketChatEndpoint; + } + + public String getRoomCollection() { + return roomCollection; + } + + public void setRoomCollection(String roomCollection) { + this.roomCollection = roomCollection; + } + + public String getMessageCollection() { + return messageCollection; + } + + public void setMessageCollection(String messageCollection) { + this.messageCollection = messageCollection; + } + + public String getFilterField() { + return filterField; + } + + public void setFilterField(String filterField) { + this.filterField = filterField; + } + + public String getFilterValue() { + return filterValue; + } + + public void setFilterValue(String filterValue) { + this.filterValue = filterValue; + } +} diff --git a/application/src/main/test/io/redlink/smarti/webservice/ConversationImporterTest.java b/application/src/main/test/io/redlink/smarti/webservice/ConversationImporterTest.java new file mode 100644 index 00000000..24bff699 --- /dev/null +++ b/application/src/main/test/io/redlink/smarti/webservice/ConversationImporterTest.java @@ -0,0 +1,124 @@ +package io.redlink.smarti.webservice; + +import java.io.IOException; + +import org.apache.http.client.ClientProtocolException; +import org.apache.http.entity.StringEntity; +import org.apache.http.localserver.LocalTestServer; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import io.redlink.smarti.services.ImporterService; +import io.redlink.smarti.services.importer.ConversationSourceRocketChatJSONFile; +import io.redlink.smarti.services.importer.ConversationSourceRocketChatMongoDB; +import io.redlink.smarti.services.importer.ConversationSourceRocketChatWebservice; +import io.redlink.smarti.services.importer.ConversationTargetSmarti; +import io.redlink.smarti.services.importer.I_ConversationSource; +import io.redlink.smarti.services.importer.I_ConversationSource.E_SourceType; +import io.redlink.smarti.services.importer.I_ConversationTarget; +import io.redlink.smarti.webservice.pojo.RocketFileConfig; +import io.redlink.smarti.webservice.pojo.RocketMongoConfig; +import io.redlink.smarti.webservice.pojo.RocketWebserviceConfig; + +/** + * + * @author Ruediger Kurz (ruediger.kurz@deutschebahn.com) + * @since 03.08.2017 + */ +public class ConversationImporterTest { + + /** The database name of Rocket.Chat. */ + private static final String ROCKETCHAT_MONGO_DB_NAME = "rocketchat"; + + /** The name of the Rocket.Chat collection that holds the chat messages. */ + private static final String ROCKETCHAT_MESSAGE_COLLECTION = "rocketchat_message"; + + /** The name of the Rocket.Chat collection that holds the rooms. */ + private static final String ROCKETCHAT_ROOM_COLLECTION = "rocketchat_room"; + + /** The host name of the Rocket.Chat MongoDB server. */ + private static final String ROCKETCHAT_MONGO_HOST = "localhost"; + + /** The port number of the Rocket.Chat MongoDB. */ + private static final int ROCKETCHAT_MONGO_PORT = 27017; + + /** The name of a room field/property that is used to filter the rooms that should be exported from Rocket.Chat. */ + private static final String ROCKETCHAT_FILTER_ROOM_FIELD_NAME = "expertise"; + + /** The value of the room field that must match; a regex can be used. */ + private static final String ROCKETCHAT_FILTER_ROOM_FIELD_VALUE = "assistify"; + + /** The absolute path to the Rocket.Chat JSON export. */ + private static final String FILE_IMPORT_SOURCE = "file:///Users/rudiger/projects/smarti/application/src/test/resources/export-sapsus-faqs.json"; + + /** The id of the client that is used to identify the knowledge domain in Smarti. */ + private static final String SMARTI_CLIENT_ID = "test.assistify.de"; + + private LocalTestServer server = new LocalTestServer(null, null); + + @Before + public void setUp() throws Exception { + server.start(); + server.register("*", (httpRequest, httpResponse, httpContext) -> + httpResponse.setEntity(new StringEntity("foobar"))); + } + + @After + public void tearDown() throws Exception { + server.stop(); + } + + @Test + public void testJSONFile() throws ClientProtocolException, IOException, Exception { + runImport(E_SourceType.RocketChatJSONFile); + } + + private void runImport(E_SourceType type) throws ClientProtocolException, IOException, Exception { + + I_ConversationTarget target = new ConversationTargetSmarti(SMARTI_CLIENT_ID, "key123"); + I_ConversationSource source = null; + + switch (type) { + + case RocketChatJSONFile: + RocketFileConfig fileConfig = new RocketFileConfig(); + fileConfig.setJsonFileURL(FILE_IMPORT_SOURCE); + source = new ConversationSourceRocketChatJSONFile (fileConfig); + break; + + case RocketChatMongoDB: + RocketMongoConfig mongoConfig = new RocketMongoConfig(); + mongoConfig.setHost(ROCKETCHAT_MONGO_HOST); + mongoConfig.setPort(ROCKETCHAT_MONGO_PORT); + mongoConfig.setDbname(ROCKETCHAT_MONGO_DB_NAME); + mongoConfig.setRoomCollection(ROCKETCHAT_ROOM_COLLECTION); + mongoConfig.setMessageCollection(ROCKETCHAT_MESSAGE_COLLECTION); + mongoConfig.setFilterField(ROCKETCHAT_FILTER_ROOM_FIELD_NAME); + mongoConfig.setFilterValue(ROCKETCHAT_FILTER_ROOM_FIELD_VALUE); + source = new ConversationSourceRocketChatMongoDB(mongoConfig); + break; + + case RocketChatWebServie: + RocketWebserviceConfig webserviceConfig = new RocketWebserviceConfig(); + webserviceConfig.setRocketChatEndpoint("magic"); + webserviceConfig.setRoomCollection(ROCKETCHAT_ROOM_COLLECTION); + webserviceConfig.setMessageCollection(ROCKETCHAT_MESSAGE_COLLECTION); + webserviceConfig.setFilterField(ROCKETCHAT_FILTER_ROOM_FIELD_NAME); + webserviceConfig.setFilterValue(ROCKETCHAT_FILTER_ROOM_FIELD_VALUE); + source = new ConversationSourceRocketChatWebservice(webserviceConfig); + break; + + case FAQs: + break; + + case Mail: + break; + + default: + break; + } + + new ImporterService().importComversation(source, target); + } +} diff --git a/application/src/main/test/io/redlink/smarti/webservice/RocketChatEndpointProxyTest.java b/application/src/main/test/io/redlink/smarti/webservice/RocketChatEndpointProxyTest.java index 870d2f1d..b306f249 100644 --- a/application/src/main/test/io/redlink/smarti/webservice/RocketChatEndpointProxyTest.java +++ b/application/src/main/test/io/redlink/smarti/webservice/RocketChatEndpointProxyTest.java @@ -49,7 +49,5 @@ public void test() throws IOException { String responseString = IOUtils.toString(response.getEntity().getContent(), Charset.defaultCharset()); Assert.assertEquals("foobar", responseString); } - } - } From 26dcc5e33092cc4a9aa5afbc2f21e896822ee6d8 Mon Sep 17 00:00:00 2001 From: ruKurz Date: Fri, 4 Aug 2017 15:50:05 +0200 Subject: [PATCH 5/7] #36 - Add a web service that provides conversation imports - correct version in pom files --- .gitignore | 1 + application/pom.xml | 10 +- .../smarti/services/I_ImporterService.java | 18 +++ .../smarti/services/ImporterService.java | 150 ++++++++++++++++++ .../importer/A_ConversationSource.java | 47 ++++++ .../importer/A_ConversationTarget.java | 22 +++ .../ConversationSourceRocketChatJSONFile.java | 42 +++++ .../ConversationSourceRocketChatMongoDB.java | 60 +++++++ ...onversationSourceRocketChatWebservice.java | 79 +++++++++ .../importer/ConversationTargetSmarti.java | 11 ++ .../importer/I_ConversationSource.java | 40 +++++ .../importer/I_ConversationTarget.java | 7 + .../webservice/ImporterServiceEndpoint.java | 129 +++++++++++++++ .../smarti/webservice/RocketChatEndpoint.java | 5 + .../pojo/A_SourceConfiguration.java | 33 ++++ .../pojo/I_SourceConfiguration.java | 9 ++ .../webservice/pojo/RocketFileConfig.java | 27 ++++ .../webservice/pojo/RocketMongoConfig.java | 103 ++++++++++++ .../pojo/RocketWebserviceConfig.java | 76 +++++++++ .../webservice/ConversationImporterTest.java | 124 +++++++++++++++ .../RocketChatEndpointProxyTest.java | 2 - core/pom.xml | 2 +- data/pom.xml | 2 +- data/solrcore-crawl-systel/pom.xml | 2 +- data/solrcore-wikipedia-de/pom.xml | 2 +- dist/pom.xml | 2 +- integration/rocket-chat/pom.xml | 2 +- lib/hasso-vocabulary-extractors/pom.xml | 2 +- lib/keyword-interestingterms/pom.xml | 2 +- lib/keyword-solr-mlt/pom.xml | 2 +- lib/ner/pom.xml | 2 +- lib/pom.xml | 2 +- lib/pos/pom.xml | 2 +- lib/query-conversation/pom.xml | 2 +- lib/query-dbsearch/pom.xml | 2 +- lib/token-processor/pom.xml | 2 +- pom.xml | 2 +- 37 files changed, 1008 insertions(+), 19 deletions(-) create mode 100644 application/src/main/java/io/redlink/smarti/services/I_ImporterService.java create mode 100644 application/src/main/java/io/redlink/smarti/services/ImporterService.java create mode 100644 application/src/main/java/io/redlink/smarti/services/importer/A_ConversationSource.java create mode 100644 application/src/main/java/io/redlink/smarti/services/importer/A_ConversationTarget.java create mode 100644 application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatJSONFile.java create mode 100644 application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatMongoDB.java create mode 100644 application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatWebservice.java create mode 100644 application/src/main/java/io/redlink/smarti/services/importer/ConversationTargetSmarti.java create mode 100644 application/src/main/java/io/redlink/smarti/services/importer/I_ConversationSource.java create mode 100644 application/src/main/java/io/redlink/smarti/services/importer/I_ConversationTarget.java create mode 100644 application/src/main/java/io/redlink/smarti/webservice/ImporterServiceEndpoint.java create mode 100644 application/src/main/java/io/redlink/smarti/webservice/pojo/A_SourceConfiguration.java create mode 100644 application/src/main/java/io/redlink/smarti/webservice/pojo/I_SourceConfiguration.java create mode 100644 application/src/main/java/io/redlink/smarti/webservice/pojo/RocketFileConfig.java create mode 100644 application/src/main/java/io/redlink/smarti/webservice/pojo/RocketMongoConfig.java create mode 100644 application/src/main/java/io/redlink/smarti/webservice/pojo/RocketWebserviceConfig.java create mode 100644 application/src/main/test/io/redlink/smarti/webservice/ConversationImporterTest.java diff --git a/.gitignore b/.gitignore index f4835cc7..f7f4e84b 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ .externalToolBuilders/ .classpath .factorypath +bin/ # Maven target/ diff --git a/application/pom.xml b/application/pom.xml index 271489dd..c971a78b 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2 + 0.2.1-SNAPSHOT smarti-application @@ -66,6 +66,14 @@ com.google.guava guava + + com.googlecode.json-simple + json-simple + + + org.mongodb + mongo-java-driver + commons-io commons-io diff --git a/application/src/main/java/io/redlink/smarti/services/I_ImporterService.java b/application/src/main/java/io/redlink/smarti/services/I_ImporterService.java new file mode 100644 index 00000000..4b693de9 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/I_ImporterService.java @@ -0,0 +1,18 @@ +package io.redlink.smarti.services; + +import io.redlink.smarti.services.importer.I_ConversationSource; +import io.redlink.smarti.services.importer.I_ConversationTarget; + +public interface I_ImporterService { + + static final String SMARTI_IMPORT_WEBSERVICE = "import"; + + /** The postfix to convert the Rocket.Chat query result to a valid JSON. */ + static final String JSON_RESULT_WRAPPER_END = "]}"; + + /** The prefix to convert the Rocket.Chat query result to a valid JSON. */ + static final String JSON_RESULT_WRAPPER_START = "{\"export\": ["; + + void importComversation(I_ConversationSource source, I_ConversationTarget target) throws Exception; + +} diff --git a/application/src/main/java/io/redlink/smarti/services/ImporterService.java b/application/src/main/java/io/redlink/smarti/services/ImporterService.java new file mode 100644 index 00000000..1810dbfe --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/ImporterService.java @@ -0,0 +1,150 @@ +package io.redlink.smarti.services; + +import static org.hamcrest.CoreMatchers.instanceOf; + +import java.io.IOException; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Iterator; + +import org.apache.http.client.ClientProtocolException; +import org.bson.types.ObjectId; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import io.redlink.smarti.services.importer.I_ConversationSource; +import io.redlink.smarti.services.importer.I_ConversationTarget; +import io.redlink.smarti.webservice.ConversationWebservice; +import io.redlink.smarti.webservice.RocketChatEndpoint; +import io.redlink.smarti.webservice.pojo.RocketEvent; + +/** + * This service provides an Smarti importer.

    + * + * Sources for import data to Smarti are: + *

  • Directly from Rocket.Chat MongoDB
    • + *
  • A Rocket.Chat MongoDB Export
    • + *
  • An E-Mailbox
    • + *
  • A FAQ website
    • + * + * @author Ruediger Kurz + */ +@Service +public class ImporterService implements I_ImporterService { + + private Logger log = LoggerFactory.getLogger(ImporterService.class); + + /** A simple ISO date formatter. */ + private static final DateFormat ISODateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + + @Autowired + private ConversationWebservice conversationWebservice; + + @Autowired + private RocketChatEndpoint rcEndpoint; + + public ImporterService() { + rcEndpoint = new RocketChatEndpoint(null, 0, null); + } + + long time; + + /** + * @see I_ImporterService#importComversation(I_ConversationSource, I_ConversationTarget) + */ + @Override + public void importComversation(I_ConversationSource source, I_ConversationTarget target) throws Exception { + + + if (source != null && target != null) { + time = new Date().getTime(); + log.info("Start export at : " + (new Date().getTime() - time) + " ms"); + String export = source.exportConversations(); + log.info("Start parsing at : " + (new Date().getTime() - time) + " ms"); + JSONObject eportedJSON = (JSONObject) new JSONParser().parse(export); + log.info("Start import at : " + (new Date().getTime() - time) + " ms"); + importJSON(eportedJSON, target); + log.info("Finished import at : " + (new Date().getTime() - time) + " ms"); + } + } + + /** + * + * @param obj + * @throws IOException + * @throws ClientProtocolException + * @throws ParseException + */ + private void importJSON(JSONObject export, I_ConversationTarget target) throws IOException, ClientProtocolException, ParseException { + + Iterator entryIterator = ((JSONArray) export.get("export")).iterator(); + while (entryIterator.hasNext()) { + + JSONObject expEntry = entryIterator.next(); + if ("r".equals((String) expEntry.get("t"))) { + // the type of this entry is a room + + String channelId = (String) expEntry.get("_id"); + String channelName = (String) expEntry.get("name"); + RocketEvent rocketEvent = null; + boolean newConversation = false; + + JSONArray messages = (JSONArray) expEntry.get("messages"); + Iterator messageIterator = messages.iterator(); + + while (messageIterator.hasNext()) { + JSONObject message = messageIterator.next(); + Object t = message.get("t"); + if (t == null) { + + rocketEvent = new RocketEvent((String) message.get("_id")); + rocketEvent.setBot(null); + rocketEvent.setToken(target.getToken()); + rocketEvent.setChannelId(channelId); + rocketEvent.setChannelName(channelName); + rocketEvent.setText((String) message.get("msg")); + + JSONObject user = (JSONObject) message.get("u"); + rocketEvent.setUserId((String) user.get("_id")); + rocketEvent.setUserId((String) user.get("username")); + + Date timestamp; + String _updatedAt; + Object updatedAt = message.get("_updatedAt"); + if (updatedAt instanceof JSONObject) { + JSONObject jso = (JSONObject) updatedAt; + String dateAsString = (jso.get("$date")).toString(); + timestamp = new Date(new Long(dateAsString)); + _updatedAt = ISODateFormatter.format(timestamp); + } else { + _updatedAt = (String) updatedAt; + timestamp = ISODateFormatter.parse(_updatedAt); + } + rocketEvent.setTimestamp(timestamp); + rocketEvent.setCallbackUrl((String) message.get("webhook_url")); + + rcEndpoint.onRocketEvent(target.getClientId(), rocketEvent); + log.info(rocketEvent.toString()); + newConversation = true; + } + } + if (newConversation && rocketEvent != null) { + ResponseEntity reE = rcEndpoint.getConversation(target.getClientId(), rocketEvent.getChannelId()); + String conversationID = String.valueOf(reE.getBody()); + conversationWebservice.complete(new ObjectId(conversationID)); + } + newConversation = false; + } + } + } +} diff --git a/application/src/main/java/io/redlink/smarti/services/importer/A_ConversationSource.java b/application/src/main/java/io/redlink/smarti/services/importer/A_ConversationSource.java new file mode 100644 index 00000000..86482cd2 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/importer/A_ConversationSource.java @@ -0,0 +1,47 @@ +package io.redlink.smarti.services.importer; + +import io.redlink.smarti.webservice.pojo.I_SourceConfiguration; + +/** + * Abstract implementation of a generic data source.

    + * + * @author Ruediger Kurz + */ +public abstract class A_ConversationSource implements I_ConversationSource { + + /** Source type of this data source. */ + private E_SourceType sourceType; + + /** Configuration options for this data source. */ + I_SourceConfiguration sourceConfiguration; + + /** + * Constructor with parameters.

    + * + * @param sourceType the source type + * @param sourceConfiguration the configuration options + */ + public A_ConversationSource (E_SourceType sourceType, I_SourceConfiguration sourceConfiguration) { + + this.sourceType = sourceType; + this.sourceConfiguration = sourceConfiguration; + } + + /** + * @see I_ConversationSource#getSourceType() + */ + @Override + public E_SourceType getSourceType() { + + return this.sourceType; + } + + /** + * @see I_SourceConfiguration#getSourceConfiguration() + */ + @Override + public I_SourceConfiguration getSourceConfiguration() { + + return this.sourceConfiguration; + } +} diff --git a/application/src/main/java/io/redlink/smarti/services/importer/A_ConversationTarget.java b/application/src/main/java/io/redlink/smarti/services/importer/A_ConversationTarget.java new file mode 100644 index 00000000..297080b5 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/importer/A_ConversationTarget.java @@ -0,0 +1,22 @@ +package io.redlink.smarti.services.importer; + +public abstract class A_ConversationTarget implements I_ConversationTarget { + + private String clientId; + private String token; + + public A_ConversationTarget (String clientId, String token) { + this.clientId = clientId; + this.token = token; + } + + @Override + public String getClientId() { + return clientId; + } + + @Override + public String getToken() { + return token; + } +} diff --git a/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatJSONFile.java b/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatJSONFile.java new file mode 100644 index 00000000..153dc8f3 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatJSONFile.java @@ -0,0 +1,42 @@ +package io.redlink.smarti.services.importer; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URL; +import java.nio.charset.Charset; + +import io.redlink.smarti.webservice.pojo.RocketFileConfig; + +public class ConversationSourceRocketChatJSONFile extends A_ConversationSource { + + RocketFileConfig config; + + public ConversationSourceRocketChatJSONFile(RocketFileConfig config) { + super(E_SourceType.RocketChatJSONFile, config); + this.config = config; + } + + @Override + public String exportConversations() throws Exception { + + return readJsonFromUrl(config.getJsonFileURL()); + } + + public static String readJsonFromUrl(String url) throws IOException { + + InputStream is = new URL(url).openStream(); + try { + BufferedReader rd = new BufferedReader(new InputStreamReader(is, Charset.forName("UTF-8"))); + StringBuilder sb = new StringBuilder(); + int cp; + while ((cp = rd.read()) != -1) { + sb.append((char) cp); + } + return sb.toString(); + } finally { + is.close(); + } + } +} diff --git a/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatMongoDB.java b/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatMongoDB.java new file mode 100644 index 00000000..8d4743f2 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatMongoDB.java @@ -0,0 +1,60 @@ +package io.redlink.smarti.services.importer; + +import java.util.Arrays; + +import org.bson.Document; + +import com.mongodb.MongoClient; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoCursor; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.Aggregates; +import com.mongodb.client.model.Filters; + +import io.redlink.smarti.services.I_ImporterService; +import io.redlink.smarti.webservice.pojo.RocketMongoConfig; + +public class ConversationSourceRocketChatMongoDB extends A_ConversationSource { + + RocketMongoConfig mongoConfig; + + public ConversationSourceRocketChatMongoDB(RocketMongoConfig config) { + + super(E_SourceType.RocketChatMongoDB, config); + mongoConfig = config; + } + + @Override + public String exportConversations() throws Exception { + + MongoClient mongoClient = new MongoClient(mongoConfig.getHost(), mongoConfig.getPort()); + + try { + MongoDatabase db = mongoClient.getDatabase(mongoConfig.getDbname()); + MongoCollection coll = db.getCollection(mongoConfig.getRoomCollection()); + // get all rooms of type 'request' and expertise 'x' joint with messages + MongoCursor iterator = coll.aggregate(Arrays.asList( + Aggregates.match(Filters.eq("t", "r")), + Aggregates.match(Filters.regex(mongoConfig.getFilterField(), mongoConfig.getFilterValue())), + Aggregates.lookup(mongoConfig.getMessageCollection(), "_id", "rid", "messages"))).iterator(); + + StringBuffer input = new StringBuffer(); + input.append(I_ImporterService.JSON_RESULT_WRAPPER_START); + + while (iterator.hasNext()) { + Document doc = iterator.next(); + String jsonDoc = doc.toJson(); + input.append(jsonDoc); + System.out.println(jsonDoc); + if (iterator.hasNext()) { + input.append(", "); + } + } + input.append(I_ImporterService.JSON_RESULT_WRAPPER_END); + + return input.toString(); + } finally { + mongoClient.close(); + } + } +} diff --git a/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatWebservice.java b/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatWebservice.java new file mode 100644 index 00000000..179f9992 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatWebservice.java @@ -0,0 +1,79 @@ +package io.redlink.smarti.services.importer; + +import java.io.IOException; +import java.nio.charset.Charset; + +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpResponse; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.ConnectionBackoffStrategy; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.HttpClientBuilder; + +import io.redlink.smarti.webservice.pojo.RocketWebserviceConfig; + +/** + * Implements the Rocket.Chat MongoDB as data source for Smarti + * + * @author Ruediger Kurz + */ +public class ConversationSourceRocketChatWebservice extends A_ConversationSource { + + RocketWebserviceConfig webserviceConfig; + + public ConversationSourceRocketChatWebservice(RocketWebserviceConfig config) { + + super(E_SourceType.RocketChatWebServie, config); + webserviceConfig = config; + } + + @Override + public String exportConversations() throws Exception { + + return getMongoRequestExport(); + } + + /** + * + * @param payload + * @return + */ + private String getMongoRequestExport() throws IOException, ClientProtocolException { + + HttpPost post = new HttpPost(webserviceConfig.getRocketChatEndpoint()); + post.addHeader("Content-Type", "application/json; charset=UTF-8"); + post.addHeader("Accept-Encoding", "gzip,deflate,sdch"); + post.addHeader("Accept-Language", "en-US,en;q=0.8"); + + StringEntity entity = new StringEntity(getSourceConfiguration().asJSON().toJSONString(), "UTF-8"); + + entity.setContentType("application/json"); + post.setEntity(entity); + + try (CloseableHttpResponse response = httpClientBuilder.build().execute(post)) { + if (200 == response.getStatusLine().getStatusCode()) { + return IOUtils.toString(response.getEntity().getContent(), + Charset.defaultCharset()); + } + } + return null; + } + + + /** A HTTP client to use the Smarti web service. */ + private final HttpClientBuilder httpClientBuilder = HttpClientBuilder.create() + .setRetryHandler((exception, executionCount, context) -> executionCount < 3) + .setConnectionBackoffStrategy(new ConnectionBackoffStrategy() { + @Override + public boolean shouldBackoff(HttpResponse resp) { + return false; + } + + @Override + public boolean shouldBackoff(Throwable t) { + return t instanceof IOException; + } + }).setUserAgent("Smarti/0.0 Rocket.Chat-Endpoint/0.1"); +} diff --git a/application/src/main/java/io/redlink/smarti/services/importer/ConversationTargetSmarti.java b/application/src/main/java/io/redlink/smarti/services/importer/ConversationTargetSmarti.java new file mode 100644 index 00000000..9a20cb7d --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/importer/ConversationTargetSmarti.java @@ -0,0 +1,11 @@ +package io.redlink.smarti.services.importer; + +public class ConversationTargetSmarti extends A_ConversationTarget { + + public ConversationTargetSmarti(String clientId, String token) { + + super(clientId, token); + } + + +} diff --git a/application/src/main/java/io/redlink/smarti/services/importer/I_ConversationSource.java b/application/src/main/java/io/redlink/smarti/services/importer/I_ConversationSource.java new file mode 100644 index 00000000..3aa0db42 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/importer/I_ConversationSource.java @@ -0,0 +1,40 @@ +package io.redlink.smarti.services.importer; + +import io.redlink.smarti.webservice.pojo.I_SourceConfiguration; + +/** + * Interface for potential conversational data sources.

    + * + * @author Ruediger Kurz + * + */ +public interface I_ConversationSource { + + /** The import source type. */ + static enum E_SourceType { + RocketChatJSONFile, RocketChatMongoDB, RocketChatWebServie, Mail, FAQs + } + + /** + * Returns the type of source.

    + * + * @return the type of source + */ + E_SourceType getSourceType(); + + /** + * Returns the source specific configuration options.

    + * + * @return the configuration options + */ + I_SourceConfiguration getSourceConfiguration(); + + /** + * Executes the export action and returns the export result as JSON.

    + * + * @return the export as JSON + * + * @throws Exception if something goes wrong + */ + String exportConversations() throws Exception; +} diff --git a/application/src/main/java/io/redlink/smarti/services/importer/I_ConversationTarget.java b/application/src/main/java/io/redlink/smarti/services/importer/I_ConversationTarget.java new file mode 100644 index 00000000..48516143 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/services/importer/I_ConversationTarget.java @@ -0,0 +1,7 @@ +package io.redlink.smarti.services.importer; + +public interface I_ConversationTarget { + + String getClientId(); + String getToken(); +} diff --git a/application/src/main/java/io/redlink/smarti/webservice/ImporterServiceEndpoint.java b/application/src/main/java/io/redlink/smarti/webservice/ImporterServiceEndpoint.java new file mode 100644 index 00000000..37a92124 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/webservice/ImporterServiceEndpoint.java @@ -0,0 +1,129 @@ +package io.redlink.smarti.webservice; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.ResponseEntity; +import org.springframework.util.MimeTypeUtils; +import org.springframework.web.bind.annotation.CrossOrigin; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import io.redlink.smarti.services.I_ImporterService; +import io.redlink.smarti.services.ImporterService; +import io.redlink.smarti.services.importer.ConversationSourceRocketChatJSONFile; +import io.redlink.smarti.services.importer.ConversationSourceRocketChatMongoDB; +import io.redlink.smarti.services.importer.ConversationSourceRocketChatWebservice; +import io.redlink.smarti.services.importer.I_ConversationSource; +import io.redlink.smarti.services.importer.I_ConversationTarget; +import io.redlink.smarti.webservice.pojo.RocketMongoConfig; +import io.redlink.smarti.webservice.pojo.RocketFileConfig; +import io.redlink.smarti.webservice.pojo.RocketWebserviceConfig; +import io.redlink.smarti.services.importer.ConversationTargetSmarti; +import io.redlink.smarti.services.importer.I_ConversationSource.E_SourceType; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; + +@CrossOrigin +@RestController +@RequestMapping(value = I_ImporterService.SMARTI_IMPORT_WEBSERVICE, + consumes = MimeTypeUtils.APPLICATION_JSON_VALUE, + produces = MimeTypeUtils.APPLICATION_JSON_VALUE) +@Api(I_ImporterService.SMARTI_IMPORT_WEBSERVICE) +public class ImporterServiceEndpoint { + + @Autowired + private ImporterService importerService; + + private Logger log = LoggerFactory.getLogger(ImporterServiceEndpoint.class); + + /** + * This web service should be part of Rocket.Chat not of Smarti. + * + * @param payload + * + * @return the exported chat messages of help request channels + */ + @ApiOperation(value = "imports the conversation from the given source", produces=MimeTypeUtils.APPLICATION_JSON_VALUE) + @RequestMapping(value = "{clientId}/{sourceType}", method = RequestMethod.GET, + produces=MimeTypeUtils.APPLICATION_JSON_VALUE, consumes=MimeTypeUtils.APPLICATION_JSON_VALUE) + public ResponseEntity getImport( + @PathVariable(value="clientId") String clientId, + @PathVariable(value="sourceType") String sourceType, + @RequestBody String payload) { + + ObjectMapper mapper = new ObjectMapper(); + + try { + E_SourceType type = E_SourceType.valueOf(sourceType); + switch (type) { + case RocketChatJSONFile: + importRocketChatJSONFile(clientId, mapper.readValue(payload, RocketFileConfig.class)); + break; + case RocketChatMongoDB: + importRocketChatMongoDB(clientId, mapper.readValue(payload, RocketMongoConfig.class)); + break; + case RocketChatWebServie: + importRocketChatWebservice(clientId, mapper.readValue(payload, RocketWebserviceConfig.class)); + break; + default: + return ResponseEntity.ok("{status: nosource}"); + } + return ResponseEntity.ok("{status: sucess}"); + } catch (Exception e) { + return ResponseEntity.status(500).build(); + } + } + + @ApiOperation(value = "imports the conversation from the given source", produces=MimeTypeUtils.APPLICATION_JSON_VALUE) + @RequestMapping(value = "{clientId}/rocketChatJSONFile", method = RequestMethod.GET, + produces=MimeTypeUtils.APPLICATION_JSON_VALUE, consumes=MimeTypeUtils.APPLICATION_JSON_VALUE) + public ResponseEntity importRocketChatJSONFile( + @PathVariable(value="clientId") String clientId, + @RequestBody RocketFileConfig fileConfig) { + + log.warn("called import webservice: '/import/" + clientId + "/rocketChatJSONFile' with config: " + fileConfig.toString()); + return runImport (new ConversationSourceRocketChatJSONFile(fileConfig), clientId); + } + + @ApiOperation(value = "imports the conversation from the given source", produces=MimeTypeUtils.APPLICATION_JSON_VALUE) + @RequestMapping(value = "{clientId}/rocketChatMongoDB", method = RequestMethod.GET, + produces=MimeTypeUtils.APPLICATION_JSON_VALUE, consumes=MimeTypeUtils.APPLICATION_JSON_VALUE) + public ResponseEntity importRocketChatMongoDB( + @PathVariable(value="clientId") String clientId, + @RequestBody RocketMongoConfig mongoConfig) { + + log.warn("called import webservice: '/import/" + clientId + "/rocketChatJSONFile' with config: " + mongoConfig.toString()); + return runImport (new ConversationSourceRocketChatMongoDB(mongoConfig), clientId); + } + + @ApiOperation(value = "imports the conversation from the given source", produces=MimeTypeUtils.APPLICATION_JSON_VALUE) + @RequestMapping(value = "{clientId}/rocketChatWebservice", method = RequestMethod.GET, + produces=MimeTypeUtils.APPLICATION_JSON_VALUE, consumes=MimeTypeUtils.APPLICATION_JSON_VALUE) + public ResponseEntity importRocketChatWebservice( + @PathVariable(value="clientId") String clientId, + @RequestBody RocketWebserviceConfig webserviceConfig) { + + log.warn("called import webservice: '/import/" + clientId + "/rocketChatJSONFile' with config: " + webserviceConfig.toString()); + return runImport (new ConversationSourceRocketChatWebservice(webserviceConfig), clientId); + } + + private ResponseEntity runImport (I_ConversationSource source, String clientId) { + // TODO: Token not yet implemented, replace key123. + I_ConversationTarget target = new ConversationTargetSmarti(clientId, "key123"); + try { + importerService.importComversation(source, target); + return ResponseEntity.ok("{status: sucess}"); + } catch (Exception e) { + log.error("Error with message: \"{}\" occured during import: ", e.getMessage(), e); + return ResponseEntity.status(500).build(); + } + + } + +} diff --git a/application/src/main/java/io/redlink/smarti/webservice/RocketChatEndpoint.java b/application/src/main/java/io/redlink/smarti/webservice/RocketChatEndpoint.java index 15ab8539..793e6bdf 100644 --- a/application/src/main/java/io/redlink/smarti/webservice/RocketChatEndpoint.java +++ b/application/src/main/java/io/redlink/smarti/webservice/RocketChatEndpoint.java @@ -146,6 +146,11 @@ public ResponseEntity onRocketEvent(@PathVariable("clientId") String clientId private void notifyRocketChat(String callbackUrl, Conversation conversation, String token) { try (CloseableHttpClient httpClient = httpClientBuilder.build()) { + + if (StringUtils.isEmpty(callbackUrl)) { + log.warn("Webhook URL is empty."); + return; + } final HttpPost post = new HttpPost(callbackUrl); post.setEntity(new StringEntity( toJsonString(new SmartiUpdatePing(conversation.getId(), conversation.getContext().getEnvironment("channel_id"), token)), diff --git a/application/src/main/java/io/redlink/smarti/webservice/pojo/A_SourceConfiguration.java b/application/src/main/java/io/redlink/smarti/webservice/pojo/A_SourceConfiguration.java new file mode 100644 index 00000000..14261a67 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/webservice/pojo/A_SourceConfiguration.java @@ -0,0 +1,33 @@ +package io.redlink.smarti.webservice.pojo; + +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class A_SourceConfiguration implements I_SourceConfiguration { + + @Override + public JSONObject asJSON() { + + try { + return (JSONObject) new JSONParser().parse(new ObjectMapper().writeValueAsString(this)); + } catch (JsonProcessingException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (ParseException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return null; + } + + @Override + public String toString() { + + return asJSON().toJSONString(); + } + +} diff --git a/application/src/main/java/io/redlink/smarti/webservice/pojo/I_SourceConfiguration.java b/application/src/main/java/io/redlink/smarti/webservice/pojo/I_SourceConfiguration.java new file mode 100644 index 00000000..ef3bbba1 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/webservice/pojo/I_SourceConfiguration.java @@ -0,0 +1,9 @@ +package io.redlink.smarti.webservice.pojo; + +import org.json.simple.JSONObject; + +public interface I_SourceConfiguration { + + JSONObject asJSON(); + +} diff --git a/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketFileConfig.java b/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketFileConfig.java new file mode 100644 index 00000000..c9d9b65c --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketFileConfig.java @@ -0,0 +1,27 @@ +package io.redlink.smarti.webservice.pojo; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ +"json_file_url" +}) +/** + * @author Ruediger Kurz (ruediger.kurz@deutschebahn.com) + * @since 03.08.2017 + */ +public class RocketFileConfig extends A_SourceConfiguration { + + @JsonProperty("json_file_url") + private String jsonFileURL; + + public String getJsonFileURL() { + return jsonFileURL; + } + + public void setJsonFileURL(String jsonFileURL) { + this.jsonFileURL = jsonFileURL; + } +} diff --git a/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketMongoConfig.java b/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketMongoConfig.java new file mode 100644 index 00000000..b1dcb79a --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketMongoConfig.java @@ -0,0 +1,103 @@ +package io.redlink.smarti.webservice.pojo; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ +"host", +"port", +"dbname", +"room_collection", +"message_collection", +"filter_field", +"filter_value" +}) +/** + * @author Ruediger Kurz (ruediger.kurz@deutschebahn.com) + * @since 03.08.2017 + */ +public class RocketMongoConfig extends A_SourceConfiguration { + + @JsonProperty("dbname") + private String dbname; + + @JsonProperty("filter_field") + private String filterField; + + @JsonProperty("filter_value") + private String filterValue; + + @JsonProperty("host") + private String host; + + @JsonProperty("message_collection") + private String messageCollection; + + @JsonProperty("port") + private int port; + + @JsonProperty("room_collection") + private String roomCollection; + + public RocketMongoConfig() { + } + + public String getDbname() { + return dbname; + } + + public String getFilterField() { + return filterField; + } + + public String getFilterValue() { + return filterValue; + } + + public String getHost() { + return host; + } + + public String getMessageCollection() { + return messageCollection; + } + + public int getPort() { + return port; + } + + public String getRoomCollection() { + return roomCollection; + } + + public void setDbname(String dbname) { + this.dbname = dbname; + } + + public void setFilterField(String filterField) { + this.filterField = filterField; + } + + public void setFilterValue(String filterValue) { + this.filterValue = filterValue; + } + + public void setHost(String host) { + this.host = host; + } + + public void setMessageCollection(String messageCollection) { + this.messageCollection = messageCollection; + } + + public void setPort(int port) { + this.port = port; + } + + public void setRoomCollection(String roomCollection) { + this.roomCollection = roomCollection; + } + +} diff --git a/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketWebserviceConfig.java b/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketWebserviceConfig.java new file mode 100644 index 00000000..14e79185 --- /dev/null +++ b/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketWebserviceConfig.java @@ -0,0 +1,76 @@ +package io.redlink.smarti.webservice.pojo; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ +"rc_endpoint", +"room_collection", +"message_collection", +"filter_field", +"filter_value" +}) +/** + * @author Ruediger Kurz (ruediger.kurz@deutschebahn.com) + * @since 03.08.2017 + */ +public class RocketWebserviceConfig extends A_SourceConfiguration { + + @JsonProperty("rc_endpoint") + private String rocketChatEndpoint; + + @JsonProperty("room_collection") + private String roomCollection; + + @JsonProperty("message_collection") + private String messageCollection; + + @JsonProperty("filter_field") + private String filterField; + + @JsonProperty("filter_value") + private String filterValue; + + public String getRocketChatEndpoint() { + return rocketChatEndpoint; + } + + public void setRocketChatEndpoint(String rocketChatEndpoint) { + this.rocketChatEndpoint = rocketChatEndpoint; + } + + public String getRoomCollection() { + return roomCollection; + } + + public void setRoomCollection(String roomCollection) { + this.roomCollection = roomCollection; + } + + public String getMessageCollection() { + return messageCollection; + } + + public void setMessageCollection(String messageCollection) { + this.messageCollection = messageCollection; + } + + public String getFilterField() { + return filterField; + } + + public void setFilterField(String filterField) { + this.filterField = filterField; + } + + public String getFilterValue() { + return filterValue; + } + + public void setFilterValue(String filterValue) { + this.filterValue = filterValue; + } +} diff --git a/application/src/main/test/io/redlink/smarti/webservice/ConversationImporterTest.java b/application/src/main/test/io/redlink/smarti/webservice/ConversationImporterTest.java new file mode 100644 index 00000000..24bff699 --- /dev/null +++ b/application/src/main/test/io/redlink/smarti/webservice/ConversationImporterTest.java @@ -0,0 +1,124 @@ +package io.redlink.smarti.webservice; + +import java.io.IOException; + +import org.apache.http.client.ClientProtocolException; +import org.apache.http.entity.StringEntity; +import org.apache.http.localserver.LocalTestServer; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import io.redlink.smarti.services.ImporterService; +import io.redlink.smarti.services.importer.ConversationSourceRocketChatJSONFile; +import io.redlink.smarti.services.importer.ConversationSourceRocketChatMongoDB; +import io.redlink.smarti.services.importer.ConversationSourceRocketChatWebservice; +import io.redlink.smarti.services.importer.ConversationTargetSmarti; +import io.redlink.smarti.services.importer.I_ConversationSource; +import io.redlink.smarti.services.importer.I_ConversationSource.E_SourceType; +import io.redlink.smarti.services.importer.I_ConversationTarget; +import io.redlink.smarti.webservice.pojo.RocketFileConfig; +import io.redlink.smarti.webservice.pojo.RocketMongoConfig; +import io.redlink.smarti.webservice.pojo.RocketWebserviceConfig; + +/** + * + * @author Ruediger Kurz (ruediger.kurz@deutschebahn.com) + * @since 03.08.2017 + */ +public class ConversationImporterTest { + + /** The database name of Rocket.Chat. */ + private static final String ROCKETCHAT_MONGO_DB_NAME = "rocketchat"; + + /** The name of the Rocket.Chat collection that holds the chat messages. */ + private static final String ROCKETCHAT_MESSAGE_COLLECTION = "rocketchat_message"; + + /** The name of the Rocket.Chat collection that holds the rooms. */ + private static final String ROCKETCHAT_ROOM_COLLECTION = "rocketchat_room"; + + /** The host name of the Rocket.Chat MongoDB server. */ + private static final String ROCKETCHAT_MONGO_HOST = "localhost"; + + /** The port number of the Rocket.Chat MongoDB. */ + private static final int ROCKETCHAT_MONGO_PORT = 27017; + + /** The name of a room field/property that is used to filter the rooms that should be exported from Rocket.Chat. */ + private static final String ROCKETCHAT_FILTER_ROOM_FIELD_NAME = "expertise"; + + /** The value of the room field that must match; a regex can be used. */ + private static final String ROCKETCHAT_FILTER_ROOM_FIELD_VALUE = "assistify"; + + /** The absolute path to the Rocket.Chat JSON export. */ + private static final String FILE_IMPORT_SOURCE = "file:///Users/rudiger/projects/smarti/application/src/test/resources/export-sapsus-faqs.json"; + + /** The id of the client that is used to identify the knowledge domain in Smarti. */ + private static final String SMARTI_CLIENT_ID = "test.assistify.de"; + + private LocalTestServer server = new LocalTestServer(null, null); + + @Before + public void setUp() throws Exception { + server.start(); + server.register("*", (httpRequest, httpResponse, httpContext) -> + httpResponse.setEntity(new StringEntity("foobar"))); + } + + @After + public void tearDown() throws Exception { + server.stop(); + } + + @Test + public void testJSONFile() throws ClientProtocolException, IOException, Exception { + runImport(E_SourceType.RocketChatJSONFile); + } + + private void runImport(E_SourceType type) throws ClientProtocolException, IOException, Exception { + + I_ConversationTarget target = new ConversationTargetSmarti(SMARTI_CLIENT_ID, "key123"); + I_ConversationSource source = null; + + switch (type) { + + case RocketChatJSONFile: + RocketFileConfig fileConfig = new RocketFileConfig(); + fileConfig.setJsonFileURL(FILE_IMPORT_SOURCE); + source = new ConversationSourceRocketChatJSONFile (fileConfig); + break; + + case RocketChatMongoDB: + RocketMongoConfig mongoConfig = new RocketMongoConfig(); + mongoConfig.setHost(ROCKETCHAT_MONGO_HOST); + mongoConfig.setPort(ROCKETCHAT_MONGO_PORT); + mongoConfig.setDbname(ROCKETCHAT_MONGO_DB_NAME); + mongoConfig.setRoomCollection(ROCKETCHAT_ROOM_COLLECTION); + mongoConfig.setMessageCollection(ROCKETCHAT_MESSAGE_COLLECTION); + mongoConfig.setFilterField(ROCKETCHAT_FILTER_ROOM_FIELD_NAME); + mongoConfig.setFilterValue(ROCKETCHAT_FILTER_ROOM_FIELD_VALUE); + source = new ConversationSourceRocketChatMongoDB(mongoConfig); + break; + + case RocketChatWebServie: + RocketWebserviceConfig webserviceConfig = new RocketWebserviceConfig(); + webserviceConfig.setRocketChatEndpoint("magic"); + webserviceConfig.setRoomCollection(ROCKETCHAT_ROOM_COLLECTION); + webserviceConfig.setMessageCollection(ROCKETCHAT_MESSAGE_COLLECTION); + webserviceConfig.setFilterField(ROCKETCHAT_FILTER_ROOM_FIELD_NAME); + webserviceConfig.setFilterValue(ROCKETCHAT_FILTER_ROOM_FIELD_VALUE); + source = new ConversationSourceRocketChatWebservice(webserviceConfig); + break; + + case FAQs: + break; + + case Mail: + break; + + default: + break; + } + + new ImporterService().importComversation(source, target); + } +} diff --git a/application/src/main/test/io/redlink/smarti/webservice/RocketChatEndpointProxyTest.java b/application/src/main/test/io/redlink/smarti/webservice/RocketChatEndpointProxyTest.java index 870d2f1d..b306f249 100644 --- a/application/src/main/test/io/redlink/smarti/webservice/RocketChatEndpointProxyTest.java +++ b/application/src/main/test/io/redlink/smarti/webservice/RocketChatEndpointProxyTest.java @@ -49,7 +49,5 @@ public void test() throws IOException { String responseString = IOUtils.toString(response.getEntity().getContent(), Charset.defaultCharset()); Assert.assertEquals("foobar", responseString); } - } - } diff --git a/core/pom.xml b/core/pom.xml index 3aa36472..002057ee 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2 + 0.2.1-SNAPSHOT smarti-core diff --git a/data/pom.xml b/data/pom.xml index b8c75c14..f81d4a69 100644 --- a/data/pom.xml +++ b/data/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2 + 0.2.1-SNAPSHOT data diff --git a/data/solrcore-crawl-systel/pom.xml b/data/solrcore-crawl-systel/pom.xml index 65e5a75a..f253df1c 100644 --- a/data/solrcore-crawl-systel/pom.xml +++ b/data/solrcore-crawl-systel/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2 + 0.2.1-SNAPSHOT ../../ diff --git a/data/solrcore-wikipedia-de/pom.xml b/data/solrcore-wikipedia-de/pom.xml index cee20ae7..651bb3d5 100644 --- a/data/solrcore-wikipedia-de/pom.xml +++ b/data/solrcore-wikipedia-de/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2 + 0.2.1-SNAPSHOT ../../ diff --git a/dist/pom.xml b/dist/pom.xml index 0476e0d3..345521b3 100644 --- a/dist/pom.xml +++ b/dist/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2 + 0.2.1-SNAPSHOT smarti-dist diff --git a/integration/rocket-chat/pom.xml b/integration/rocket-chat/pom.xml index b38fde55..93b6e6fe 100644 --- a/integration/rocket-chat/pom.xml +++ b/integration/rocket-chat/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2 + 0.2.1-SNAPSHOT ../.. diff --git a/lib/hasso-vocabulary-extractors/pom.xml b/lib/hasso-vocabulary-extractors/pom.xml index 77178fac..5b490b3c 100644 --- a/lib/hasso-vocabulary-extractors/pom.xml +++ b/lib/hasso-vocabulary-extractors/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2 + 0.2.1-SNAPSHOT ../../ diff --git a/lib/keyword-interestingterms/pom.xml b/lib/keyword-interestingterms/pom.xml index 43b11768..8efac099 100644 --- a/lib/keyword-interestingterms/pom.xml +++ b/lib/keyword-interestingterms/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2 + 0.2.1-SNAPSHOT ../../ diff --git a/lib/keyword-solr-mlt/pom.xml b/lib/keyword-solr-mlt/pom.xml index 2a4f806b..876c9a2e 100644 --- a/lib/keyword-solr-mlt/pom.xml +++ b/lib/keyword-solr-mlt/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2 + 0.2.1-SNAPSHOT ../../ diff --git a/lib/ner/pom.xml b/lib/ner/pom.xml index 16daabe0..0065da0a 100644 --- a/lib/ner/pom.xml +++ b/lib/ner/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2 + 0.2.1-SNAPSHOT ../../ diff --git a/lib/pom.xml b/lib/pom.xml index 35cfbd5f..d577f2a5 100644 --- a/lib/pom.xml +++ b/lib/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2 + 0.2.1-SNAPSHOT lib diff --git a/lib/pos/pom.xml b/lib/pos/pom.xml index 9ab7e549..b8c517bf 100644 --- a/lib/pos/pom.xml +++ b/lib/pos/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2 + 0.2.1-SNAPSHOT ../../ diff --git a/lib/query-conversation/pom.xml b/lib/query-conversation/pom.xml index 1cc50be8..3e90dea1 100644 --- a/lib/query-conversation/pom.xml +++ b/lib/query-conversation/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2 + 0.2.1-SNAPSHOT ../../ diff --git a/lib/query-dbsearch/pom.xml b/lib/query-dbsearch/pom.xml index 2908b351..0a3c265c 100644 --- a/lib/query-dbsearch/pom.xml +++ b/lib/query-dbsearch/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2 + 0.2.1-SNAPSHOT ../../ diff --git a/lib/token-processor/pom.xml b/lib/token-processor/pom.xml index 33c570cf..90d14b21 100644 --- a/lib/token-processor/pom.xml +++ b/lib/token-processor/pom.xml @@ -22,7 +22,7 @@ io.redlink.smarti smarti - 0.2.0-RC2 + 0.2.1-SNAPSHOT ../../ diff --git a/pom.xml b/pom.xml index 85d561dc..6a76e85a 100644 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,7 @@ io.redlink.smarti smarti - 0.2.0-RC2 + 0.2.1-SNAPSHOT pom smarti From 03d7554a8b2654a91f147c7678ba7165a6439770 Mon Sep 17 00:00:00 2001 From: ruKurz Date: Fri, 4 Aug 2017 16:09:15 +0200 Subject: [PATCH 6/7] #36 - Add a web service that provides conversation imports - correct version in pom files --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6a76e85a..ceb80890 100644 --- a/pom.xml +++ b/pom.xml @@ -109,7 +109,7 @@ scm:git:https://github.com/redlink-gmbh/smarti.git scm:git:git@github.com:redlink-gmbh/smarti.git https://github.com/redlink-gmbh/smarti/src - smarti-0.2.0-RC2 + HEAD github From bd21d2054ac7eba686b0032a5ef2f3d99f86a109 Mon Sep 17 00:00:00 2001 From: ruKurz Date: Fri, 4 Aug 2017 16:09:15 +0200 Subject: [PATCH 7/7] #36 - Add a web service that provides conversation imports - corrections from review - Do not use Webservices in a Service. - Use spring to manage your service beans. - Just use the ConversationService and Message object. - Use JacksonMapper / direct serialization with ObjectMapper - Removed order from JSON/Beans --- .../smarti/services/I_ImporterService.java | 6 - .../smarti/services/ImporterService.java | 111 +++++++----------- .../ConversationSourceRocketChatMongoDB.java | 23 ++-- ...onversationSourceRocketChatWebservice.java | 17 ++- .../pojo/A_SourceConfiguration.java | 27 ++--- .../pojo/I_SourceConfiguration.java | 6 +- .../smarti/webservice/pojo/RocketBot.java | 2 +- .../webservice/pojo/RocketFileConfig.java | 7 ++ .../webservice/pojo/RocketMongoConfig.java | 20 ++-- .../pojo/RocketWebserviceConfig.java | 18 +-- 10 files changed, 106 insertions(+), 131 deletions(-) diff --git a/application/src/main/java/io/redlink/smarti/services/I_ImporterService.java b/application/src/main/java/io/redlink/smarti/services/I_ImporterService.java index 4b693de9..4b62c067 100644 --- a/application/src/main/java/io/redlink/smarti/services/I_ImporterService.java +++ b/application/src/main/java/io/redlink/smarti/services/I_ImporterService.java @@ -6,12 +6,6 @@ public interface I_ImporterService { static final String SMARTI_IMPORT_WEBSERVICE = "import"; - - /** The postfix to convert the Rocket.Chat query result to a valid JSON. */ - static final String JSON_RESULT_WRAPPER_END = "]}"; - - /** The prefix to convert the Rocket.Chat query result to a valid JSON. */ - static final String JSON_RESULT_WRAPPER_START = "{\"export\": ["; void importComversation(I_ConversationSource source, I_ConversationTarget target) throws Exception; diff --git a/application/src/main/java/io/redlink/smarti/services/ImporterService.java b/application/src/main/java/io/redlink/smarti/services/ImporterService.java index 1810dbfe..0593a9d6 100644 --- a/application/src/main/java/io/redlink/smarti/services/ImporterService.java +++ b/application/src/main/java/io/redlink/smarti/services/ImporterService.java @@ -1,32 +1,27 @@ package io.redlink.smarti.services; -import static org.hamcrest.CoreMatchers.instanceOf; - import java.io.IOException; -import java.text.DateFormat; + import java.text.ParseException; -import java.text.SimpleDateFormat; import java.util.Date; import java.util.Iterator; import org.apache.http.client.ClientProtocolException; -import org.bson.types.ObjectId; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; -import com.fasterxml.jackson.databind.ObjectMapper; - +import io.redlink.smarti.api.StoreService; +import io.redlink.smarti.model.Conversation; +import io.redlink.smarti.model.ConversationMeta; +import io.redlink.smarti.model.Message; +import io.redlink.smarti.model.User; import io.redlink.smarti.services.importer.I_ConversationSource; import io.redlink.smarti.services.importer.I_ConversationTarget; -import io.redlink.smarti.webservice.ConversationWebservice; -import io.redlink.smarti.webservice.RocketChatEndpoint; -import io.redlink.smarti.webservice.pojo.RocketEvent; /** * This service provides an Smarti importer.

    @@ -44,20 +39,11 @@ public class ImporterService implements I_ImporterService { private Logger log = LoggerFactory.getLogger(ImporterService.class); - /** A simple ISO date formatter. */ - private static final DateFormat ISODateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); - @Autowired - private ConversationWebservice conversationWebservice; + private ConversationService conversationService; @Autowired - private RocketChatEndpoint rcEndpoint; - - public ImporterService() { - rcEndpoint = new RocketChatEndpoint(null, 0, null); - } - - long time; + private StoreService storeService; /** * @see I_ImporterService#importComversation(I_ConversationSource, I_ConversationTarget) @@ -65,27 +51,29 @@ public ImporterService() { @Override public void importComversation(I_ConversationSource source, I_ConversationTarget target) throws Exception { - if (source != null && target != null) { - time = new Date().getTime(); + long time = new Date().getTime(); log.info("Start export at : " + (new Date().getTime() - time) + " ms"); String export = source.exportConversations(); log.info("Start parsing at : " + (new Date().getTime() - time) + " ms"); JSONObject eportedJSON = (JSONObject) new JSONParser().parse(export); log.info("Start import at : " + (new Date().getTime() - time) + " ms"); - importJSON(eportedJSON, target); + importJSON(source, target, eportedJSON); log.info("Finished import at : " + (new Date().getTime() - time) + " ms"); } } /** + * @param source + * @param target + * @param export * - * @param obj * @throws IOException * @throws ClientProtocolException - * @throws ParseException + * @throws ParseException */ - private void importJSON(JSONObject export, I_ConversationTarget target) throws IOException, ClientProtocolException, ParseException { + @SuppressWarnings("unchecked") + private void importJSON(I_ConversationSource source, I_ConversationTarget target, JSONObject export) throws IOException, ClientProtocolException, ParseException { Iterator entryIterator = ((JSONArray) export.get("export")).iterator(); while (entryIterator.hasNext()) { @@ -96,54 +84,45 @@ private void importJSON(JSONObject export, I_ConversationTarget target) throws I String channelId = (String) expEntry.get("_id"); String channelName = (String) expEntry.get("name"); - RocketEvent rocketEvent = null; - boolean newConversation = false; JSONArray messages = (JSONArray) expEntry.get("messages"); Iterator messageIterator = messages.iterator(); + Message m = null; + Conversation conversation = null; + while (messageIterator.hasNext()) { JSONObject message = messageIterator.next(); Object t = message.get("t"); if (t == null) { - - rocketEvent = new RocketEvent((String) message.get("_id")); - rocketEvent.setBot(null); - rocketEvent.setToken(target.getToken()); - rocketEvent.setChannelId(channelId); - rocketEvent.setChannelName(channelName); - rocketEvent.setText((String) message.get("msg")); - - JSONObject user = (JSONObject) message.get("u"); - rocketEvent.setUserId((String) user.get("_id")); - rocketEvent.setUserId((String) user.get("username")); - - Date timestamp; - String _updatedAt; - Object updatedAt = message.get("_updatedAt"); - if (updatedAt instanceof JSONObject) { - JSONObject jso = (JSONObject) updatedAt; - String dateAsString = (jso.get("$date")).toString(); - timestamp = new Date(new Long(dateAsString)); - _updatedAt = ISODateFormatter.format(timestamp); - } else { - _updatedAt = (String) updatedAt; - timestamp = ISODateFormatter.parse(_updatedAt); - } - rocketEvent.setTimestamp(timestamp); - rocketEvent.setCallbackUrl((String) message.get("webhook_url")); - - rcEndpoint.onRocketEvent(target.getClientId(), rocketEvent); - log.info(rocketEvent.toString()); - newConversation = true; + conversation = storeService.getCurrentConversationByChannelId(channelId, () -> { + final Conversation newConversation = new Conversation(); + newConversation.getContext().setContextType(source.getSourceType().toString()); + newConversation.getContext().setDomain(target.getClientId()); + newConversation.getContext().setEnvironment("channel", channelName); + newConversation.getContext().setEnvironment("channel_id", channelId); + newConversation.getContext().setEnvironment("token", target.getToken()); + return newConversation; + }); + + m = new Message(); + m.setId((String) message.get("_id")); + m.setContent((String) message.get("msg")); + m.setTime(new Date((Long) message.get("_updatedAt"))); + m.setOrigin(Message.Origin.User); + + // TODO: Use a UserService to actually *store* the users + JSONObject user = (JSONObject) message.get("u"); + final User userO = new User((String) user.get("_id")); + userO.setDisplayName((String) user.get("username")); + m.setUser(userO); + + conversation = conversationService.appendMessage(conversation, m); + conversation.getMeta().setStatus(ConversationMeta.Status.Complete); + conversationService.completeConversation(conversation); + log.info(conversation.toString()); } } - if (newConversation && rocketEvent != null) { - ResponseEntity reE = rcEndpoint.getConversation(target.getClientId(), rocketEvent.getChannelId()); - String conversationID = String.valueOf(reE.getBody()); - conversationWebservice.complete(new ObjectId(conversationID)); - } - newConversation = false; } } } diff --git a/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatMongoDB.java b/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatMongoDB.java index 8d4743f2..2e6d0202 100644 --- a/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatMongoDB.java +++ b/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatMongoDB.java @@ -1,9 +1,12 @@ package io.redlink.smarti.services.importer; import java.util.Arrays; +import java.util.Collections; import org.bson.Document; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; import com.mongodb.MongoClient; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; @@ -11,7 +14,6 @@ import com.mongodb.client.model.Aggregates; import com.mongodb.client.model.Filters; -import io.redlink.smarti.services.I_ImporterService; import io.redlink.smarti.webservice.pojo.RocketMongoConfig; public class ConversationSourceRocketChatMongoDB extends A_ConversationSource { @@ -38,21 +40,10 @@ public String exportConversations() throws Exception { Aggregates.match(Filters.regex(mongoConfig.getFilterField(), mongoConfig.getFilterValue())), Aggregates.lookup(mongoConfig.getMessageCollection(), "_id", "rid", "messages"))).iterator(); - StringBuffer input = new StringBuffer(); - input.append(I_ImporterService.JSON_RESULT_WRAPPER_START); - - while (iterator.hasNext()) { - Document doc = iterator.next(); - String jsonDoc = doc.toJson(); - input.append(jsonDoc); - System.out.println(jsonDoc); - if (iterator.hasNext()) { - input.append(", "); - } - } - input.append(I_ImporterService.JSON_RESULT_WRAPPER_END); - - return input.toString(); + ObjectMapper mapper = new ObjectMapper(); + String mappString = mapper.writeValueAsString(Collections.singletonMap("export", ImmutableList.copyOf(iterator))); + + return mappString; } finally { mongoClient.close(); } diff --git a/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatWebservice.java b/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatWebservice.java index 179f9992..2eac664b 100644 --- a/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatWebservice.java +++ b/application/src/main/java/io/redlink/smarti/services/importer/ConversationSourceRocketChatWebservice.java @@ -2,6 +2,7 @@ import java.io.IOException; import java.nio.charset.Charset; +import java.nio.charset.UnsupportedCharsetException; import org.apache.commons.io.IOUtils; import org.apache.http.HttpResponse; @@ -37,17 +38,22 @@ public String exportConversations() throws Exception { /** * - * @param payload * @return + * + * @throws IOException + * @throws ClientProtocolException + * @throws IOException + * @throws ClientProtocolException + * @throws UnsupportedCharsetException */ - private String getMongoRequestExport() throws IOException, ClientProtocolException { + private String getMongoRequestExport() throws IOException, ClientProtocolException, IOException, ClientProtocolException, UnsupportedCharsetException { HttpPost post = new HttpPost(webserviceConfig.getRocketChatEndpoint()); post.addHeader("Content-Type", "application/json; charset=UTF-8"); post.addHeader("Accept-Encoding", "gzip,deflate,sdch"); post.addHeader("Accept-Language", "en-US,en;q=0.8"); - StringEntity entity = new StringEntity(getSourceConfiguration().asJSON().toJSONString(), "UTF-8"); + StringEntity entity = new StringEntity(getSourceConfiguration().asString(), "UTF-8"); entity.setContentType("application/json"); post.setEntity(entity); @@ -61,8 +67,9 @@ private String getMongoRequestExport() throws IOException, ClientProtocolExcepti return null; } - - /** A HTTP client to use the Smarti web service. */ + /** + * A HTTP client to use the Smarti web service.

    + */ private final HttpClientBuilder httpClientBuilder = HttpClientBuilder.create() .setRetryHandler((exception, executionCount, context) -> executionCount < 3) .setConnectionBackoffStrategy(new ConnectionBackoffStrategy() { diff --git a/application/src/main/java/io/redlink/smarti/webservice/pojo/A_SourceConfiguration.java b/application/src/main/java/io/redlink/smarti/webservice/pojo/A_SourceConfiguration.java index 14261a67..819df061 100644 --- a/application/src/main/java/io/redlink/smarti/webservice/pojo/A_SourceConfiguration.java +++ b/application/src/main/java/io/redlink/smarti/webservice/pojo/A_SourceConfiguration.java @@ -8,26 +8,17 @@ import com.fasterxml.jackson.databind.ObjectMapper; public class A_SourceConfiguration implements I_SourceConfiguration { - + @Override - public JSONObject asJSON() { - - try { - return (JSONObject) new JSONParser().parse(new ObjectMapper().writeValueAsString(this)); - } catch (JsonProcessingException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (ParseException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - return null; + public JSONObject asJSON() throws ParseException, JsonProcessingException { + + return (JSONObject) new JSONParser().parse(toString()); } - - @Override - public String toString() { - return asJSON().toJSONString(); + public String asString() throws JsonProcessingException { + + ObjectMapper mapper = new ObjectMapper(); + return mapper.writeValueAsString(this); } -} +} \ No newline at end of file diff --git a/application/src/main/java/io/redlink/smarti/webservice/pojo/I_SourceConfiguration.java b/application/src/main/java/io/redlink/smarti/webservice/pojo/I_SourceConfiguration.java index ef3bbba1..e0e57479 100644 --- a/application/src/main/java/io/redlink/smarti/webservice/pojo/I_SourceConfiguration.java +++ b/application/src/main/java/io/redlink/smarti/webservice/pojo/I_SourceConfiguration.java @@ -1,9 +1,13 @@ package io.redlink.smarti.webservice.pojo; import org.json.simple.JSONObject; +import org.json.simple.parser.ParseException; + +import com.fasterxml.jackson.core.JsonProcessingException; public interface I_SourceConfiguration { - JSONObject asJSON(); + JSONObject asJSON() throws ParseException, JsonProcessingException; + String asString() throws JsonProcessingException; } diff --git a/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketBot.java b/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketBot.java index d73fe06e..a71cce81 100644 --- a/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketBot.java +++ b/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketBot.java @@ -42,7 +42,7 @@ public void setIdentifier(String identifier) { this.identifier = identifier; } - public static class JacksonDeserializer extends JsonDeserializer { + public static class JacksonDeserializer extends JsonDeserializer { public RocketBot deserialize(JsonParser parser, DeserializationContext context) throws IOException, JsonProcessingException { if(JsonToken.START_OBJECT.equals(parser.getCurrentToken())) { ObjectMapper mapper = new ObjectMapper(); diff --git a/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketFileConfig.java b/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketFileConfig.java index c9d9b65c..f7cacb8a 100644 --- a/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketFileConfig.java +++ b/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketFileConfig.java @@ -14,6 +14,13 @@ */ public class RocketFileConfig extends A_SourceConfiguration { + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("RocketFileConfig [jsonFileURL=").append(jsonFileURL).append("]"); + return builder.toString(); + } + @JsonProperty("json_file_url") private String jsonFileURL; diff --git a/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketMongoConfig.java b/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketMongoConfig.java index b1dcb79a..3aebaabd 100644 --- a/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketMongoConfig.java +++ b/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketMongoConfig.java @@ -2,24 +2,24 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; @JsonInclude(JsonInclude.Include.NON_NULL) -@JsonPropertyOrder({ -"host", -"port", -"dbname", -"room_collection", -"message_collection", -"filter_field", -"filter_value" -}) /** * @author Ruediger Kurz (ruediger.kurz@deutschebahn.com) * @since 03.08.2017 */ public class RocketMongoConfig extends A_SourceConfiguration { + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("RocketMongoConfig [dbname=").append(dbname).append(", filterField=").append(filterField) + .append(", filterValue=").append(filterValue).append(", host=").append(host) + .append(", messageCollection=").append(messageCollection).append(", port=").append(port) + .append(", roomCollection=").append(roomCollection).append("]"); + return builder.toString(); + } + @JsonProperty("dbname") private String dbname; diff --git a/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketWebserviceConfig.java b/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketWebserviceConfig.java index 14e79185..74e72ca7 100644 --- a/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketWebserviceConfig.java +++ b/application/src/main/java/io/redlink/smarti/webservice/pojo/RocketWebserviceConfig.java @@ -2,17 +2,9 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; @JsonInclude(JsonInclude.Include.NON_NULL) -@JsonPropertyOrder({ -"rc_endpoint", -"room_collection", -"message_collection", -"filter_field", -"filter_value" -}) /** * @author Ruediger Kurz (ruediger.kurz@deutschebahn.com) * @since 03.08.2017 @@ -28,6 +20,16 @@ public class RocketWebserviceConfig extends A_SourceConfiguration { @JsonProperty("message_collection") private String messageCollection; + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("RocketWebserviceConfig [rocketChatEndpoint=").append(rocketChatEndpoint) + .append(", roomCollection=").append(roomCollection).append(", messageCollection=") + .append(messageCollection).append(", filterField=").append(filterField).append(", filterValue=") + .append(filterValue).append("]"); + return builder.toString(); + } + @JsonProperty("filter_field") private String filterField;