LIDAKRAEXT-199 : Integrate Google Custom Search API

app/assets/javascripts/results.jsx

@@ -338,7 +338,6 @@ var Container = React.createClass({
     },
     onMerge: function(){
       let data = this.state.mergeData;
-      console.log(data);
       let mergeUrl = context + '/' + this.props.searchUid + '/merge';
         $.ajax({
             url: mergeUrl,
@@ -1187,8 +1186,9 @@ var ResultsContainer = React.createClass({
         if (loadmore)
             loadMore = "&loadMoreResults=true";
 
-        var searchUrl = context + "/engine/api/searches/" + this.props.searchUid + "/results?entityType=" + eType + "&sources=" + sourcesDirty + "&types=" + typesDirty + "&exact=" + exactMatching + loadMore;
 
+
+        var searchUrl = context + "/engine/api/searches/" + this.props.searchUid + "/results?entityType=" + eType + "&sources=" + sourcesDirty + "&types=" + typesDirty + "&exact=" + exactMatching + loadMore;
         $.ajax({
             url: searchUrl,
             dataType: 'json',
@@ -1611,7 +1611,6 @@ var ResultsList = React.createClass({
 
         var already_crawled = this.props.crawled;
         var resultsNodes = resultsNodesSorted.map(function (result,idx) {
-            console.log(result);
             if (result["@type"] === "foaf:Person") {
                 return (
                     <PersonResultElement
@@ -1703,8 +1702,9 @@ var ResultsList = React.createClass({
                     return (
                         <WebResultElement
                             uri = {result["@id"]}
-                            img={context + "/assets/images/datasources/TorLogo.png"}
+                            img={context + "/assets/images/datasources/" + result["fs:source"] + ".png"}
                             onion_url={result["url"]}
+                            title = {result["fs:label"]}
                             comment={result["fs:comment"]}
                             source={result["fs:source"]}
                             onion_label={result["rdfs:label"]}
@@ -1851,7 +1851,7 @@ var WebResultElement = React.createClass({
                     <div className="summary-main-wrapper col-md-8">
                         <div className="summary-main">
                             <h2 className="title">
-                                {this.props.onion_url}
+                                {this.props.title || this.props.onion_url}
                             </h2>
                             <div className="subtitle">
                                 <p><b>Web title</b>: {this.props.onion_label}</p>

app/assets/javascripts/search.jsx

@@ -262,6 +262,7 @@ var SearchForm = React.createClass({
         typesForSource["occrp"] = ["document"];
         typesForSource["tor2web"]  = ["website"];
         typesForSource["elasticsearch"]  = ["website"];
+        typesForSource["gcs"]  = ["website"];
         //typesForSource["vk"] = ["person"];
         //typesForSource["darknetmarkets"] = ["product"];
         //typesForSource["pipl"] = ["person"];

app/assets/javascripts/translation.js

@@ -101,7 +101,7 @@ var dictEng = {
     "interests": "Interests",
     "server_error": "There was an error during search or no active tokens where available... Please try again!",
     "no_results": "No results found.",
-    "tor_websites": "Onion web sites",
+    "tor_websites": "Onion / Web sites",
     "select_file": "Select file",
     "tor_invalid_websites": "The web site you are trying to crawl seems to be not online anymore. Crawling Job not started.",
     "invalid_website": "The website is offline",

app/controllers/de/fuhsen/wrappers/GoogleCustomSearchWrapper.scala

@@ -0,0 +1,72 @@
+/*
+ * Copyright (C) 2016 EIS Uni-Bonn
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package controllers.de.fuhsen.wrappers
+
+import com.typesafe.config.ConfigFactory
+import controllers.de.fuhsen.wrappers.dataintegration.{SilkTransformableTrait, SilkTransformationTask}
+
+/**
+  * Wrapper for the Google Custom Search REST API.
+  * Search result type: Websites
+  * Search results from
+  *     www.pdr.net/
+  *     www.drugs.com/pdra.html
+  *     www.drugs.com/international/
+  *     ndews.umd.edu/
+  */
+class GoogleCustomSearchWrapper extends RestApiWrapperTrait with SilkTransformableTrait {
+
+  /** Query parameters that should be added to the request. */
+  override def queryParams: Map[String, String] = Map(
+    "key" -> ConfigFactory.load.getString("gcs.search.api.key"),
+    "cx" -> ConfigFactory.load.getString("gcs.search.engine.id"))
+
+  /** Headers that should be added to the request. */
+  override def headersParams: Map[String, String] = Map()
+
+  /** Returns for a given query string the representation as query parameter for the specific API. */
+  override def searchQueryAsParam(queryString: String): Map[String, String] = {
+    Map("q" -> queryString)
+  }
+
+  /** The REST endpoint URL */
+  override def apiUrl: String = ConfigFactory.load.getString("gcs.search.url")
+
+  /**
+    * Returns the globally unique URI String of the source that is wrapped. This is used to track provenance.
+    */
+  override def sourceLocalName: String = "gcs"
+
+  /** SILK Transformation Trait **/
+  override def silkTransformationRequestTasks = Seq(
+    SilkTransformationTask(
+      transformationTaskId = "GoogleCustomWebsiteTransformation",
+      createSilkTransformationRequestBody(
+        basePath = "/items",
+        uriPattern = "http://vocab.cs.uni-bonn.de/fuhsen/search/entity/{link}"
+      )
+    )
+  )
+
+  /** The type of the transformation input. */
+  override def datasetPluginType: DatasetPluginType = DatasetPluginType.JsonDatasetPlugin
+
+  /** The project id of the Silk project */
+  override def projectId: String = ConfigFactory.load.getString("silk.socialApiProject.id")
+
+
+  override def requestType: String = "GET"
+}

app/controllers/de/fuhsen/wrappers/WrapperController.scala

@@ -503,7 +503,9 @@ object WrapperController {
     //vk
     new VkWrapper(),
     //darknetmarkets
-    new DarknetMarketsWrapper()
+    new DarknetMarketsWrapper(),
+    //Google Custom Search
+    new GoogleCustomSearchWrapper()
   )
   val wrapperMap: Map[String, RestApiWrapperTrait] = wrappers.map { wrapper =>
     (wrapper.sourceLocalName, wrapper)

conf/application.conf

@@ -170,6 +170,11 @@ vk.login.redirect.uri = "http://localhost:9000"${?play.http.context}
 vk.search.url = "https://api.vk.com/method/users.search"
 vk.search.fields = "uid,first_name,last_name,bdate,screen_name,sex,city,country,photo_50,timezone,last_seen,verified"
 
+#Google Custom Search
+gcs.search.url = "https://www.googleapis.com/customsearch/v1"
+gcs.search.engine.id = ""
+gcs.search.api.key = ""
+
 #WS Request Timeout
 play.ws.timeout.connection = 5000
 play.ws.timeout.idle = 100000

schema/ontofuhsen.ttl

@@ -152,6 +152,13 @@ fs:ElasticSearch	rdf:type	fs:InformationSource;
 			        fs:finds "website";
 			        fs:key "elasticsearch" .
 
+fs:GCS	rdf:type	fs:InformationSource;
+			        rdfs:label 	"Google Custom Search Results";
+			        fs:help_url "https://en.wikipedia.org/wiki/Google_Custom_Search"@en,
+                           "https://en.wikipedia.org/wiki/Google_Custom_Search"@de;
+			        fs:finds "website";
+			        fs:key "gcs" .
+
 #fs:Vk rdf:type fs:InformationSource;
 #                    rdfs:label 	"Vk";
 #                    fs:help_url "https://en.wikipedia.org/wiki/VK_(social_networking)"@en,