diff --git a/app/assets/javascripts/results.jsx b/app/assets/javascripts/results.jsx index b70d235..4125ffc 100644 --- a/app/assets/javascripts/results.jsx +++ b/app/assets/javascripts/results.jsx @@ -338,7 +338,6 @@ var Container = React.createClass({ }, onMerge: function(){ let data = this.state.mergeData; - console.log(data); let mergeUrl = context + '/' + this.props.searchUid + '/merge'; $.ajax({ url: mergeUrl, @@ -1187,8 +1186,9 @@ var ResultsContainer = React.createClass({ if (loadmore) loadMore = "&loadMoreResults=true"; - var searchUrl = context + "/engine/api/searches/" + this.props.searchUid + "/results?entityType=" + eType + "&sources=" + sourcesDirty + "&types=" + typesDirty + "&exact=" + exactMatching + loadMore; + + var searchUrl = context + "/engine/api/searches/" + this.props.searchUid + "/results?entityType=" + eType + "&sources=" + sourcesDirty + "&types=" + typesDirty + "&exact=" + exactMatching + loadMore; $.ajax({ url: searchUrl, dataType: 'json', @@ -1611,7 +1611,6 @@ var ResultsList = React.createClass({ var already_crawled = this.props.crawled; var resultsNodes = resultsNodesSorted.map(function (result,idx) { - console.log(result); if (result["@type"] === "foaf:Person") { return (

- {this.props.onion_url} + {this.props.title || this.props.onion_url}

Web title: {this.props.onion_label}

diff --git a/app/assets/javascripts/search.jsx b/app/assets/javascripts/search.jsx index 72aa87f..f22a26d 100644 --- a/app/assets/javascripts/search.jsx +++ b/app/assets/javascripts/search.jsx @@ -262,6 +262,7 @@ var SearchForm = React.createClass({ typesForSource["occrp"] = ["document"]; typesForSource["tor2web"] = ["website"]; typesForSource["elasticsearch"] = ["website"]; + typesForSource["gcs"] = ["website"]; //typesForSource["vk"] = ["person"]; //typesForSource["darknetmarkets"] = ["product"]; //typesForSource["pipl"] = ["person"]; diff --git a/app/assets/javascripts/translation.js b/app/assets/javascripts/translation.js index cb07b6e..0643ffe 100644 --- a/app/assets/javascripts/translation.js +++ b/app/assets/javascripts/translation.js @@ -101,7 +101,7 @@ var dictEng = { "interests": "Interests", "server_error": "There was an error during search or no active tokens where available... Please try again!", "no_results": "No results found.", - "tor_websites": "Onion web sites", + "tor_websites": "Onion / Web sites", "select_file": "Select file", "tor_invalid_websites": "The web site you are trying to crawl seems to be not online anymore. Crawling Job not started.", "invalid_website": "The website is offline", diff --git a/app/controllers/de/fuhsen/wrappers/GoogleCustomSearchWrapper.scala b/app/controllers/de/fuhsen/wrappers/GoogleCustomSearchWrapper.scala new file mode 100644 index 0000000..d61fd30 --- /dev/null +++ b/app/controllers/de/fuhsen/wrappers/GoogleCustomSearchWrapper.scala @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2016 EIS Uni-Bonn + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package controllers.de.fuhsen.wrappers + +import com.typesafe.config.ConfigFactory +import controllers.de.fuhsen.wrappers.dataintegration.{SilkTransformableTrait, SilkTransformationTask} + +/** + * Wrapper for the Google Custom Search REST API. + * Search result type: Websites + * Search results from + * www.pdr.net/ + * www.drugs.com/pdra.html + * www.drugs.com/international/ + * ndews.umd.edu/ + */ +class GoogleCustomSearchWrapper extends RestApiWrapperTrait with SilkTransformableTrait { + + /** Query parameters that should be added to the request. */ + override def queryParams: Map[String, String] = Map( + "key" -> ConfigFactory.load.getString("gcs.search.api.key"), + "cx" -> ConfigFactory.load.getString("gcs.search.engine.id")) + + /** Headers that should be added to the request. */ + override def headersParams: Map[String, String] = Map() + + /** Returns for a given query string the representation as query parameter for the specific API. */ + override def searchQueryAsParam(queryString: String): Map[String, String] = { + Map("q" -> queryString) + } + + /** The REST endpoint URL */ + override def apiUrl: String = ConfigFactory.load.getString("gcs.search.url") + + /** + * Returns the globally unique URI String of the source that is wrapped. This is used to track provenance. + */ + override def sourceLocalName: String = "gcs" + + /** SILK Transformation Trait **/ + override def silkTransformationRequestTasks = Seq( + SilkTransformationTask( + transformationTaskId = "GoogleCustomWebsiteTransformation", + createSilkTransformationRequestBody( + basePath = "/items", + uriPattern = "http://vocab.cs.uni-bonn.de/fuhsen/search/entity/{link}" + ) + ) + ) + + /** The type of the transformation input. */ + override def datasetPluginType: DatasetPluginType = DatasetPluginType.JsonDatasetPlugin + + /** The project id of the Silk project */ + override def projectId: String = ConfigFactory.load.getString("silk.socialApiProject.id") + + + override def requestType: String = "GET" +} diff --git a/app/controllers/de/fuhsen/wrappers/WrapperController.scala b/app/controllers/de/fuhsen/wrappers/WrapperController.scala index be14597..a7cde5b 100644 --- a/app/controllers/de/fuhsen/wrappers/WrapperController.scala +++ b/app/controllers/de/fuhsen/wrappers/WrapperController.scala @@ -503,7 +503,9 @@ object WrapperController { //vk new VkWrapper(), //darknetmarkets - new DarknetMarketsWrapper() + new DarknetMarketsWrapper(), + //Google Custom Search + new GoogleCustomSearchWrapper() ) val wrapperMap: Map[String, RestApiWrapperTrait] = wrappers.map { wrapper => (wrapper.sourceLocalName, wrapper) diff --git a/conf/application.conf b/conf/application.conf index 435fbbb..fd56513 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -170,6 +170,11 @@ vk.login.redirect.uri = "http://localhost:9000"${?play.http.context} vk.search.url = "https://api.vk.com/method/users.search" vk.search.fields = "uid,first_name,last_name,bdate,screen_name,sex,city,country,photo_50,timezone,last_seen,verified" +#Google Custom Search +gcs.search.url = "https://www.googleapis.com/customsearch/v1" +gcs.search.engine.id = "" +gcs.search.api.key = "" + #WS Request Timeout play.ws.timeout.connection = 5000 play.ws.timeout.idle = 100000 diff --git a/public/images/datasources/GCS.png b/public/images/datasources/GCS.png new file mode 100644 index 0000000..27280f4 Binary files /dev/null and b/public/images/datasources/GCS.png differ diff --git a/schema/ontofuhsen.ttl b/schema/ontofuhsen.ttl index bcebcc2..b0a3b93 100644 --- a/schema/ontofuhsen.ttl +++ b/schema/ontofuhsen.ttl @@ -152,6 +152,13 @@ fs:ElasticSearch rdf:type fs:InformationSource; fs:finds "website"; fs:key "elasticsearch" . +fs:GCS rdf:type fs:InformationSource; + rdfs:label "Google Custom Search Results"; + fs:help_url "https://en.wikipedia.org/wiki/Google_Custom_Search"@en, + "https://en.wikipedia.org/wiki/Google_Custom_Search"@de; + fs:finds "website"; + fs:key "gcs" . + #fs:Vk rdf:type fs:InformationSource; # rdfs:label "Vk"; # fs:help_url "https://en.wikipedia.org/wiki/VK_(social_networking)"@en,