diff --git a/solr/configsets/scholars-discovery/conf/managed-schema.xml b/solr/configsets/scholars-discovery/conf/managed-schema.xml
index 9b732b66a..db33a7e48 100644
--- a/solr/configsets/scholars-discovery/conf/managed-schema.xml
+++ b/solr/configsets/scholars-discovery/conf/managed-schema.xml
@@ -202,10 +202,6 @@
field first in an ascending sort and last in a descending sort.
-->
-
-
-
-
@@ -358,6 +354,10 @@
+
+
+
+
diff --git a/solr/configsets/scholars-discovery/conf/solrconfig.xml b/solr/configsets/scholars-discovery/conf/solrconfig.xml
index e01f9abbb..bc3d62f63 100644
--- a/solr/configsets/scholars-discovery/conf/solrconfig.xml
+++ b/solr/configsets/scholars-discovery/conf/solrconfig.xml
@@ -35,7 +35,7 @@
that you fully re-index after changing this setting as it can
affect both how text is indexed and queried.
-->
- 9.4
+ 9.7
@@ -385,10 +385,9 @@
to occupy. Note that when this option is specified, the size
and initialSize parameters are ignored.
-->
-
-
+
-
-
+
-
+
+ autowarmCount="128"/>
+
-
-
-
-
-
+
+
-
-
-
-
-
-
+
+
@@ -990,48 +969,6 @@
[EEE, ]dd MMM yyyy HH:mm[:ss] z
EEEE, dd-MMM-yy HH:mm:ss z
EEE MMM ppd HH:mm:ss [z ]yyyy
-
- yyyy-MM-dd'T'HH:mm:ss.SSS
- yyyy-MM-dd'T'HH:mm:ss.SSSz
- yyyy-MM-dd'T'HH:mm:ss.SSSZ
- yyyy-MM-dd'T'HH:mm:ss.SSSZZ
-
- yyyy-MM-dd'T'HH:mm:ss,SSS
- yyyy-MM-dd'T'HH:mm:ss,SSSz
- yyyy-MM-dd'T'HH:mm:ss,SSSZ
- yyyy-MM-dd'T'HH:mm:ss,SSSZZ
-
- yyyy-MM-dd'T'HH:mm:ss
- yyyy-MM-dd'T'HH:mm:ssz
- yyyy-MM-dd'T'HH:mm:ssZ
- yyyy-MM-dd'T'HH:mm:ssZZ
-
- yyyy-MM-dd'T'HH:mm
- yyyy-MM-dd'T'HH:mmz
- yyyy-MM-dd'T'HH:mmZ
- yyyy-MM-dd'T'HH:mmZZ
-
- yyyy-MM-dd HH:mm:ss.SSS
- yyyy-MM-dd HH:mm:ss.SSSz
- yyyy-MM-dd HH:mm:ss.SSSZ
- yyyy-MM-dd HH:mm:ss.SSSZZ
-
- yyyy-MM-dd HH:mm:ss,SSS
- yyyy-MM-dd HH:mm:ss,SSSz
- yyyy-MM-dd HH:mm:ss,SSSZ
- yyyy-MM-dd HH:mm:ss,SSSZZ
-
- yyyy-MM-dd HH:mm:ss
- yyyy-MM-dd HH:mm:ssz
- yyyy-MM-dd HH:mm:ssZ
- yyyy-MM-dd HH:mm:ssZZ
-
- yyyy-MM-dd HH:mm
- yyyy-MM-dd HH:mmz
- yyyy-MM-dd HH:mmZ
- yyyy-MM-dd HH:mmZZ
-
- yyyy-MM-dd
diff --git a/src/main/java/edu/tamu/scholars/middleware/config/WebSocketSecurityConfig.java b/src/main/java/edu/tamu/scholars/middleware/config/WebSocketSecurityConfig.java
index 05a805a44..0bd757e1b 100644
--- a/src/main/java/edu/tamu/scholars/middleware/config/WebSocketSecurityConfig.java
+++ b/src/main/java/edu/tamu/scholars/middleware/config/WebSocketSecurityConfig.java
@@ -29,6 +29,7 @@ protected void configureInbound(MessageSecurityMetadataSourceRegistry messages)
.simpSubscribeDestMatchers(
"/queue/public",
"/queue/themes",
+ "/queue/dataAndAnalyticsViews",
"/queue/directoryViews",
"/queue/discoveryViews",
"/queue/displayViews",
diff --git a/src/main/java/edu/tamu/scholars/middleware/discovery/DiscoveryConstants.java b/src/main/java/edu/tamu/scholars/middleware/discovery/DiscoveryConstants.java
index 4f718293a..0e7f2ec00 100644
--- a/src/main/java/edu/tamu/scholars/middleware/discovery/DiscoveryConstants.java
+++ b/src/main/java/edu/tamu/scholars/middleware/discovery/DiscoveryConstants.java
@@ -10,12 +10,16 @@ public class DiscoveryConstants {
public static final String CLASS = "class";
+ public static final String ABSTRACT = "abstract";
+
public static final String TYPE = "type";
public static final String SNIPPET = "snippet";
public static final String MOD_TIME = "modTime";
+ public static final String SYNC_IDS = "syncIds";
+
public static final String QUERY_DELIMETER = ":";
public static final String DEFAULT_QUERY = WILDCARD + QUERY_DELIMETER + WILDCARD;
diff --git a/src/main/java/edu/tamu/scholars/middleware/discovery/component/Harvester.java b/src/main/java/edu/tamu/scholars/middleware/discovery/component/Harvester.java
index 6371170ba..1302adc30 100644
--- a/src/main/java/edu/tamu/scholars/middleware/discovery/component/Harvester.java
+++ b/src/main/java/edu/tamu/scholars/middleware/discovery/component/Harvester.java
@@ -1,13 +1,14 @@
package edu.tamu.scholars.middleware.discovery.component;
import edu.tamu.scholars.middleware.discovery.model.AbstractIndexDocument;
+import edu.tamu.scholars.middleware.discovery.model.Individual;
import reactor.core.publisher.Flux;
public interface Harvester {
- public Flux harvest();
+ public Flux harvest();
- public AbstractIndexDocument harvest(String subject);
+ public Individual harvest(String subject);
public Class type();
diff --git a/src/main/java/edu/tamu/scholars/middleware/discovery/component/Indexer.java b/src/main/java/edu/tamu/scholars/middleware/discovery/component/Indexer.java
index 10c07f3e5..81556bfe2 100644
--- a/src/main/java/edu/tamu/scholars/middleware/discovery/component/Indexer.java
+++ b/src/main/java/edu/tamu/scholars/middleware/discovery/component/Indexer.java
@@ -3,14 +3,15 @@
import java.util.Collection;
import edu.tamu.scholars.middleware.discovery.model.AbstractIndexDocument;
+import edu.tamu.scholars.middleware.discovery.model.Individual;
public interface Indexer {
public void init();
- public void index(Collection documents);
+ public void index(Collection documents);
- public void index(AbstractIndexDocument document);
+ public void index(Individual document);
public void optimize();
diff --git a/src/main/java/edu/tamu/scholars/middleware/discovery/component/jena/TriplestoreHarvester.java b/src/main/java/edu/tamu/scholars/middleware/discovery/component/jena/TriplestoreHarvester.java
index 4df50b731..95642cd9f 100644
--- a/src/main/java/edu/tamu/scholars/middleware/discovery/component/jena/TriplestoreHarvester.java
+++ b/src/main/java/edu/tamu/scholars/middleware/discovery/component/jena/TriplestoreHarvester.java
@@ -1,6 +1,5 @@
package edu.tamu.scholars.middleware.discovery.component.jena;
-import static edu.tamu.scholars.middleware.discovery.DiscoveryConstants.ID;
import static edu.tamu.scholars.middleware.discovery.DiscoveryConstants.NESTED_DELIMITER;
import java.lang.reflect.Field;
@@ -32,6 +31,7 @@
import edu.tamu.scholars.middleware.discovery.annotation.FieldType;
import edu.tamu.scholars.middleware.discovery.component.Harvester;
import edu.tamu.scholars.middleware.discovery.model.AbstractIndexDocument;
+import edu.tamu.scholars.middleware.discovery.model.Individual;
import edu.tamu.scholars.middleware.service.TemplateService;
import edu.tamu.scholars.middleware.service.Triplestore;
import reactor.core.publisher.Flux;
@@ -40,7 +40,7 @@ public class TriplestoreHarvester implements Harvester {
private static final Logger logger = LoggerFactory.getLogger(TriplestoreHarvester.class);
- private static final String COLLECTION_SPARQL_TEMPLATE = "collection";
+ private static final String COLLECTION = "collection";
private static final String FORWARD_SLASH = "/";
@@ -58,36 +58,39 @@ public class TriplestoreHarvester implements Harvester {
private final List propertySourceTypeOps;
- private final List indexedFields;
+ private final List nestedFields;
public TriplestoreHarvester(Class type) {
this.type = type;
- this.propertySourceTypeOps = FieldUtils.getFieldsListWithAnnotation(type, FieldSource.class).stream().map(this::getTypeOp).collect(Collectors.toList());
- this.indexedFields = FieldUtils.getFieldsListWithAnnotation(type, FieldType.class);
+ this.propertySourceTypeOps = FieldUtils.getFieldsListWithAnnotation(type, FieldSource.class)
+ .stream()
+ .map(this::getTypeOp)
+ .collect(Collectors.toList());
+ this.nestedFields = FieldUtils.getFieldsListWithAnnotation(type, FieldType.class)
+ .stream()
+ .filter(this::isNestedField)
+ .collect(Collectors.toList());
}
- public Flux harvest() {
+ public Flux harvest() {
CollectionSource source = type.getAnnotation(CollectionSource.class);
- String query = templateService.templateSparql(COLLECTION_SPARQL_TEMPLATE, source.predicate());
- logger.debug("{}:\n{}", COLLECTION_SPARQL_TEMPLATE, query);
+ String query = templateService.templateSparql(COLLECTION, source.predicate());
+ logger.debug("{}:\n{}", COLLECTION, query);
QueryExecution queryExecution = triplestore.createQueryExecution(query);
Iterator tripleIterator = queryExecution.execConstructTriples();
- Iterable triples = () -> tripleIterator;
- return Flux.fromIterable(triples)
+
+ return Flux.fromIterable(() -> tripleIterator)
.map(this::subject)
.map(this::harvest)
.doFinally(onFinally -> queryExecution.close());
}
- public AbstractIndexDocument harvest(String subject) {
+ public Individual harvest(String subject) {
try {
- return createDocument(subject);
+ return createIndividual(subject);
} catch (Exception e) {
- logger.error("Unable to index {}: {}", type.getSimpleName(), parse(subject));
- logger.error("Error: {}", e.getMessage());
- if (logger.isDebugEnabled()) {
- e.printStackTrace();
- }
+ logger.debug("Error harvesting", e);
+ logger.error("Unable to harvest {} {}: {}", type.getSimpleName(), parse(subject), e.getMessage());
throw new RuntimeException(e);
}
}
@@ -96,31 +99,30 @@ public Class type() {
return type;
}
- private String subject(Triple triple) {
- return triple.getSubject().toString();
- }
+ private Individual createIndividual(String subject) throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException {
+ Individual individual = new Individual();
+ individual.setId(parse(subject));
+ lookupProperties(individual, subject);
+ lookupSyncIds(individual);
+ individual.setProxy(type.getSimpleName());
- private AbstractIndexDocument createDocument(String subject) throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException {
- AbstractIndexDocument document = construct();
- document.setId(parse(subject));
- lookupProperties(document, subject);
- lookupSyncIds(document);
- return document;
+ return individual;
}
- private void lookupProperties(AbstractIndexDocument document, String subject) {
+ private void lookupProperties(Individual individual, String subject) {
propertySourceTypeOps.parallelStream().forEach(typeOp -> {
try {
FieldSource source = typeOp.getPropertySource();
Model model = queryForModel(source, subject);
List