diff --git a/.gitignore b/.gitignore index 66e38e631..9d672a540 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,5 @@ triplestore/ assets/ logs/ + +pgdata/ diff --git a/Dockerfile b/Dockerfile index 0837031b8..7750f92c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ ARG HOME_DIR=/$USER_NAME ARG SOURCE_DIR=$HOME_DIR/source # Maven stage. -FROM maven:3-eclipse-temurin-17-alpine as maven +FROM maven:3-eclipse-temurin-17-alpine AS maven ARG USER_ID ARG USER_NAME ARG HOME_DIR @@ -24,6 +24,7 @@ RUN apk -U upgrade WORKDIR $SOURCE_DIR # Copy files over. +COPY ./checkstyles.xml ./checkstyles.xml COPY ./pom.xml ./pom.xml COPY ./src ./src COPY ./solr ./solr diff --git a/README.md b/README.md index e2c45ea02..86cb551e5 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,48 @@ docker run -d -p 9000:9000 -e SPRING_APPLICATION_JSON="{\"spring\":{\"data\":{\" > The environment variable `SPRING_APPLICATION_JSON` will override properties in application.yml. +## Docker Compose for Development + +```bash +docker-compose up +``` + +This will provide Postgres database at localhost:5432 and Solr at localhost:8983. There should be two volume mounts at relative path `pgdata` and `solr/data`. + +To run the `mvn spring-boot:run` command with `SPRING_APPLICATION_JSON` defined, you can use the following approach: + +``` +SPRING_APPLICATION_JSON='{"spring.datasource.driver-class-name":"org.postgresql.Driver","spring.datasource.url":"jdbc:postgresql://localhost:5432/scholars","spring.jpa.database-platform":"org.hibernate.dialect.PostgreSQLDialect","spring.sql.init.platform":"postgres"}' mvn spring-boot:run +``` + +Save the following as `config.json`. + +```json +{ + "spring.datasource.driver-class-name": "org.postgresql.Driver", + "spring.datasource.url": "jdbc:postgresql://localhost:5432/scholars", + "spring.jpa.database-platform": "org.hibernate.dialect.PostgreSQLDialect", + "spring.sql.init.platform": "postgres" +} +``` + +``` +SPRING_APPLICATION_JSON=$(cat config.json) mvn spring-boot:run +``` + + +For Windows Command Prompt, the syntax is slightly different: + +``` +set SPRING_APPLICATION_JSON={"spring.datasource.driver-class-name":"org.postgresql.Driver","spring.datasource.url":"jdbc:postgresql://localhost:5432/scholars","spring.jpa.database-platform":"org.hibernate.dialect.PostgreSQLDialect","spring.sql.init.platform":"postgres"} && mvn spring-boot:run +``` + +For Windows PowerShell: + +``` +$env:SPRING_APPLICATION_JSON='{"spring.datasource.driver-class-name":"org.postgresql.Driver","spring.datasource.url":"jdbc:postgresql://localhost:5432/scholars","spring.jpa.database-platform":"org.hibernate.dialect.PostgreSQLDialect","spring.sql.init.platform":"postgres"}'; mvn spring-boot:run +``` + ## Verify Installation With the above installation instructions, the following service endpoints can be verified: diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..d204bfcee --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,23 @@ +services: + + db: + container_name: scholars-db + image: postgres + environment: + - POSTGRES_DB=scholars + - POSTGRES_USER=scholars + - POSTGRES_PASSWORD=scholars + volumes: + - ./pgdata:/var/lib/postgresql/data + ports: + - 5432:5432 + + solr: + container_name: scholars-solr + build: + context: ./solr/ + dockerfile: Dockerfile + volumes: + - ./solr/data:/var/solr/data + ports: + - 8983:8983 diff --git a/pom.xml b/pom.xml index 2722fb821..99733f4a5 100644 --- a/pom.xml +++ b/pom.xml @@ -355,7 +355,7 @@ checkstyles.xml true warning - true + false true false false diff --git a/solr/.gitignore b/solr/.gitignore new file mode 100644 index 000000000..9ceaa0d58 --- /dev/null +++ b/solr/.gitignore @@ -0,0 +1 @@ +data*/ diff --git a/src/main/java/edu/tamu/scholars/middleware/discovery/annotation/FieldSource.java b/src/main/java/edu/tamu/scholars/middleware/discovery/annotation/FieldSource.java index 2b8cfefed..cf83f40bf 100644 --- a/src/main/java/edu/tamu/scholars/middleware/discovery/annotation/FieldSource.java +++ b/src/main/java/edu/tamu/scholars/middleware/discovery/annotation/FieldSource.java @@ -26,4 +26,19 @@ boolean split() default false; + CacheableLookup[] lookup() default {}; + + // allow to pass field source values into another template + // each result is appended to one set + @Documented + @Target(FIELD) + @Retention(RUNTIME) + public @interface CacheableLookup { + + String template(); + + String predicate(); + + } + } diff --git a/src/main/java/edu/tamu/scholars/middleware/discovery/argument/DiscoveryAcademicAgeDescriptor.java b/src/main/java/edu/tamu/scholars/middleware/discovery/argument/DiscoveryAcademicAgeDescriptor.java index 9354d868d..bb8fca4d3 100644 --- a/src/main/java/edu/tamu/scholars/middleware/discovery/argument/DiscoveryAcademicAgeDescriptor.java +++ b/src/main/java/edu/tamu/scholars/middleware/discovery/argument/DiscoveryAcademicAgeDescriptor.java @@ -4,6 +4,8 @@ import java.util.ArrayList; import java.util.List; +import edu.tamu.scholars.middleware.discovery.utility.DiscoveryUtility; + /** * */ @@ -31,7 +33,7 @@ private DiscoveryAcademicAgeDescriptor( Integer groupingIntervalInYears) { super(); this.label = label; - this.dateField = dateField; + this.dateField = DiscoveryUtility.findProperty(dateField); this.accumulateMultivaluedDate = accumulateMultivaluedDate; this.averageOverInterval = averageOverInterval; this.upperLimitInYears = upperLimitInYears; diff --git a/src/main/java/edu/tamu/scholars/middleware/discovery/argument/DiscoveryNetworkDescriptor.java b/src/main/java/edu/tamu/scholars/middleware/discovery/argument/DiscoveryNetworkDescriptor.java index eb5f2938d..beec6132d 100644 --- a/src/main/java/edu/tamu/scholars/middleware/discovery/argument/DiscoveryNetworkDescriptor.java +++ b/src/main/java/edu/tamu/scholars/middleware/discovery/argument/DiscoveryNetworkDescriptor.java @@ -11,6 +11,8 @@ import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.common.params.SolrParams; +import edu.tamu.scholars.middleware.discovery.utility.DiscoveryUtility; + /** * */ @@ -28,8 +30,8 @@ public class DiscoveryNetworkDescriptor { private DiscoveryNetworkDescriptor(String id, String dateField, List dataFields, String typeFilter) { super(); this.id = id; - this.dateField = dateField; - this.dataFields = dataFields; + this.dateField = DiscoveryUtility.findProperty(dateField); + this.dataFields = DiscoveryUtility.processFields(dataFields); this.typeFilter = typeFilter; } diff --git a/src/main/java/edu/tamu/scholars/middleware/discovery/argument/DiscoveryQuantityDistributionDescriptor.java b/src/main/java/edu/tamu/scholars/middleware/discovery/argument/DiscoveryQuantityDistributionDescriptor.java index 85bc13010..9ce46476d 100644 --- a/src/main/java/edu/tamu/scholars/middleware/discovery/argument/DiscoveryQuantityDistributionDescriptor.java +++ b/src/main/java/edu/tamu/scholars/middleware/discovery/argument/DiscoveryQuantityDistributionDescriptor.java @@ -1,5 +1,7 @@ package edu.tamu.scholars.middleware.discovery.argument; +import edu.tamu.scholars.middleware.discovery.utility.DiscoveryUtility; + /** * */ @@ -13,7 +15,7 @@ public class DiscoveryQuantityDistributionDescriptor { private DiscoveryQuantityDistributionDescriptor(String label, String field) { super(); this.label = label; - this.field = field; + this.field = DiscoveryUtility.findProperty(field); } public String getLabel() { diff --git a/src/main/java/edu/tamu/scholars/middleware/discovery/component/jena/TriplestoreHarvester.java b/src/main/java/edu/tamu/scholars/middleware/discovery/component/jena/TriplestoreHarvester.java index 1e901d8f0..11bf4e398 100644 --- a/src/main/java/edu/tamu/scholars/middleware/discovery/component/jena/TriplestoreHarvester.java +++ b/src/main/java/edu/tamu/scholars/middleware/discovery/component/jena/TriplestoreHarvester.java @@ -2,14 +2,15 @@ import static edu.tamu.scholars.middleware.discovery.DiscoveryConstants.NESTED_DELIMITER; +import java.lang.annotation.Annotation; import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.ParameterizedType; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -17,11 +18,11 @@ import org.apache.jena.graph.Triple; import org.apache.jena.query.QueryExecution; import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.Property; import org.apache.jena.rdf.model.ResIterator; import org.apache.jena.rdf.model.Resource; import org.apache.jena.rdf.model.Statement; import org.apache.jena.rdf.model.StmtIterator; -import org.apache.jena.shared.InvalidPropertyURIException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -29,10 +30,13 @@ import edu.tamu.scholars.middleware.discovery.annotation.CollectionSource; import edu.tamu.scholars.middleware.discovery.annotation.FieldSource; +import edu.tamu.scholars.middleware.discovery.annotation.FieldSource.CacheableLookup; import edu.tamu.scholars.middleware.discovery.annotation.FieldType; +import edu.tamu.scholars.middleware.discovery.annotation.NestedObject; import edu.tamu.scholars.middleware.discovery.component.Harvester; import edu.tamu.scholars.middleware.discovery.model.AbstractIndexDocument; import edu.tamu.scholars.middleware.discovery.model.Individual; +import edu.tamu.scholars.middleware.service.CacheService; import edu.tamu.scholars.middleware.service.TemplateService; import edu.tamu.scholars.middleware.service.Triplestore; @@ -57,18 +61,18 @@ public class TriplestoreHarvester implements Harvester { @Autowired private TemplateService templateService; + @Autowired + private CacheService cacheService; + private final Class type; - private final List propertySourceTypeOps; + private final List fields; private final List nestedFields; public TriplestoreHarvester(Class type) { this.type = type; - this.propertySourceTypeOps = FieldUtils.getFieldsListWithAnnotation(type, FieldSource.class) - .stream() - .map(this::getTypeOp) - .collect(Collectors.toList()); + this.fields = FieldUtils.getFieldsListWithAnnotation(type, FieldSource.class); this.nestedFields = FieldUtils.getFieldsListWithAnnotation(type, FieldType.class) .stream() .filter(this::isNestedField) @@ -103,8 +107,7 @@ public Class type() { } private Individual createIndividual(String subject) - throws InstantiationException, IllegalAccessException, IllegalArgumentException, - InvocationTargetException, NoSuchMethodException, SecurityException { + throws IllegalArgumentException, SecurityException { Individual individual = new Individual(); individual.setId(parse(subject)); lookupProperties(individual, subject); @@ -115,28 +118,85 @@ private Individual createIndividual(String subject) } private void lookupProperties(Individual individual, String subject) { - propertySourceTypeOps.parallelStream().forEach(typeOp -> { + fields.parallelStream().forEach(field -> { + FieldSource source = field.getAnnotation(FieldSource.class); try { - FieldSource source = typeOp.getPropertySource(); Model model = queryForModel(source, subject); - List values = lookupProperty(typeOp, source, model); + List values = lookupProperty(source, model); + + // get cacheable lookup + if (source.lookup().length > 0) { + Set uniqueValues = new HashSet<>(); + boolean isNestedObject = field.isAnnotationPresent(NestedObject.class); + for (CacheableLookup lookup : source.lookup()) { + FieldSource cacheableSource = getCacheableSource(source, lookup); + for (Object value : values) { + if (isNestedObject) { + String[] parts = ((String) value).split(NESTED_DELIMITER); + String[] ids = Arrays.copyOfRange(parts, 1, parts.length); + String[] refIds = ids.length > 1 ? Arrays.copyOfRange(ids, 0, ids.length - 1) : new String[] {}; + String newSubjectId = ids[ids.length - 1]; + String newSubject = subject.replaceAll("[^/]+$", newSubjectId); + for (Object cached : queryForValues(cacheableSource, newSubject)) { + String[] cachedParts = ((String) cached).split(NESTED_DELIMITER, 2); + String label = cachedParts[0]; + StringBuilder result = new StringBuilder(label); + if (refIds.length > 0) { + result.append(NESTED_DELIMITER); + result.append(String.join(NESTED_DELIMITER, refIds)); + } + if (cachedParts.length > 1) { + result.append(NESTED_DELIMITER); + result.append(cachedParts[1]); + } + + uniqueValues.add(result.toString()); + } + } else { + String newSubject = ((String) value); + for (Object cached : queryForValues(cacheableSource, newSubject)) { + uniqueValues.add(cached); + } + } + } + } + values = new ArrayList<>(uniqueValues); + } + if (!values.isEmpty()) { - populate(individual, typeOp.getField(), values); + populate(individual, field, values); } else { - logger.debug("Could not find values for {}", typeOp.getField().getName()); + logger.debug("Could not find values for {}", field.getName()); } } catch (Exception e) { logger.error( - "Unable to populate individual {} {}: {}", + "Unable to populate individual {} {}: {}\n{}", type.getSimpleName(), parse(subject), - e.getMessage() + e.getMessage(), + source.template() ); logger.debug("Error populating individual", e); } }); } + private List queryForValues(FieldSource source, String subject) { + String key = toKey(source, subject); + List values = cacheService.get(key); + if (Objects.isNull(values)) { + Model model = queryForModel(source, subject); + values = lookupProperty(source, model); + cacheService.put(key, values); + } + + return values; + } + + private String toKey(FieldSource source, String subject) { + return String.format("%s::%s::%s", source.predicate(), source.template(), subject); + } + private Model queryForModel(FieldSource source, String subject) { String query = templateService.templateSparql(source.template(), subject); logger.debug("{}:\n{}", source.template(), query); @@ -150,25 +210,23 @@ private Model queryForModel(FieldSource source, String subject) { } } - private List lookupProperty(TypeOp typeOp, FieldSource source, Model model) { + private List lookupProperty(FieldSource source, Model model) { List values = new ArrayList<>(); ResIterator resources = model.listSubjects(); + + Property property = model.createProperty(source.predicate()); + while (resources.hasNext()) { Resource resource = resources.next(); - values.addAll(queryForProperty(typeOp, source, model, resource)); + values.addAll(queryForProperty(source, resource, property)); } + return values; } - private List queryForProperty(TypeOp typeOp, FieldSource source, Model model, Resource resource) { + private List queryForProperty(FieldSource source, Resource resource, Property property) { List values = new ArrayList<>(); - StmtIterator statements; - try { - statements = resource.listProperties(model.createProperty(source.predicate())); - } catch (InvalidPropertyURIException e) { - logger.error("{} lookup by {}: {}", typeOp.getField().getName(), source.predicate(), e.getMessage()); - throw e; - } + StmtIterator statements = resource.listProperties(property); while (statements.hasNext()) { Statement statement = statements.next(); @@ -179,14 +237,12 @@ private List queryForProperty(TypeOp typeOp, FieldSource source, Model m value = value.substring(0, value.indexOf("^^")); } if (source.unique() && values.stream().map(v -> v.toString()).anyMatch(value::equalsIgnoreCase)) { - logger.debug("{} has duplicate value {}", typeOp.getField().getName(), value); + logger.debug("duplicate value {}", value); } else { if (source.split()) { - for (String v : value.split("\\|\\|")) { - values.add(typeOp.type(v)); - } + values.addAll(Arrays.asList(value.split("\\|\\|"))); } else { - values.add(typeOp.type(value)); + values.add(value); } } } @@ -198,7 +254,7 @@ private void populate( Individual document, Field field, List values - ) throws IllegalArgumentException, IllegalAccessException { + ) throws IllegalArgumentException { if (values.isEmpty()) { logger.debug("Could not find values for {}", field.getName()); } else { @@ -211,7 +267,7 @@ private void populate( } private void lookupSyncIds(Individual individual) { - Set syncIds = new HashSet(); + Set syncIds = new HashSet<>(); syncIds.add(individual.getId()); nestedFields.stream() .forEach(field -> { @@ -230,10 +286,7 @@ private void lookupSyncIds(Individual individual) { } private void addSyncId(Set syncIds, String value) { - String[] valueParts = value.split(NESTED_DELIMITER); - for (int i = 1; i < valueParts.length; i++) { - syncIds.add(valueParts[i]); - } + syncIds.addAll(Arrays.asList(value.split(NESTED_DELIMITER))); } private boolean isNestedField(Field field) { @@ -251,96 +304,43 @@ private String parse(String uri) { return uri.substring(uri.lastIndexOf(uri.contains(HASH_TAG) ? HASH_TAG : FORWARD_SLASH) + 1); } - private TypeOp getTypeOp(Field field) { - if (Collection.class.isAssignableFrom(field.getType())) { - ParameterizedType parameterizedType = (ParameterizedType) field.getGenericType(); - Class collectionType = (Class) parameterizedType.getActualTypeArguments()[0]; - if (String.class.isAssignableFrom(collectionType)) { - return new StringOp(field); - } else if (Integer.class.isAssignableFrom(collectionType)) { - return new IntegerOp(field); - } else if (Float.class.isAssignableFrom(collectionType)) { - return new FloatOp(field); - } else if (Double.class.isAssignableFrom(collectionType)) { - return new DoubleOp(field); + private FieldSource getCacheableSource(FieldSource source, CacheableLookup lookup) { + return new FieldSource() { + @Override + public String template() { + return lookup.template(); } - } else if (String.class.isAssignableFrom(field.getType())) { - return new StringOp(field); - } else if (Integer.class.isAssignableFrom(field.getType())) { - return new IntegerOp(field); - } else if (Float.class.isAssignableFrom(field.getType())) { - return new FloatOp(field); - } else if (Double.class.isAssignableFrom(field.getType())) { - return new DoubleOp(field); - } - return new StringOp(field); - } - - private interface TypeOp { - public Object type(String value); - - public Field getField(); - - public FieldSource getPropertySource(); - } - - private abstract class AbstractTypeOp implements TypeOp { - private final Field field; - private final FieldSource source; - - public AbstractTypeOp(Field field) { - this.field = field; - this.source = field.getAnnotation(FieldSource.class); - } - - public Field getField() { - return field; - } - - public FieldSource getPropertySource() { - return source; - } - } - - private class StringOp extends AbstractTypeOp { - public StringOp(Field field) { - super(field); - } - - public Object type(String value) { - return value; - } - } - - private class IntegerOp extends AbstractTypeOp { - public IntegerOp(Field field) { - super(field); - } + @Override + public String predicate() { + return lookup.predicate(); + } - public Object type(String value) { - return Integer.parseInt(value); - } - } + @Override + public boolean parse() { + return source.parse(); + } - private class FloatOp extends AbstractTypeOp { - public FloatOp(Field field) { - super(field); - } + @Override + public boolean unique() { + return source.unique(); + } - public Object type(String value) { - return Float.parseFloat(value); - } - } + @Override + public boolean split() { + return source.split(); + } - private class DoubleOp extends AbstractTypeOp { - public DoubleOp(Field field) { - super(field); - } + @Override + public CacheableLookup[] lookup() { + return new CacheableLookup[] {}; + } - public Object type(String value) { - return Double.parseDouble(value); - } + @Override + public Class annotationType() { + throw new UnsupportedOperationException("Unimplemented method 'annotationType'"); + } + }; } } diff --git a/src/main/java/edu/tamu/scholars/middleware/discovery/indicator/IndexHealthIndicator.java b/src/main/java/edu/tamu/scholars/middleware/discovery/indicator/IndexHealthIndicator.java index ea3d76e6d..c4b0e01ce 100644 --- a/src/main/java/edu/tamu/scholars/middleware/discovery/indicator/IndexHealthIndicator.java +++ b/src/main/java/edu/tamu/scholars/middleware/discovery/indicator/IndexHealthIndicator.java @@ -3,8 +3,8 @@ import static edu.tamu.scholars.middleware.discovery.DiscoveryConstants.DEFAULT_QUERY; import java.io.IOException; +import java.util.Arrays; import java.util.HashMap; -import java.util.List; import java.util.Map; import org.apache.solr.client.solrj.SolrClient; @@ -57,7 +57,7 @@ public Health health() { if (response.getStatus() == 0 && message.equals("OK")) { - long count = individualRepo.count(DEFAULT_QUERY, List.of()); + long count = individualRepo.count(DEFAULT_QUERY, Arrays.asList()); details.put("count", count); diff --git a/src/main/java/edu/tamu/scholars/middleware/discovery/model/Document.java b/src/main/java/edu/tamu/scholars/middleware/discovery/model/Document.java index 58bad8ac0..c1bdaa420 100644 --- a/src/main/java/edu/tamu/scholars/middleware/discovery/model/Document.java +++ b/src/main/java/edu/tamu/scholars/middleware/discovery/model/Document.java @@ -10,6 +10,7 @@ import edu.tamu.scholars.middleware.discovery.annotation.CollectionSource; import edu.tamu.scholars.middleware.discovery.annotation.FieldSource; +import edu.tamu.scholars.middleware.discovery.annotation.FieldSource.CacheableLookup; import edu.tamu.scholars.middleware.discovery.annotation.FieldType; import edu.tamu.scholars.middleware.discovery.annotation.NestedMultiValuedProperty; import edu.tamu.scholars.middleware.discovery.annotation.NestedObject; @@ -93,7 +94,13 @@ public class Document extends Common { @FieldType(type = "nested_whole_strings") @FieldSource( template = "document/authorOrganization", - predicate = "http://www.w3.org/2000/01/rdf-schema#label" + predicate = "http://www.w3.org/2000/01/rdf-schema#label", + lookup = { + @CacheableLookup( + template = "document/authorOrganizations", + predicate = "http://www.w3.org/2000/01/rdf-schema#label" + ) + } ) private List authorOrganization; diff --git a/src/main/java/edu/tamu/scholars/middleware/discovery/model/Person.java b/src/main/java/edu/tamu/scholars/middleware/discovery/model/Person.java index 21b906ace..51d1d6b28 100644 --- a/src/main/java/edu/tamu/scholars/middleware/discovery/model/Person.java +++ b/src/main/java/edu/tamu/scholars/middleware/discovery/model/Person.java @@ -8,6 +8,7 @@ import edu.tamu.scholars.middleware.discovery.annotation.CollectionSource; import edu.tamu.scholars.middleware.discovery.annotation.FieldSource; +import edu.tamu.scholars.middleware.discovery.annotation.FieldSource.CacheableLookup; import edu.tamu.scholars.middleware.discovery.annotation.FieldType; import edu.tamu.scholars.middleware.discovery.annotation.NestedMultiValuedProperty; import edu.tamu.scholars.middleware.discovery.annotation.NestedObject; @@ -1204,7 +1205,13 @@ public class Person extends Common { @FieldType(type = "whole_strings") @FieldSource( template = "person/organization", - predicate = "http://www.w3.org/2000/01/rdf-schema#label" + predicate = "http://vivoweb.org/ontology/core#relates", + lookup = { + @CacheableLookup( + template = "person/organizations", + predicate = "http://www.w3.org/2000/01/rdf-schema#label" + ) + } ) private List organizations; diff --git a/src/main/java/edu/tamu/scholars/middleware/discovery/model/repo/IndividualRepo.java b/src/main/java/edu/tamu/scholars/middleware/discovery/model/repo/IndividualRepo.java index f6ab6f1a4..652ac9e4d 100644 --- a/src/main/java/edu/tamu/scholars/middleware/discovery/model/repo/IndividualRepo.java +++ b/src/main/java/edu/tamu/scholars/middleware/discovery/model/repo/IndividualRepo.java @@ -14,6 +14,7 @@ import java.util.Calendar; import java.util.Date; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; import java.util.regex.Matcher; @@ -407,8 +408,8 @@ private Individual getById(String id) { try { SolrDocument document = solrClient.getById(collectionName, id); - return Individual.from(document); - } catch (IOException | SolrServerException e) { + return Objects.nonNull(document) ? Individual.from(document) : null; + } catch (IOException | NullPointerException | SolrServerException e) { throw new SolrRequestException("Failed to get document by id", e); } } diff --git a/src/main/java/edu/tamu/scholars/middleware/discovery/response/DiscoveryAcademicAge.java b/src/main/java/edu/tamu/scholars/middleware/discovery/response/DiscoveryAcademicAge.java index 2ecf3d32e..4d934cca7 100644 --- a/src/main/java/edu/tamu/scholars/middleware/discovery/response/DiscoveryAcademicAge.java +++ b/src/main/java/edu/tamu/scholars/middleware/discovery/response/DiscoveryAcademicAge.java @@ -53,10 +53,9 @@ public void from(DiscoveryAcademicAgeDescriptor academicAgeDescriptor, SolrDocum List labeledRanges = academicAgeDescriptor.getLabeledRanges(); - AtomicInteger sum = new AtomicInteger(0); - AtomicInteger total = new AtomicInteger(0); + int sum = 0; - labeledRanges.parallelStream().forEach(lr -> { + for (LabeledRange lr : labeledRanges) { int subtotal = 0; @@ -70,7 +69,7 @@ public void from(DiscoveryAcademicAgeDescriptor academicAgeDescriptor, SolrDocum boolean inRange = false; if (lr.isFirst) { - sum.getAndAdd(age); + sum += age; inRange = age < lr.to; } else if (lr.isLast) { inRange = age >= lr.from; @@ -90,19 +89,16 @@ public void from(DiscoveryAcademicAgeDescriptor academicAgeDescriptor, SolrDocum } } - total.addAndGet(subtotal); - Integer value = academicAgeDescriptor.getAverageOverInterval() && set.size() > 0 ? subtotal / set.size() : subtotal; add(lr.index, lr.range, lr.label, value); - - }); + } Collections.sort(groups, new AgeGroupComparator()); - this.mean = results.size() > 0 ? sum.get() / results.size() : 0; + this.mean = results.size() > 0 ? sum / results.size() : 0; this.median = results.size() > 0 ? DateUtility.ageInYearsFromEpochSecond((long) results.get(results.size() / 2).getFieldValue(ageField)) : 0; diff --git a/src/main/java/edu/tamu/scholars/middleware/discovery/utility/DiscoveryUtility.java b/src/main/java/edu/tamu/scholars/middleware/discovery/utility/DiscoveryUtility.java index 308ba434a..e967d3fd9 100644 --- a/src/main/java/edu/tamu/scholars/middleware/discovery/utility/DiscoveryUtility.java +++ b/src/main/java/edu/tamu/scholars/middleware/discovery/utility/DiscoveryUtility.java @@ -87,11 +87,15 @@ public static Map> getDiscoveryDocumentTypeFields(String name) } public static String[] processFields(String[] fields) { - return Arrays.asList(fields) + return processFields(Arrays.asList(fields)) + .toArray(new String[fields.length]); + } + + public static List processFields(List fields) { + return fields .stream() .map(DiscoveryUtility::findProperty) - .collect(Collectors.toList()) - .toArray(new String[fields.length]); + .collect(Collectors.toList()); } public static String processFields(String fields) { diff --git a/src/main/java/edu/tamu/scholars/middleware/service/CacheService.java b/src/main/java/edu/tamu/scholars/middleware/service/CacheService.java new file mode 100644 index 000000000..1d8a7f30d --- /dev/null +++ b/src/main/java/edu/tamu/scholars/middleware/service/CacheService.java @@ -0,0 +1,44 @@ +package edu.tamu.scholars.middleware.service; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.springframework.stereotype.Service; + +/** + * A cache service autowired in {@see edu.tamu.scholars.middleware.discovery.component.TriplestoreHarvester}. + */ +@Service +public class CacheService { + + private static final int MAX_CAPACITY = 7000; + + private final Map> cache; + + public CacheService() { + cache = new ConcurrentHashMap<>(MAX_CAPACITY); + } + + /** + * Get list of values or null if not found. + * + * @param key key to lookup in cache + * @return See the {@see java.util.concurrent.ConcurrentHashMap#get} + */ + public List get(String key) { + return cache.get(key); + } + + /** + * Put into the cache and return previous list of values for key. + * + * @param key key to lookup in cache + * @param values current values for key to store + * @return See the {@see java.util.concurrent.ConcurrentHashMap#put} + */ + public List put(String key, List values) { + return cache.put(key, values); + } + +} diff --git a/src/main/resources/defaults/dataAndanalyticViews.yml b/src/main/resources/defaults/dataAndanalyticViews.yml index 196a4fc00..bfb334ae9 100644 --- a/src/main/resources/defaults/dataAndanalyticViews.yml +++ b/src/main/resources/defaults/dataAndanalyticViews.yml @@ -57,19 +57,9 @@ collapsed: false useDialog: true hidden: false - - name: Start Year - field: publicationDate - type: DATE_YEAR - sort: INDEX - direction: ASC - pageSize: 10 - expandable: false - collapsible: false - collapsed: false - useDialog: true - hidden: false - - name: End Year + - name: Year field: publicationDate + opKey: BETWEEN type: DATE_YEAR sort: INDEX direction: DESC diff --git a/src/main/resources/defaults/directoryViews.yml b/src/main/resources/defaults/directoryViews.yml index a3953c830..d9b993af9 100644 --- a/src/main/resources/defaults/directoryViews.yml +++ b/src/main/resources/defaults/directoryViews.yml @@ -210,6 +210,9 @@ filters: - field: class value: Document + - field: title + value: '*' + opKey: EXPRESSION boosts: [] sort: - field: title_sort diff --git a/src/main/resources/defaults/discoveryViews.yml b/src/main/resources/defaults/discoveryViews.yml index 5887bcffa..67bb1764b 100644 --- a/src/main/resources/defaults/discoveryViews.yml +++ b/src/main/resources/defaults/discoveryViews.yml @@ -303,6 +303,9 @@ - field: class value: Document OR type:creativeWork opKey: EXPRESSION + - field: title + value: '*' + opKey: EXPRESSION boosts: - field: title value: 2 diff --git a/src/main/resources/defaults/displayViews/persons/asideTemplate.html b/src/main/resources/defaults/displayViews/persons/asideTemplate.html index 6dee3a40e..effa24b3a 100644 --- a/src/main/resources/defaults/displayViews/persons/asideTemplate.html +++ b/src/main/resources/defaults/displayViews/persons/asideTemplate.html @@ -94,6 +94,20 @@ {{/if}} + {{#if twitter}} +
+
+
+ +
+ +
+ {{/if}} + {{#if futureResearchIdeas}}
@@ -152,4 +166,8 @@ height: 25px; margin: auto; } + + .x-logo svg { + height: 23px; + } diff --git a/src/main/resources/templates/sparql/document/authorOrganization.sparql b/src/main/resources/templates/sparql/document/authorOrganization.sparql index 60086c88f..de3e71373 100644 --- a/src/main/resources/templates/sparql/document/authorOrganization.sparql +++ b/src/main/resources/templates/sparql/document/authorOrganization.sparql @@ -15,7 +15,5 @@ CONSTRUCT { ?position vivo:relatedBy ?organization . ?organization a foaf:Organization . ?organization rdfs:label ?label . - ?organization vitro:mostSpecificType ?mostSpecificType . BIND( CONCAT( STR(?label), REPLACE(STR(?author), "(^.*/)", "::"), REPLACE(STR(?organization), "(^.*/)", "::") ) AS ?labelWithId ) . - FILTER(lang(?label) = '' && (?mostSpecificType = vivo:AcademicDepartment || ?mostSpecificType = vivo:Center || ?mostSpecificType = vivo:College || ?mostSpecificType = vivo:Institute || ?mostSpecificType = vivo:Laboratory || ?mostSpecificType = vivo:Library || ?mostSpecificType = vivo:Program || ?mostSpecificType = vivo:School || ?mostSpecificType = vivo:University)) } diff --git a/src/main/resources/templates/sparql/document/authorOrganizations.sparql b/src/main/resources/templates/sparql/document/authorOrganizations.sparql new file mode 100644 index 000000000..453f157bc --- /dev/null +++ b/src/main/resources/templates/sparql/document/authorOrganizations.sparql @@ -0,0 +1,18 @@ +PREFIX obo: +PREFIX vivo: +PREFIX vitro: +PREFIX foaf: +PREFIX rdfs: +PREFIX rdf: + +CONSTRUCT { + ?organization rdfs:label ?labelWithId . +} WHERE { + BIND(<{{uri}}> as ?baseOrg) + + ?baseOrg (obo:BFO_0000050)* ?organization . + ?organization a foaf:Organization . + ?organization rdfs:label ?label . + + BIND( CONCAT( STR(?label), REPLACE(STR(?organization), "(^.*/)", "::") ) AS ?labelWithId ) . +} diff --git a/src/main/resources/templates/sparql/person/capstoneAdvisedOfURL.sparql b/src/main/resources/templates/sparql/person/capstoneAdvisedOfUrl.sparql similarity index 100% rename from src/main/resources/templates/sparql/person/capstoneAdvisedOfURL.sparql rename to src/main/resources/templates/sparql/person/capstoneAdvisedOfUrl.sparql diff --git a/src/main/resources/templates/sparql/person/etdChairOfURL.sparql b/src/main/resources/templates/sparql/person/etdChairOfUrl.sparql similarity index 100% rename from src/main/resources/templates/sparql/person/etdChairOfURL.sparql rename to src/main/resources/templates/sparql/person/etdChairOfUrl.sparql diff --git a/src/main/resources/templates/sparql/person/organization.sparql b/src/main/resources/templates/sparql/person/organization.sparql index 31184ea38..02c17a2da 100644 --- a/src/main/resources/templates/sparql/person/organization.sparql +++ b/src/main/resources/templates/sparql/person/organization.sparql @@ -1,13 +1,14 @@ +PREFIX obo: PREFIX vivo: PREFIX rdfs: PREFIX foaf: CONSTRUCT { - ?organization rdfs:label ?label . -} WHERE { + <{{uri}}> vivo:relates ?organization . +} +WHERE { <{{uri}}> vivo:relatedBy ?position . - ?position a vivo:Position . + ?position a vivo:Position . ?position vivo:relates ?organization . ?organization a foaf:Organization . - ?organization rdfs:label ?label . -} +} diff --git a/src/main/resources/templates/sparql/person/organizations.sparql b/src/main/resources/templates/sparql/person/organizations.sparql new file mode 100644 index 000000000..c592043c8 --- /dev/null +++ b/src/main/resources/templates/sparql/person/organizations.sparql @@ -0,0 +1,14 @@ +PREFIX obo: +PREFIX rdfs: +PREFIX foaf: + +CONSTRUCT { + ?organization rdfs:label ?label . +} +WHERE { + BIND(<{{uri}}> as ?baseOrg) + + ?baseOrg (obo:BFO_0000050)* ?organization . + ?organization a foaf:Organization . + ?organization rdfs:label ?label . +} diff --git a/src/main/resources/templates/sparql/relationship/endDateTime.sparql b/src/main/resources/templates/sparql/relationship/endDateTime.sparql index d42ef82bc..f33137e1f 100644 --- a/src/main/resources/templates/sparql/relationship/endDateTime.sparql +++ b/src/main/resources/templates/sparql/relationship/endDateTime.sparql @@ -4,6 +4,6 @@ CONSTRUCT { ?dateTimeStart vivo:dateTime ?dateTime . } WHERE { <{{uri}}> vivo:dateTimeInterval ?dateTimeInterval . - ?dateTimeInterval vivo:start ?dateTimeStart . + ?dateTimeInterval vivo:end ?dateTimeStart . ?dateTimeStart vivo:dateTime ?dateTime . } diff --git a/src/main/resources/templates/sparql/relationship/startDateTime.sparql b/src/main/resources/templates/sparql/relationship/startDateTime.sparql index c06f68a46..f6350b4fd 100644 --- a/src/main/resources/templates/sparql/relationship/startDateTime.sparql +++ b/src/main/resources/templates/sparql/relationship/startDateTime.sparql @@ -4,6 +4,6 @@ CONSTRUCT { ?dateTimeEnd vivo:dateTime ?dateTime . } WHERE { <{{uri}}> vivo:dateTimeInterval ?dateTimeInterval . - ?dateTimeInterval vivo:end ?dateTimeEnd . + ?dateTimeInterval vivo:start ?dateTimeEnd . ?dateTimeEnd vivo:dateTime ?dateTime . }