Feature/libris search ns

importers/src/main/groovy/whelk/importer/ImporterMain.groovy

@@ -31,7 +31,7 @@ class ImporterMain {
     }
 
     @Command(args='SOURCE_URL DATASET_URI [DATASET_DESCRIPTION_FILE]',
-             flags='--skip-index --replace-main-ids --force-delete --skip-dependers')
+             flags='--skip-index --replace-main-ids --force-delete --skip-dependers --allow-id-removal')
     void dataset(Map flags, String sourceUrl, String datasetUri, String datasetDescPath=null) {
         Whelk whelk = Whelk.createLoadedSearchWhelk(props)
         if (flags['skip-index']) {
@@ -40,6 +40,9 @@ class ImporterMain {
         if (flags['skip-dependers']) {
             whelk.setSkipIndexDependers(true)
         }
+        if (flags['allow-id-removal']) {
+            whelk.storage.doVerifyDocumentIdRetention = false
+        }
         new DatasetImporter(whelk, datasetUri, flags, datasetDescPath).importDataset(sourceUrl)
     }
 

librisxl-tools/elasticsearch/libris_search_boost.json

@@ -84,6 +84,16 @@
           "missing": 0
         },
         "weight": 10
+      },
+      {
+        "type": "field_value_factor",
+        "params": {
+          "field": "reverseLinks.totalItemsByRelation.instanceOf.language",
+          "factor": 10,
+          "modifier": "ln1p",
+          "missing": 0
+        },
+        "weight": 10
       }
     ],
     "score_mode": "sum",

rest/build.gradle

@@ -125,7 +125,8 @@ task(appRun, dependsOn: "classes", type: JavaExec) {
             'xl.secret.properties': System.getProperty("xl.secret.properties"),
             'xl.logRoot': System.getProperty("xl.logRoot", "./logs"),
             'xl.http.port': System.getProperty("xl.http.port", "8180"),
-            'xl.test.rewriteSparqlResultIds': System.getProperty("xl.test.rewriteSparqlResultIds", "false")
+            'xl.test.rewriteSparqlResultIds': System.getProperty("xl.test.rewriteSparqlResultIds", "false"),
+            'xl.test.apps.jsonld': System.getProperty("xl.test.apps.jsonld")
     )
     args(System.getProperty("args", "").split() as String[])
 

rest/src/main/groovy/se/kb/libris/digi/DigitalReproductionAPI.groovy

@@ -34,11 +34,13 @@ import static javax.servlet.http.HttpServletResponse.SC_PRECONDITION_FAILED
 import static se.kb.libris.digi.DigitalReproductionAPI.Type.ARRAY
 import static se.kb.libris.digi.DigitalReproductionAPI.Type.STRING
 
+// TODO clean up digital vs electronic when type normalization has landed in production
+
 /**
  Creates a record for a digital reproduction.
- Takes JSON-LD with an Electronic describing the reproduction as input.
+ Takes JSON-LD with a DigitalResource/Electronic describing the reproduction as input.
 
- - Validates that minimal required data is present in Electronic (all additional data is kept).
+ - Validates that minimal required data is present in DigitalResource/Electronic (all additional data is kept).
  - Extracts and links work entity from reproduction and original (physical thing)
  - Copies title from original
  - Adds DIGI and DST bibliographies if applicable
@@ -127,7 +129,12 @@ class DigitalReproductionAPI extends HttpServlet {
 
         def electronic = readJson(request)
 
-        check(electronic, ['@type'], 'Electronic')
+        try {
+            check(electronic, ['@type'], 'Electronic')
+        } catch (RequestException ignored) {
+            check(electronic, ['@type'], 'DigitalResource')
+        }
+
         check(electronic, ['reproductionOf', '@id'], STRING)
         check(electronic, ['production'], ARRAY)
         if (!isLink(getAtPath(electronic, ['production', 0]))) { // minimal valid shape, so just check the first one 
@@ -203,6 +210,9 @@ class ReproductionService {
         if (physicalThing['@type'] == 'Electronic') {
             throw badRequest("Thing linked in reproductionOf cannot be Electronic")
         }
+        if (physicalThing['@type'] == 'DigitalResource') {
+            throw badRequest("Thing linked in reproductionOf cannot be DigitalResource")
+        }
 
         def holdingsToCreate = getAtPath(electronicThing, ['@reverse', 'itemOf'], [])
         def badHeldBy = holdingsToCreate
@@ -227,7 +237,11 @@ class ReproductionService {
         }
 
         if (isOnline(electronicThing)) {
-            electronicThing.carrierType = asSet(electronicThing.carrierType) << ONLINE
+            if (electronicThing['@type'] == 'DigitalResource') {
+                electronicThing.category = asSet(electronicThing.category) << ONLINE
+            } else {
+                electronicThing.carrierType = asSet(electronicThing.carrierType) << ONLINE
+            }
         }
 
         def record = asMap(electronicThing.remove('meta'))

rest/src/main/groovy/whelk/rest/api/RefreshAPI.java

@@ -13,6 +13,7 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import static whelk.FeatureFlags.Flag.INDEX_BLANK_WORKS;
 import static whelk.util.Jackson.mapper;
 
 /**
@@ -110,5 +111,10 @@ void refreshLoudly(Document doc) {
     void refreshQuietly(Document doc) {
         whelk.getStorage().refreshDerivativeTables(doc);
         whelk.elastic.index(doc, whelk);
+        if (whelk.getFeatures().isEnabled(INDEX_BLANK_WORKS)) {
+            for (var id : doc.getVirtualRecordIds()) {
+                whelk.elastic.index(doc.getVirtualRecord(id), whelk);
+            }
+        }
     }
 }

rest/src/main/groovy/whelk/rest/api/SearchUtils2.java

@@ -25,7 +25,7 @@ public class SearchUtils2 {
     SearchUtils2(Whelk whelk) {
         this.whelk = whelk;
         this.esSettings = new ESSettings(whelk);
-        this.vocabMappings = new VocabMappings(whelk);
+        this.vocabMappings = VocabMappings.load(whelk);
     }
 
     Map<String, Object> doSearch(Map<String, String[]> queryParameters) throws InvalidQueryException, IOException {
@@ -34,7 +34,7 @@ Map<String, Object> doSearch(Map<String, String[]> queryParameters) throws Inval
         }
 
         QueryParams queryParams = new QueryParams(queryParameters);
-        AppParams appParams = new AppParams(getAppConfig(queryParameters), queryParams);
+        AppParams appParams = new AppParams(getAppConfig(queryParameters), whelk.getJsonld());
 
         Query query = Query.init(queryParams, appParams, vocabMappings, esSettings, whelk);
 
@@ -59,40 +59,12 @@ private Map<String, Object> getAppConfig(Map<String, String[]> queryParameters)
     public Map<String, Object> buildAppConfig(Map<String, Object> findDesc) {
         Map<String, Object> config = new LinkedHashMap<>();
 
-        Optional.ofNullable((Map<?, ?>) findDesc.get("statistics"))
-                .map(s -> (List<?>) s.get("sliceList"))
-                .map(SearchUtils2::buildStatsReprFromSliceSpec)
-                .ifPresent(statsRepr -> config.put("_statsRepr", statsRepr));
-
-        Stream.of("filterAliases", "defaultSiteFilters", "optionalSiteFilters", "relationFilters")
+        Stream.of("statistics", "filterAliases", "defaultSiteFilters", "optionalSiteFilters", "relationFilters")
                 .forEach(key ->
                         Optional.ofNullable(findDesc.get(key))
-                                .ifPresent(filters -> config.put("_" + key, filters))
+                                .ifPresent(filters -> config.put(key, filters))
                 );
 
         return config;
     }
-
-    private static Map<String, Object> buildStatsReprFromSliceSpec(List<?> sliceList) {
-        Map<String, Object> statsRepr = new LinkedHashMap<>();
-        for (var s : sliceList) {
-            var slice = ((Map<?, ?>) s);
-            String key = (String) ((List<?>) slice.get("dimensionChain")).getFirst();
-            int limit = (Integer) slice.get("itemLimit");
-            Boolean range = (Boolean) slice.get("range");
-            String connective = (String) slice.get("connective");
-            var m = new HashMap<>();
-            m.put("sort", "count");
-            m.put("sortOrder", "desc");
-            m.put("size", limit);
-            if (range != null) {
-                m.put("range", range);
-            }
-            if (connective != null) {
-                m.put("connective", connective);
-            }
-            statsRepr.put(key, m);
-        }
-        return statsRepr;
-    }
 }

rest/src/main/groovy/whelk/rest/api/SiteSearch.java

@@ -1,12 +1,15 @@
 package whelk.rest.api;
 
+import com.google.common.io.Files;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import whelk.JsonLd;
 import whelk.Whelk;
 import whelk.exception.InvalidQueryException;
 
+import java.io.File;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -70,10 +73,26 @@ protected synchronized void setupApplicationSearchData() {
         }
     }
 
-    protected Map getAndIndexDescription(String id) {
-        Map data = whelk.loadData(id);
+    protected Map<?, ?> getAndIndexDescription(String id) {
+        Map<?, ?> data;
+
+        var appsOverride = System.getProperty("xl.test.apps.jsonld");
+        if (appsOverride != null && !appsOverride.isEmpty()) {
+            log.info("Using {} for {}", appsOverride, id);
+            try {
+                data = mapper.readValue(
+                        Files.asCharSource(new File(appsOverride), StandardCharsets.UTF_8).read(),
+                        Map.class
+                );
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        } else {
+            data = whelk.loadData(id);
+        }
+
         if (data != null) {
-            Map desc = (Map) findInData(data, id);
+            var desc = (Map<?, ?>) findInData(data, id);
             if (desc != null) {
                 appsIndex.put(id, desc);
                 return desc;

sru/src/main/java/whelk/sru/servlet/SruServlet.java

@@ -75,7 +75,7 @@ public void doGet(HttpServletRequest req, HttpServletResponse res) throws IOExce
             paramsAsIfSearch.put("_q", q);
             paramsAsIfSearch.put("_stats", new String[] { "false" }); // don't need facets
             QueryParams qp = new QueryParams(paramsAsIfSearch);
-            AppParams ap = new AppParams(new HashMap<>(), qp);
+            AppParams ap = new AppParams(new HashMap<>(), whelk.getJsonld());
             Query query = new Query(qp, ap, vocabMappings, esSettings, whelk);
             results = query.collectResults();
         } catch (InvalidQueryException e) {

sru/src/main/java/whelk/sru/servlet/XSearchServlet.java

@@ -213,7 +213,7 @@ public void doGet2(HttpServletRequest req, HttpServletResponse res) throws IOExc
             }
 
             QueryParams qp = new QueryParams(paramsAsIfSearch);
-            AppParams ap = new AppParams(new HashMap<>(), qp);
+            AppParams ap = new AppParams(new HashMap<>(), whelk.getJsonld());
             var results = new Query(qp, ap, vocabMappings, esSettings, whelk).collectResults();
 
             @SuppressWarnings("unchecked")