Merge branch 'feature/typenormalization' of github.com:libris/librisxl into feature/typenormalization

Author: branchedelac

gradle.properties

@@ -12,4 +12,5 @@ httpComponentsCoreVersion = 5.3.6
 httpComponentsClientVersion = 5.5
 systemProp.https.protocols=TLSv1.1,TLSv1.2
 guavaVersion=33.5.0-jre
-jacocoVersion=0.8.11
+jacocoVersion=0.8.11
+jacksonVersion=2.20.1

housekeeping/src/main/groovy/whelk/housekeeping/NotificationGenerator.groovy

@@ -1,6 +1,6 @@
 package whelk.housekeeping
 
-import org.codehaus.jackson.map.JsonMappingException
+import com.fasterxml.jackson.databind.JsonMappingException
 import whelk.Document
 import whelk.IdGenerator
 import whelk.JsonLd

importers/build.gradle

@@ -71,7 +71,7 @@ dependencies {
 
     // Common tools
     implementation "org.apache.groovy:groovy-all:${groovyVersion}"
-    implementation 'org.codehaus.jackson:jackson-mapper-asl:1.9.12'
+    implementation "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}"
 
     // Integration
     implementation 'mysql:mysql-connector-java:8.0.17'

importers/src/main/groovy/whelk/reindexer/ElasticReindexer.groovy

@@ -7,10 +7,10 @@ import whelk.util.BlockingThreadPool
 
 @Log
 class ElasticReindexer {
-
     static final int BATCH_SIZE = 300
     static final int MAX_RETRIES = 5
-    static final int RETRY_WAIT_MS = 3000
+    static final int INITIAL_RETRY_WAIT_MS = 3000
+    static final int MAX_RETRY_WAIT_MS = 60000
 
     Whelk whelk
 
@@ -106,12 +106,14 @@ class ElasticReindexer {
 
     private void bulkIndexWithRetries(List<Document> docs, Whelk whelk) {
         int retriesLeft = MAX_RETRIES
+        int retryCount = 0
 
         Exception error
         while(error = tryBulkIndex(docs, whelk)) {
             if (retriesLeft-- > 0) {
-                log.warn("Failed to index batch: [${error}], retrying after ${RETRY_WAIT_MS} ms")
-                sleep()
+                int waitMs = calculateBackoffMs(retryCount++)
+                log.warn("Failed to index batch: [${error}], retrying after ${waitMs} ms")
+                sleep(waitMs)
             } else {
                 log.warn("Failed to index batch: [${error}], max retries exceeded")
                 throw error
@@ -129,9 +131,16 @@ class ElasticReindexer {
         }
     }
 
-    private void sleep() {
+    private int calculateBackoffMs(int retryCount) {
+        int backoffMs = (int) (Math.pow(2, retryCount) * INITIAL_RETRY_WAIT_MS)
+        int cappedBackoffMs = Math.min(backoffMs, MAX_RETRY_WAIT_MS)
+        int jitter = (int) (Math.random() * cappedBackoffMs * 0.2)
+        return cappedBackoffMs + jitter
+    }
+
+    private void sleep(int ms) {
         try {
-            Thread.sleep(RETRY_WAIT_MS)
+            Thread.sleep(ms)
         }
         catch (InterruptedException e) {
             log.warn("Woke up early", e)

librisxl-tools/examples/OaiPmhToJsonLdDataset.groovy

@@ -1,5 +1,5 @@
 import groovy.xml.XmlUtil
-import org.codehaus.jackson.map.ObjectMapper
+import com.fasterxml.jackson.databind.ObjectMapper
 import whelk.importer.BasicOaiPmhImporter
 import whelk.converter.MarcJSONConverter
 import whelk.plugin.JsonLdToTurtle

rest/build.gradle

@@ -78,8 +78,8 @@ dependencies {
     implementation "org.apache.httpcomponents.core5:httpcore5:${httpComponentsCoreVersion}"
     implementation 'commons-cli:commons-cli:1.2'
     implementation 'commons-io:commons-io:2.20.0'
-    implementation 'org.codehaus.jackson:jackson-core-asl:1.9.13'
-    implementation 'org.codehaus.jackson:jackson-mapper-asl:1.9.13'
+    implementation "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}"
+    implementation "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}"
     implementation 'xml-apis:xml-apis:1.4.01'
 
     // metrics

rest/src/main/groovy/se/kb/libris/digi/DigitalReproductionAPI.groovy

@@ -2,8 +2,8 @@ package se.kb.libris.digi
 
 import groovy.transform.MapConstructor
 import groovy.util.logging.Log4j2 as Log
-import org.codehaus.jackson.JsonParseException
-import org.codehaus.jackson.map.ObjectMapper
+import com.fasterxml.jackson.core.JsonParseException
+import com.fasterxml.jackson.databind.ObjectMapper
 import whelk.Configuration
 
 import javax.servlet.ServletException

rest/src/main/groovy/whelk/rest/api/ConverterAPI.java

@@ -3,7 +3,7 @@
 import org.apache.hc.core5.http.ContentType;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.codehaus.jackson.map.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import whelk.Document;
 import whelk.Whelk;
 import whelk.converter.marc.MarcFrameConverter;

rest/src/main/groovy/whelk/rest/api/Crud.java

@@ -10,10 +10,8 @@
 import whelk.JsonLdValidator;
 import whelk.TargetVocabMapper;
 import whelk.Whelk;
-import whelk.component.PostgreSQLComponent;
 import whelk.exception.ElasticIOException;
 import whelk.exception.InvalidQueryException;
-import whelk.exception.StaleUpdateException;
 import whelk.exception.UnexpectedHttpStatusException;
 import whelk.exception.WhelkRuntimeException;
 import whelk.history.History;
@@ -35,8 +33,6 @@
 import java.lang.management.ManagementFactory;
 import java.net.URI;
 import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.stream.Collectors;
 
 import static whelk.rest.api.CrudUtils.ETag;
 import static whelk.util.Jackson.mapper;
@@ -160,6 +156,9 @@ public void handleQuery(HttpServletRequest request, HttpServletResponse response
         } catch (InvalidQueryException e) {
             log.info("Invalid query: {}", queryParameters);
             throw new BadRequestException("Invalid query, please check the documentation. " + e.getMessage());
+        } catch (IOException e) {
+            log.error("IOException during query: {}", e.toString(), e);
+            throw new WhelkRuntimeException("IOException during query.", e);
         }
     }