Upgrade to Solr 9.1.0 (#2580)

Co-authored-by: Ere Maijala <[email protected]>

build.xml

@@ -45,7 +45,7 @@
   <property name="snooze_multiplier" value="1" /><!-- can be used to slow down tests (selenium only) -->
   <property name="solr_startup_sleep" value="0" />
   <property name="solr_additional_jvm_options" value="-Dlog4j2.formatMsgNoLookups=true" />
-  <property name="solr_version" value="8.11.2" />
+  <property name="solr_version" value="9.1.0" />
   <property name="skip_phpdoc" value="false" />
   <property name="phpdoc_version" value="3.3.1" />
 
@@ -271,7 +271,7 @@
           <not><available file="${srcdir}/downloads/solr-${solr_version}.tgz" /></not>
            <then>
             <mkdir dir="${srcdir}/downloads" />
-            <httpget url="http://archive.apache.org/dist/lucene/solr/${solr_version}/solr-${solr_version}.tgz" dir="${srcdir}/downloads" />
+            <httpget url="http://archive.apache.org/dist/solr/solr/${solr_version}/solr-${solr_version}.tgz" dir="${srcdir}/downloads" />
           </then>
         </if>
         <!-- unpack the archive into solr/vendor -->

import-marc.bat

@@ -118,7 +118,7 @@ for %%a in (%VUFIND_HOME%\import\solrmarc_core_*.jar) do set JAR_FILE=%%a
 rem #####################################################
 rem # Execute Importer
 rem #####################################################
-set RUN_CMD=%JAVA% %INDEX_OPTIONS% -Duser.timezone=UTC -Dlog4j.configuration="file:///%LOG4J_CONFIG%" %EXTRA_SOLRMARC_SETTINGS% -jar %JAR_FILE% %PROPERTIES_FILE% -solrj %VUFIND_HOME%\solr\vendor\dist\solrj-lib -lib_local %VUFIND_HOME%\import\lib_local;%VUFIND_HOME%\solr\vendor\contrib\analysis-extras\lib %1
+set RUN_CMD=%JAVA% %INDEX_OPTIONS% -Duser.timezone=UTC -Dlog4j.configuration="file:///%LOG4J_CONFIG%" %EXTRA_SOLRMARC_SETTINGS% -jar %JAR_FILE% %PROPERTIES_FILE% -solrj %VUFIND_HOME%\solr\vendor\server\solr-webapp\webapp\WEB-INF\lib -lib_local %VUFIND_HOME%\import\lib_local;%VUFIND_HOME%\solr\vendor\modules\analysis-extras\lib %1
 echo Now Importing %1 ...
 echo %RUN_CMD%
 %RUN_CMD%

import-marc.sh

@@ -135,11 +135,30 @@ MARC_PATH=`dirname $1`
 MARC_PATH=`cd $MARC_PATH && pwd`
 MARC_FILE=`basename $1`
 
+#####################################################
+# Set up SolrJ symlinks for performance (searching
+# all the Solr .jar files slows things down; this
+# helps by pointing to only the necessary ones)
+#####################################################
+if [ -z "$SOLRJ_DIR" ]
+then
+  SOLRJ_DIR="$VUFIND_HOME/solr/vendor/.solrj"
+fi
+
+if [ ! -d "$SOLRJ_DIR" ]
+then
+  mkdir -p $SOLRJ_DIR
+  for file in $VUFIND_HOME/solr/vendor/server/solr-webapp/webapp/WEB-INF/lib/solr*.jar $VUFIND_HOME/solr/vendor/server/solr-webapp/webapp/WEB-INF/lib/http*.jar
+  do
+    ln -s $file $SOLRJ_DIR/`basename $file`
+  done
+fi
+
 #####################################################
 # Execute Importer
 #####################################################
 
-RUN_CMD="$JAVA $INDEX_OPTIONS -Duser.timezone=UTC -Dlog4j.configuration=file://$LOG4J_CONFIG $EXTRA_SOLRMARC_SETTINGS -jar $JAR_FILE $PROPERTIES_FILE -solrj $VUFIND_HOME/solr/vendor/dist/solrj-lib -lib_local "$VUFIND_HOME/import/lib_local\;$VUFIND_HOME/solr/vendor/contrib/analysis-extras/lib" $MARC_PATH/$MARC_FILE"
+RUN_CMD="$JAVA $INDEX_OPTIONS -Duser.timezone=UTC -Dlog4j.configuration=file://$LOG4J_CONFIG $EXTRA_SOLRMARC_SETTINGS -jar $JAR_FILE $PROPERTIES_FILE -solrj $SOLRJ_DIR -lib_local "$VUFIND_HOME/import/lib_local\;$VUFIND_HOME/solr/vendor/modules/analysis-extras/lib" $MARC_PATH/$MARC_FILE"
 echo "Now Importing $1 ..."
 # solrmarc writes log messages to stderr, write RUN_CMD to the same place
 echo "`date '+%h %d, %H:%M:%S'` $RUN_CMD" >&2

index-alphabetic-browse.bat

@@ -1,6 +1,6 @@
 @echo off
 rem #####################################################
-rem Make sure that environment edits are local and that we have access to the 
+rem Make sure that environment edits are local and that we have access to the
 rem Windows command extensions.
 rem #####################################################
 setlocal enableextensions
@@ -13,7 +13,7 @@ rem ##################################################
 rem # Set SOLR_HOME
 rem ##################################################
 if not "!%VUFIND_HOME%!"=="!!" goto vufindhomefound
-rem VUFIND_HOME not set -- try to call env.bat to 
+rem VUFIND_HOME not set -- try to call env.bat to
 rem fix the problem before we give up completely
 if exist env.bat goto useenvbat
 rem If env.bat doesn't exist, the user hasn't run the installer yet.
@@ -44,7 +44,7 @@ set JAVA="%JAVA_HOME%\bin\java"
 :javaset
 
 cd %VUFIND_HOME%\import
-SET CLASSPATH="browse-indexing.jar;%SOLR_HOME%\jars\*;%SOLR_HOME%\..\vendor\contrib\analysis-extras\lib\*;%SOLR_HOME%\..\vendor\server\solr-webapp\webapp\WEB-INF\lib\*"
+SET CLASSPATH="browse-indexing.jar;%VUFIND_HOME%\import\lib\*;%SOLR_HOME%\jars\*;%SOLR_HOME%\..\vendor\modules\analysis-extras\lib\*;%SOLR_HOME%\..\vendor\server\solr-webapp\webapp\WEB-INF\lib\*"
 
 SET bib_index=%SOLR_HOME%\biblio\index
 SET auth_index=%SOLR_HOME%\authority\index

index-alphabetic-browse.sh

@@ -35,7 +35,7 @@ set -e
 set -x
 
 cd "`dirname $0`/import"
-CLASSPATH="browse-indexing.jar:${SOLR_HOME}/jars/*:${SOLR_HOME}/../vendor/contrib/analysis-extras/lib/*:${SOLR_HOME}/../vendor/server/solr-webapp/webapp/WEB-INF/lib/*"
+CLASSPATH="browse-indexing.jar:${VUFIND_HOME}/import/lib/*:${SOLR_HOME}/jars/*:${SOLR_HOME}/../vendor/modules/analysis-extras/lib/*:${SOLR_HOME}/../vendor/server/solr-webapp/webapp/WEB-INF/lib/*"
 
 # make index work with replicated index
 # current index is stored in the last line of index.properties

solr.bat

@@ -18,6 +18,10 @@ rem
 rem SOLR_PORT
 rem   Network port for Solr. Defaults to 8983.
 rem
+rem SOLR_SECURITY_MANAGER_ENABLED
+rem   Whether or not to enable the Java security manager (incompatible with
+rem   AlphaBrowse handler). Defaults to false.
+rem
 rem JAVA_HOME
 rem   Home of Java installation (not directly used by this script, but passed along to
 rem   the standard Solr control script).
@@ -83,6 +87,11 @@ if not "!%SOLR_PORT%!"=="!!" goto solrportset
 set SOLR_PORT=8983
 :solrportset
 
+rem Set SOLR_SECURITY_MANAGER_ENABLED
+if not "!%SOLR_SECURITY_MANAGER_ENABLED%!"=="!!" goto solrsecmanset
+set SOLR_SECURITY_MANAGER_ENABLED=false
+:solrsecmanset
+
 call %SOLR_BIN%\solr.cmd %1 %SOLR_ADDITIONAL_START_OPTIONS% -p %SOLR_PORT% -s %SOLR_HOME% -m %SOLR_HEAP% -a "-Ddisable.configEdit=true -Dsolr.log=%SOLR_LOGS_DIR% %SOLR_ADDITIONAL_JVM_OPTIONS%"
 goto end
 

solr.sh

@@ -19,6 +19,10 @@
 # SOLR_PORT
 #   Network port for Solr. Defaults to 8983.
 #
+# SOLR_SECURITY_MANAGER_ENABLED
+#   Whether or not to enable the Java security manager (incompatible with
+#   AlphaBrowse handler). Defaults to false.
+#
 # JAVA_HOME
 #   Home of Java installation (not directly used by this script, but passed along to
 #   the standard Solr control script).
@@ -77,6 +81,11 @@ then
   SOLR_PORT="8983"
 fi
 
+if [ -z "$SOLR_SECURITY_MANAGER_ENABLED" ]
+then
+  export SOLR_SECURITY_MANAGER_ENABLED="false"
+fi
+
 if [ -z "$SOLR_ADDITIONAL_START_OPTIONS" ]
 then
   SOLR_ADDITIONAL_START_OPTIONS=""

solr/vufind/authority/conf/solrconfig.xml

@@ -32,7 +32,7 @@
        that you fully re-index after changing this setting as it can
        affect both how text is indexed and queried.
     -->
-  <luceneMatchVersion>8.11.2</luceneMatchVersion>
+  <luceneMatchVersion>9.1.0</luceneMatchVersion>
 
   <!-- Use the classic schema style by default for VuFind -->
   <schemaFactory class="ClassicIndexSchemaFactory"/>
@@ -152,26 +152,33 @@
     <maxBooleanClauses>1024</maxBooleanClauses>
 
 
-    <!-- Cache used by SolrIndexSearcher for filters (DocSets),
-         unordered sets of *all* documents that match a query.
-         When a new searcher is opened, its caches may be prepopulated
-         or "autowarmed" using data from caches in the old searcher.
-         autowarmCount is the number of items to prepopulate.  For LRUCache,
-         the autowarmed items will be the most recently accessed items.
-       Parameters:
-         class - the SolrCache implementation (currently only LRUCache)
-         size - the maximum number of entries in the cache
-         initialSize - the initial capacity (number of entries) of
-           the cache.  (seel java.util.HashMap)
-         autowarmCount - the number of entries to prepopulate from
-           and old cache.
-         -->
+    <!-- Filter Cache
+
+         Cache used by SolrIndexSearcher for filters (DocSets),
+         unordered sets of *all* documents that match a query.  When a
+         new searcher is opened, its caches may be prepopulated or
+         "autowarmed" using data from caches in the old searcher.
+         autowarmCount is the number of items to prepopulate. For
+         CaffeineCache, the autowarmed items will be the most recently
+         accessed items.
+
+         Parameters:
+           class - the SolrCache implementation (CaffeineCache by default)
+           size - the maximum number of entries in the cache
+           initialSize - the initial capacity (number of entries) of
+               the cache.  (see java.util.HashMap)
+           autowarmCount - the number of entries to prepopulate from
+               an old cache.
+           maxRamMB - the maximum amount of RAM (in MB) that this cache is allowed
+                      to occupy. Note that when this option is specified, the size
+                      and initialSize parameters are ignored.
+      -->
 
     <!-- VuFind uses the default Solr settings for filterCache, which should produce a high
          hit rate. You can carefully adjust the filterCache and check the hit ratio in
          admin>plugins>stats -->
     <filterCache
-      class="solr.FastLRUCache"
+      class="solr.search.CaffeineCache"
       size="512"
       initialSize="512"
       autowarmCount="256"/>
@@ -180,15 +187,15 @@
          document ids (DocList) based on a query, a sort, and the range
          of documents requested.  -->
     <queryResultCache
-      class="solr.LRUCache"
+      class="solr.search.CaffeineCache"
       size="512"
       initialSize="512"
       autowarmCount="256"/>
 
   <!-- documentCache caches Lucene Document objects (the stored fields for each document).
        Since Lucene internal document ids are transient, this cache will not be autowarmed.  -->
     <documentCache
-      class="solr.LRUCache"
+      class="solr.search.CaffeineCache"
       size="16384"
       initialSize="16384"/>
 
@@ -200,21 +207,6 @@
     -->
     <enableLazyFieldLoading>false</enableLazyFieldLoading>
 
-    <!-- Example of a generic cache.  These caches may be accessed by name
-         through SolrIndexSearcher.getCache(),cacheLookup(), and cacheInsert().
-         The purpose is to enable easy caching of user/application level data.
-         The regenerator argument should be specified as an implementation
-         of solr.search.CacheRegenerator if autowarming is desired.  -->
-    <!--
-    <cache name="myUserCache"
-      class="solr.LRUCache"
-      size="4096"
-      initialSize="1024"
-      autowarmCount="1024"
-      regenerator="org.mycompany.mypackage.MyRegenerator"
-      />
-    -->
-
    <!-- An optimization that attempts to use a filter to satisfy a search.
          If the requested sort does not include score, then the filterCache
          will be checked for a filter matching the query. If found, the filter
@@ -523,14 +515,6 @@
     <queryResponseWriter name="custom" class="com.example.MyResponseWriter"/>
   -->
 
-  <!-- XSLT response writer transforms the XML output by any xslt file found
-       in Solr's conf/xslt directory.  Changes to xslt files are checked for
-       every xsltCacheLifetimeSeconds.
-   -->
-  <queryResponseWriter name="xslt" class="org.apache.solr.response.XSLTResponseWriter">
-    <int name="xsltCacheLifetimeSeconds">5</int>
-  </queryResponseWriter>
-
   <!-- Mitigate CVE-2017-12629 -->
   <queryParser name="xmlparser" class="solr.ExtendedDismaxQParserPlugin"/>
 

solr/vufind/biblio/conf/solrconfig.xml

@@ -32,7 +32,7 @@
        that you fully re-index after changing this setting as it can
        affect both how text is indexed and queried.
     -->
-  <luceneMatchVersion>8.11.2</luceneMatchVersion>
+  <luceneMatchVersion>9.1.0</luceneMatchVersion>
 
   <!-- Use the classic schema style by default for VuFind -->
   <schemaFactory class="ClassicIndexSchemaFactory"/>
@@ -42,8 +42,8 @@
        If replication is in use, this should match the replication configuration. -->
   <dataDir>${solr.solr.home:./solr}/biblio</dataDir>
 
-  <lib dir="../../vendor/contrib/analysis-extras/lib" regex=".*\.jar" />
-  <lib dir="../../vendor/contrib/analysis-extras/lucene-libs" regex=".*\.jar" />
+  <lib dir="../../../import/lib" regex="marc4j.*\.jar" />
+  <lib dir="../../vendor/modules/analysis-extras/lib" regex=".*\.jar" />
   <lib dir="../jars" regex=".*\.jar" />
 
   <indexConfig>
@@ -155,26 +155,33 @@
     <maxBooleanClauses>1024</maxBooleanClauses>
 
 
-    <!-- Cache used by SolrIndexSearcher for filters (DocSets),
-         unordered sets of *all* documents that match a query.
-         When a new searcher is opened, its caches may be prepopulated
-         or "autowarmed" using data from caches in the old searcher.
-         autowarmCount is the number of items to prepopulate.  For LRUCache,
-         the autowarmed items will be the most recently accessed items.
-       Parameters:
-         class - the SolrCache implementation (currently only LRUCache)
-         size - the maximum number of entries in the cache
-         initialSize - the initial capacity (number of entries) of
-           the cache.  (seel java.util.HashMap)
-         autowarmCount - the number of entries to prepopulate from
-           and old cache.
-         -->
+    <!-- Filter Cache
+
+         Cache used by SolrIndexSearcher for filters (DocSets),
+         unordered sets of *all* documents that match a query.  When a
+         new searcher is opened, its caches may be prepopulated or
+         "autowarmed" using data from caches in the old searcher.
+         autowarmCount is the number of items to prepopulate. For
+         CaffeineCache, the autowarmed items will be the most recently
+         accessed items.
+
+         Parameters:
+           class - the SolrCache implementation (CaffeineCache by default)
+           size - the maximum number of entries in the cache
+           initialSize - the initial capacity (number of entries) of
+               the cache.  (see java.util.HashMap)
+           autowarmCount - the number of entries to prepopulate from
+               an old cache.
+           maxRamMB - the maximum amount of RAM (in MB) that this cache is allowed
+                      to occupy. Note that when this option is specified, the size
+                      and initialSize parameters are ignored.
+      -->
 
     <!-- VuFind uses the default Solr settings for filterCache, which should produce a high
          hit rate. You can carefully adjust the filterCache and check the hit ratio in
          admin>plugins>stats -->
     <filterCache
-      class="solr.FastLRUCache"
+      class="solr.search.CaffeineCache"
       size="512"
       initialSize="512"
       autowarmCount="256"/>
@@ -183,15 +190,15 @@
          document ids (DocList) based on a query, a sort, and the range
          of documents requested.  -->
     <queryResultCache
-      class="solr.LRUCache"
+      class="solr.search.CaffeineCache"
       size="512"
       initialSize="512"
       autowarmCount="256"/>
 
   <!-- documentCache caches Lucene Document objects (the stored fields for each document).
        Since Lucene internal document ids are transient, this cache will not be autowarmed.  -->
     <documentCache
-      class="solr.LRUCache"
+      class="solr.search.CaffeineCache"
       size="16384"
       initialSize="16384"/>
 
@@ -203,21 +210,6 @@
     -->
     <enableLazyFieldLoading>true</enableLazyFieldLoading>
 
-    <!-- Example of a generic cache.  These caches may be accessed by name
-         through SolrIndexSearcher.getCache(),cacheLookup(), and cacheInsert().
-         The purpose is to enable easy caching of user/application level data.
-         The regenerator argument should be specified as an implementation
-         of solr.search.CacheRegenerator if autowarming is desired.  -->
-    <!--
-    <cache name="myUserCache"
-      class="solr.LRUCache"
-      size="4096"
-      initialSize="1024"
-      autowarmCount="1024"
-      regenerator="org.mycompany.mypackage.MyRegenerator"
-      />
-    -->
-
    <!-- An optimization that attempts to use a filter to satisfy a search.
          If the requested sort does not include score, then the filterCache
          will be checked for a filter matching the query. If found, the filter
@@ -706,14 +698,6 @@
     <queryResponseWriter name="custom" class="com.example.MyResponseWriter"/>
   -->
 
-  <!-- XSLT response writer transforms the XML output by any xslt file found
-       in Solr's conf/xslt directory.  Changes to xslt files are checked for
-       every xsltCacheLifetimeSeconds.
-   -->
-  <queryResponseWriter name="xslt" class="org.apache.solr.response.XSLTResponseWriter">
-    <int name="xsltCacheLifetimeSeconds">5</int>
-  </queryResponseWriter>
-
   <!-- Mitigate CVE-2017-12629 -->
   <queryParser name="xmlparser" class="solr.ExtendedDismaxQParserPlugin"/>