Improve phing build script and related scripts for improved cross-pla…

…tform support and error handling (vufind-org#1970) - Use more built-in phing functions - Check return values - Use passthru instead of buffering output of external commands - Avoid errors from deletion of files on shutdown - Always define mysqlpwswitch so that -verbose doesn't complain - Allow dash and underscore in base path. - Make import-marc-auth.sh work on non-GNU POSIX systems.
demiankatz · May 26, 2021 · fee3e14 · fee3e14
1 parent 6447344
commit fee3e14
Show file tree

Hide file tree

Showing 3 changed files with 115 additions and 63 deletions.
diff --git a/build.xml b/build.xml
@@ -35,9 +35,17 @@
   <property name="version" value="7.1" />
 
   <!-- We only need the -p switch if the password is non-blank -->
-  <if><not><equals arg1="${mysqlrootpass}" arg2="" /></not><then>
-    <property name="mysqlpwswitch" value="-p" />
-  </then></if>
+  <if>
+    <not>
+      <equals arg1="${mysqlrootpass}" arg2="" />
+    </not>
+    <then>
+      <property name="mysqlpwswitch" value="-p" />
+    </then>
+    <else>
+      <property name="mysqlpwswitch" value="" />
+    </else>
+  </if>
 
   <!-- Main Target -->
   <target name="main" description="main target">
@@ -230,7 +238,12 @@
         <!-- unpack the archive into solr/vendor -->
         <mkdir dir="${builddir}/solr" />
         <untar file="${srcdir}/downloads/solr-${solr_version}.tgz" todir="${builddir}/solr" />
-        <delete dir="${srcdir}/solr/vendor" includeemptydirs="true" failonerror="false" />
+        <if>
+          <available file="${srcdir}/solr/vendor" type="dir" />
+          <then>
+            <delete dir="${srcdir}/solr/vendor" includeemptydirs="true" failonerror="false" />
+          </then>
+        </if>
         <move file="${builddir}/solr/solr-${solr_version}" tofile="${srcdir}/solr/vendor" />
         <!-- make scripts executable -->
         <chmod mode="0755">
@@ -300,7 +313,7 @@
 
     <!-- Generate basic configuration -->
     <php expression="end(explode('/', '${vufindurl}'))" returnProperty="basepath" />
-    <exec command="php ${srcdir}/install.php --basepath=/${basepath} --non-interactive" />
+    <exec command="php ${srcdir}/install.php --basepath=/${basepath} --non-interactive" passthru="true" checkreturn="true" />
 
     <!-- Activate Selenium (if a path has been provided) -->
     <if>
@@ -315,8 +328,7 @@
       <istrue value="${apachectl}" />
       <then>
         <copy file="${srcdir}/local/httpd-vufind.conf" tofile="${apacheconfdir}/vufindtest.conf" />
-        <exec command="${apachectl} restart" outputProperty="LASTOUTPUT" />
-        <echo message="${LASTOUTPUT}" />
+        <exec command="${apachectl} restart" passthru="true" />
       </then>
     </if>
 
@@ -365,8 +377,7 @@
     <exec command="chmod a+w ${srcdir}/vufind-exception.log" />
 
     <!-- start Solr (use restart in case of old PID files) -->
-    <exec command="SOLR_PID_DIR=${srcdir}/local VUFIND_HOME=${srcdir} ${srcdir}/solr.sh restart" outputProperty="LASTOUTPUT" checkreturn="true" />
-    <echo message="${LASTOUTPUT}" />
+    <exec command="SOLR_PID_DIR=${srcdir}/local VUFIND_HOME=${srcdir} ${srcdir}/solr.sh restart" passthru="true" checkreturn="true" />
     <!-- fail if Solr did not start up successfully: -->
     <if>
       <not><available file="${solr_pid_file}" /></not>
@@ -386,15 +397,32 @@
       </else>
     </if>
 
-    <!-- import marc test records into vufind index (note: the marc test records have prefix "testsample#") -->
-    <exec escape="false" command="find ${srcdir}/tests/data -name '*.mrc' -printf '%p,'" outputProperty="buglist" />
-    <foreach list="${buglist}" param="filename" delimiter="," target="importrec" />
-    <exec escape="false" command="find ${srcdir}/tests/data/authority -name '*.mrc' -printf '%p,'" outputProperty="authlist" />
-    <foreach list="${authlist}" param="filename" delimiter="," target="importauthrec" />
+    <phingcall target="import_biblios" />
+    <phingcall target="import_authorities" />
+    <phingcall target="build_alphabrowse" />
+  </target>
+
+  <!-- Import test biblio data -->
+  <target name="import_biblios" description="import all biblio test records">
+    <foreach param="relfilename" absparam="filename" target="importrec">
+      <fileset dir="${srcdir}/tests/data">
+          <include name="*.mrc" />
+      </fileset>
+    </foreach>
+  </target>
+
+  <!-- Import test authority data -->
+  <target name="import_authorities" description="import all authority test records">
+    <foreach param="relfilename" absparam="filename" target="importauthrec">
+      <fileset dir="${srcdir}/tests/data/authority">
+          <include name="*.mrc" />
+      </fileset>
+    </foreach>
+  </target>
 
-    <!-- build alphabrowse index -->
-    <exec command="VUFIND_HOME=${srcdir} VUFIND_LOCAL_DIR=${srcdir}/local ${srcdir}/index-alphabetic-browse.sh" outputProperty="LASTOUTPUT" />
-    <echo message="${LASTOUTPUT}" />
+  <!-- Build alphabrowse index -->
+  <target name="build_alphabrowse" description="build the alphabrowse index">
+    <exec command="VUFIND_HOME=${srcdir} VUFIND_LOCAL_DIR=${srcdir}/local ${srcdir}/index-alphabetic-browse.sh" passthru="true" />
   </target>
 
   <!-- Uninstall and Deactivate VuFind -->
@@ -423,8 +451,7 @@
     </if>
 
     <!-- stop Solr -->
-    <exec command="SOLR_PID_DIR=${srcdir}/local VUFIND_HOME=${srcdir} ${srcdir}/solr.sh stop" outputProperty="LASTOUTPUT" />
-    <echo message="${LASTOUTPUT}" />
+    <exec command="SOLR_PID_DIR=${srcdir}/local VUFIND_HOME=${srcdir} ${srcdir}/solr.sh stop" passthru="true" />
 
     <!-- run extra cleanup action, if any -->
     <if>
@@ -439,18 +466,44 @@
       <istrue value="${seleniumjar}" />
       <then>
         <exec command="pkill -f xvfb" />
-        <exec command="pkill -f &quot;java -jar `basename ${seleniumjar}`&quot;" />
+        <basename file="${seleniumjar}" property="seleniumbase" />
+        <exec command="pkill -f &quot;java -jar ${seleniumbase}&quot;" />
       </then>
     </if>
 
     <!-- delete the configuration, sample index, logs and cache data -->
-    <delete file="${srcdir}/import/solrmarc.log" failonerror="true" />
-    <delete dir="${srcdir}/solr" includeemptydirs="true" failonerror="true" />
-    <delete file="${srcdir}/composer.phar" failonerror="false" />
-    <delete file="${srcdir}/vufind-exception.log" failonerror="false" />
+    <if>
+      <available file="${srcdir}/import/solrmarc.log" />
+      <then>
+        <delete file="${srcdir}/import/solrmarc.log" failonerror="true" />
+      </then>
+    </if>
+    <if>
+      <available file="${srcdir}/solr" type="dir" />
+      <then>
+        <delete dir="${srcdir}/solr" includeemptydirs="true" failonerror="true" />
+      </then>
+    </if>
+    <if>
+      <available file="${srcdir}/composer.phar" />
+      <then>
+        <delete file="${srcdir}/composer.phar" failonerror="true" />
+      </then>
+    </if>
+    <if>
+      <available file="${srcdir}/vufind-exception.log" />
+      <then>
+        <delete file="${srcdir}/vufind-exception.log" failonerror="true" />
+      </then>
+    </if>
     <delete dir="${srcdir}/vendor" includeemptydirs="true" failonerror="false" />
-    <delete dir="${srcdir}/local" includeemptydirs="true" failonerror="true" />
-    <exec command="git reset --hard" />
+    <if>
+      <available file="${srcdir}/solr" type="dir" />
+      <then>
+        <delete dir="${srcdir}/local" includeemptydirs="true" failonerror="true" />
+      </then>
+    </if>
+    <exec command="git reset --hard" passthru="true" />
   </target>
 
   <!-- Prepare VuFind for distribution -->
@@ -502,48 +555,46 @@
   </target>
 
   <target name="importauthrec" description="import each of the MARC authority test records">
-    <if>
-    <istrue value="${filename}"/>       <!-- To ignore the last token, as find command output list has ',' after last filename -->
-      <then>
-        <exec command="basename ${filename}" outputProperty="BASENAME" />
-
-        <!-- perform the import -->
-        <exec command="VUFIND_HOME=${srcdir} VUFIND_LOCAL_DIR=${srcdir}/local ${srcdir}/import-marc-auth.sh ${filename} marc_auth_ils.properties" outputProperty="LASTOUTPUT" />
-        <echo message="${LASTOUTPUT}" />
-      </then>
-    </if>
+    <!-- perform the import -->
+    <exec command="VUFIND_HOME=${srcdir} VUFIND_LOCAL_DIR=${srcdir}/local ${srcdir}/import-marc-auth.sh ${filename} marc_auth_ils.properties" passthru="true" checkreturn="true" />
   </target>
 
   <target name="importrec" description="import each of the MARC bibliographic test records">
-    <if>
-    <istrue value="${filename}"/>       <!-- To ignore the last token, as find command output list has ',' after last filename -->
-      <then>
-        <exec command="basename ${filename}" outputProperty="BASENAME" />
+    <basename file="${filename}" property="BASENAME" />
 
-        <!-- create custom import configurations to load the MARC filename into the building facet to help
-             test cases to limit searches to within specific collections of test records. -->
-        <copy file="${srcdir}/import/marc_local.properties" tofile="${srcdir}/local/import/marc-${BASENAME}.properties" />
-        <exec command="echo building=\&quot;${BASENAME}\&quot; &gt;&gt; ${srcdir}/local/import/marc-${BASENAME}.properties" />
-        <exec command="sed -e &quot;s!marc_local.properties!marc-${BASENAME}.properties!&quot; ${srcdir}/local/import/import.properties &gt; ${srcdir}/local/import/import-${BASENAME}.properties" />
+    <!-- create custom import configurations to load the MARC filename into the building facet to help
+          test cases to limit searches to within specific collections of test records.
+          Note: ${srcdir}/local/import/import.properties is created in install.php. -->
+    <copy file="${srcdir}/import/marc_local.properties" tofile="${srcdir}/local/import/marc-${BASENAME}.properties" overwrite="true" />
+    <append destFile="${srcdir}/local/import/marc-${BASENAME}.properties" text="building=&quot;${BASENAME}&quot;${line.separator}" />
 
-        <!-- if there is a file-specific import configuration, load it now: -->
-        <if>
-          <available file="${filename}.properties" />
-          <then>
-            <echo message="Found custom import configs for ${filename}." />
-            <exec command="cat ${filename}.properties &gt;&gt; ${srcdir}/local/import/marc-${BASENAME}.properties" />
-          </then>
-        </if>
-
-        <!-- perform the import -->
-        <exec command="VUFIND_HOME=${srcdir} VUFIND_LOCAL_DIR=${srcdir}/local ${srcdir}/import-marc.sh -p ${srcdir}/local/import/import-${BASENAME}.properties ${filename}" outputProperty="LASTOUTPUT" />
-        <echo message="${LASTOUTPUT}" />
+    <copy file="${srcdir}/local/import/import.properties" tofile="${srcdir}/local/import/import-${BASENAME}.properties" />
+    <reflexive>
+      <fileset dir="${srcdir}/local/import">
+        <include pattern="import-${BASENAME}.properties" />
+      </fileset>
+      <filterchain>
+        <replaceregexp>
+          <regexp pattern="marc_local.properties" replace="marc-${BASENAME}.properties" />
+        </replaceregexp>
+      </filterchain>
+    </reflexive>
 
-        <!-- clean up temp files -->
-        <delete file="${srcdir}/local/import/marc-${BASENAME}.properties" />
-        <delete file="${srcdir}/local/import/import-${BASENAME}.properties" />
+    <!-- if there is a file-specific import configuration, load it now: -->
+    <if>
+      <available file="${filename}.properties" />
+      <then>
+        <echo message="Found custom import configs for ${filename}." />
+        <append file="${filename}.properties" destFile="${srcdir}/local/import/marc-${BASENAME}.properties" />
       </then>
     </if>
+
+    <!-- perform the import -->
+    <exec command="VUFIND_HOME=${srcdir} VUFIND_LOCAL_DIR=${srcdir}/local ${srcdir}/import-marc.sh -p ${srcdir}/local/import/import-${BASENAME}.properties ${filename}" passthru="true" checkreturn="true" />
+
+    <!-- clean up temp files -->
+    <delete file="${srcdir}/local/import/marc-${BASENAME}.properties" />
+    <delete file="${srcdir}/local/import/import-${BASENAME}.properties" />
   </target>
 
   <target name="patch-dependencies" description="apply fixes to dependencies">

diff --git a/import-marc-auth.sh b/import-marc-auth.sh
@@ -36,7 +36,7 @@ fi
 
 # Always use the authority mappings from PROPERTIES_FILE
 # if the user specified an override file, add that to the setting.
-MAPPINGS_FILENAMES=($(sed --quiet --expression='s/^\(solr.indexer.properties\s*=\s*\)\(.*\)/\2/p' $PROPERTIES_FILE | tr "," " "))
+MAPPINGS_FILENAMES=($(grep '^solr\.indexer\.properties *=' $PROPERTIES_FILE | sed 's/^solr.indexer.properties *= *\(.*\)/\1/p' | tr "," " "))
 if [ $# -gt 1 ]
 then
   MAPPINGS_FILENAMES+=($2)

diff --git a/module/VuFindConsole/src/VuFindConsole/Command/Install/InstallCommand.php b/module/VuFindConsole/src/VuFindConsole/Command/Install/InstallCommand.php
@@ -271,9 +271,10 @@ protected function validateBasePath($basePath, $allowEmpty = false)
         if ($allowEmpty && empty($basePath)) {
             return true;
         }
-        return preg_match('/^\/\w*$/', $basePath)
+        return preg_match('/^\/[\w_-]*$/', $basePath)
             ? true
-            : 'Error: Base path must be alphanumeric and start with a slash.';
+            : 'Error: Base path must start with a slash and contain only'
+                . ' alphanumeric characters, dash or underscore.';
     }
 
     /**