Merge pull request #5420 from IQSS/develop

v4.10
IQSS · Dec 20, 2018 · a88bbb6 · a88bbb6
2 parents 57b3b7e + 38851fa
commit a88bbb6
Show file tree

Hide file tree

Showing 456 changed files with 27,341 additions and 3,036 deletions.
diff --git a/.gitignore b/.gitignore
@@ -44,3 +44,7 @@ conf/docker-aio/dv/install/dvinstall.zip
 conf/docker-aio/testdata/
 scripts/installer/default.config
 *.pem
+
+# do not track IntelliJ IDEA files
+.idea
+**/*.iml
diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md
@@ -15,7 +15,7 @@ Welcome! New contributors should at least glance at [CONTRIBUTING.md](/CONTRIBUT
 - [ ] Merged latest from "develop" [branch][] and resolved conflicts
 
 [tests]: http://guides.dataverse.org/en/latest/developers/testing.html
-[SQL updates]: https://github.com/IQSS/dataverse/tree/develop/scripts/database/upgrades
+[SQL updates]: http://guides.dataverse.org/en/latest/developers/sql-upgrade-scripts.html
 [Solr updates]: https://github.com/IQSS/dataverse/blob/develop/conf/solr/7.3.0/schema.xml
 [docs]: http://guides.dataverse.org/en/latest/developers/documentation.html
 [branch]: http://guides.dataverse.org/en/latest/developers/branching-strategy.html
diff --git a/conf/docker-aio/c7.dockerfile b/conf/docker-aio/c7.dockerfile
@@ -3,7 +3,7 @@ FROM centos:7
 RUN yum install -y https://download.postgresql.org/pub/repos/yum/9.6/redhat/rhel-7-x86_64/pgdg-centos96-9.6-3.noarch.rpm
 #RUN yum install -y java-1.8.0-openjdk-headless postgresql-server sudo epel-release unzip perl curl httpd
 RUN yum install -y java-1.8.0-openjdk-devel postgresql96-server sudo epel-release unzip perl curl httpd
-RUN yum install -y jq lsof
+RUN yum install -y jq lsof awscli
 
 # copy and unpack dependencies (solr, glassfish)
 COPY dv /tmp/dv

diff --git a/conf/docker-aio/readme.md b/conf/docker-aio/readme.md
@@ -57,4 +57,4 @@ There isn't any strict requirement on the local port (8083, 8084 in this doc), t
 
 * `empty reply from server` or `Failed to connect to ::1: Cannot assign requested address` tend to indicate either that you haven't given glassfish enough time to start, or your docker setup is in an inconsistent state and should probably be restarted.
 
-* For manually fiddling around with the created dataverse, use user `dataverseAdmin` with password `admin`.
+* For manually fiddling around with the created dataverse, use user `dataverseAdmin` with password `admin1`.
diff --git a/conf/docker-aio/run-test-suite.sh b/conf/docker-aio/run-test-suite.sh
@@ -8,4 +8,4 @@ fi
 
 # Please note the "dataverse.test.baseurl" is set to run for "all-in-one" Docker environment.
 # TODO: Rather than hard-coding the list of "IT" classes here, add a profile to pom.xml.
-mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT -Ddataverse.test.baseurl=$dvurl
+mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT -Ddataverse.test.baseurl=$dvurl
diff --git a/conf/docker-aio/testscripts/post b/conf/docker-aio/testscripts/post
@@ -1,6 +1,6 @@
 #/bin/sh
 cd scripts/api
-./setup-all.sh --insecure | tee /tmp/setup-all.sh.out
+./setup-all.sh --insecure -p=admin1 | tee /tmp/setup-all.sh.out
 cd ../..
 psql -U dvnapp dvndb -f scripts/database/reference_data.sql
 psql -U dvnapp dvndb -f doc/sphinx-guides/source/_static/util/pg8-createsequence-prep.sql

diff --git a/conf/docker-dcm/0prep.sh b/conf/docker-dcm/0prep.sh
@@ -1,4 +1,3 @@
 #!/bin/sh
 
-
-wget https://github.com/sbgrid/data-capture-module/releases/download/0.2/dcm-0.2-0.noarch.rpm
+wget https://github.com/sbgrid/data-capture-module/releases/download/0.5/dcm-0.5-0.noarch.rpm
diff --git a/conf/docker-dcm/dcmsrv.dockerfile b/conf/docker-dcm/dcmsrv.dockerfile
@@ -1,13 +1,14 @@
 # build from repo root
 FROM centos:6
 RUN yum install -y epel-release
-ARG RPMFILE=dcm-0.2-0.noarch.rpm
+ARG RPMFILE=dcm-0.5-0.noarch.rpm
 COPY ${RPMFILE} /tmp/
 COPY bashrc /root/.bashrc
 COPY test_install.sh /root/
 RUN yum localinstall -y /tmp/${RPMFILE}
 RUN pip install -r /opt/dcm/requirements.txt
 RUN pip install awscli==1.15.75
+run export PATH=~/.local/bin:$PATH
 RUN /root/test_install.sh
 COPY rq-init-d /etc/init.d/rq
 RUN useradd glassfish

diff --git a/conf/docker/solr/solrconfig_master.xml b/conf/docker/solr/solrconfig_master.xml
@@ -686,13 +686,55 @@
        of SearchComponents (see below) and supports distributed
        queries across multiple shards
     -->
-  <requestHandler name="/select" class="solr.SearchHandler">
+   <requestHandler name="/select" class="solr.SearchHandler">
     <!-- default values for query parameters can be specified, these
          will be overridden by parameters in the request
       -->
     <lst name="defaults">
       <str name="echoParams">explicit</str>
       <int name="rows">10</int>
+      <str name="defType">edismax</str>
+      <float name="tie">0.075</float>
+        <str name="qf">
+            dvName^400
+            authorName^180
+            dvSubject^190
+            dvDescription^180
+            dvAffiliation^170
+            title^130
+            subject^120
+            keyword^110
+            topicClassValue^100
+            dsDescriptionValue^90
+            authorAffiliation^80
+            publicationCitation^60
+            producerName^50
+            fileName^30
+            fileDescription^30
+            variableLabel^20
+            variableName^10
+            _text_^1.0
+        </str>
+        <str name="pf">
+            dvName^200
+            authorName^100
+            dvSubject^100
+            dvDescription^100
+            dvAffiliation^100
+            title^75
+            subject^75
+            keyword^75
+            topicClassValue^75
+            dsDescriptionValue^75
+            authorAffiliation^75
+            publicationCitation^75
+            producerName^75
+        </str>
+        <!-- Even though this number is huge it only seems to apply a boost of ~1.5x to final result -MAD 4.9.3 -->
+        <str name="bq">
+            isHarvested:false^25000
+        </str>
+
       <!-- Default search field
          <str name="df">text</str> 
         -->
@@ -924,27 +966,6 @@
       <str name="spellcheck.collateExtendedResults">true</str>
       <str name="spellcheck.maxCollationTries">10</str>
       <str name="spellcheck.maxCollations">5</str>
-      <str name="defType">edismax</str>
-<str name="qf">
-dvName^170
-dvSubject^160
-dvDescription^150
-dvAffiliation^140
-title^130
-subject^120
-keyword^110
-topicClassValue^100
-dsDescriptionValue^90
-authorName^80
-authorAffiliation^70
-publicationCitation^60
-producerName^50
-fileName^40
-fileDescription^30
-variableLabel^20
-variableName^10
-text^1.0
-</str>
     </lst>
     <arr name="last-components">
       <str>spellcheck</str>

diff --git a/conf/docker/solr/solrconfig_slave.xml b/conf/docker/solr/solrconfig_slave.xml
@@ -686,13 +686,55 @@
        of SearchComponents (see below) and supports distributed
        queries across multiple shards
     -->
-  <requestHandler name="/select" class="solr.SearchHandler">
+   <requestHandler name="/select" class="solr.SearchHandler">
     <!-- default values for query parameters can be specified, these
          will be overridden by parameters in the request
       -->
     <lst name="defaults">
       <str name="echoParams">explicit</str>
       <int name="rows">10</int>
+      <str name="defType">edismax</str>
+      <float name="tie">0.075</float>
+        <str name="qf">
+            dvName^400
+            authorName^180
+            dvSubject^190
+            dvDescription^180
+            dvAffiliation^170
+            title^130
+            subject^120
+            keyword^110
+            topicClassValue^100
+            dsDescriptionValue^90
+            authorAffiliation^80
+            publicationCitation^60
+            producerName^50
+            fileName^30
+            fileDescription^30
+            variableLabel^20
+            variableName^10
+            _text_^1.0
+        </str>
+        <str name="pf">
+            dvName^200
+            authorName^100
+            dvSubject^100
+            dvDescription^100
+            dvAffiliation^100
+            title^75
+            subject^75
+            keyword^75
+            topicClassValue^75
+            dsDescriptionValue^75
+            authorAffiliation^75
+            publicationCitation^75
+            producerName^75
+        </str>
+        <!-- Even though this number is huge it only seems to apply a boost of ~1.5x to final result -MAD 4.9.3 -->
+        <str name="bq">
+            isHarvested:false^25000
+        </str>
+
       <!-- Default search field
          <str name="df">text</str> 
         -->
@@ -924,27 +966,6 @@
       <str name="spellcheck.collateExtendedResults">true</str>
       <str name="spellcheck.maxCollationTries">10</str>
       <str name="spellcheck.maxCollations">5</str>
-      <str name="defType">edismax</str>
-<str name="qf">
-dvName^170
-dvSubject^160
-dvDescription^150
-dvAffiliation^140
-title^130
-subject^120
-keyword^110
-topicClassValue^100
-dsDescriptionValue^90
-authorName^80
-authorAffiliation^70
-publicationCitation^60
-producerName^50
-fileName^40
-fileDescription^30
-variableLabel^20
-variableName^10
-text^1.0
-</str>
     </lst>
     <arr name="last-components">
       <str>spellcheck</str>

diff --git a/conf/solr/7.3.0/schema.xml b/conf/solr/7.3.0/schema.xml
@@ -147,11 +147,14 @@
     <field name="isHarvested" type="boolean" stored="true" indexed="true" multiValued="false"/>
 
     <field name="dvName" type="text_en" stored="true" indexed="true" multiValued="false"/>
+    <field name="dvAlias" type="text_en" stored="true" indexed="true" multiValued="false"/>
     <field name="dvAffiliation" type="text_en" stored="true" indexed="true" multiValued="false"/>
     <field name="dvDescription" type="text_en" stored="true" indexed="true" multiValued="false"/>
 
     <field name="dvCategory" type="string" stored="true" indexed="true" multiValued="false"/>
-
+    <field name="categoryOfDataverse" type="string" stored="true" indexed="true" multiValued="false"/>
+    <field name="identifierOfDataverse" type="string" stored="true" indexed="true" multiValued="false"/>
+
     <field name="publicationDate" type="string" stored="true" indexed="true" multiValued="false"/>
     <field name="dsPublicationDate" type="string" stored="true" indexed="true" multiValued="false"/>
 
@@ -456,6 +459,7 @@
     <copyField source="variableLabel" dest="_text_" maxChars="3000"/>
     <!-- Make dataverse subject and affiliation searchable from basic search: https://github.com/IQSS/dataverse/issues/1431 -->
     <copyField source="dvSubject" dest="_text_" maxChars="3000"/>
+    <copyField source="dvAlias" dest="_text_" maxChars="3000"/>
     <copyField source="dvAffiliation" dest="_text_" maxChars="3000"/>
     <copyField source="dsPersistentId" dest="_text_" maxChars="3000"/>
     <!-- copyField commands copy one field to another at the time a document

diff --git a/conf/solr/7.3.0/solrconfig.xml b/conf/solr/7.3.0/solrconfig.xml
@@ -693,6 +693,48 @@
     <lst name="defaults">
       <str name="echoParams">explicit</str>
       <int name="rows">10</int>
+      <str name="defType">edismax</str>
+      <float name="tie">0.075</float>
+        <str name="qf">
+            dvName^400
+            authorName^180
+            dvSubject^190
+            dvDescription^180
+            dvAffiliation^170
+            title^130
+            subject^120
+            keyword^110
+            topicClassValue^100
+            dsDescriptionValue^90
+            authorAffiliation^80
+            publicationCitation^60
+            producerName^50
+            fileName^30
+            fileDescription^30
+            variableLabel^20
+            variableName^10
+            _text_^1.0
+        </str>
+        <str name="pf">
+            dvName^200
+            authorName^100
+            dvSubject^100
+            dvDescription^100
+            dvAffiliation^100
+            title^75
+            subject^75
+            keyword^75
+            topicClassValue^75
+            dsDescriptionValue^75
+            authorAffiliation^75
+            publicationCitation^75
+            producerName^75
+        </str>
+        <!-- Even though this number is huge it only seems to apply a boost of ~1.5x to final result -MAD 4.9.3--> 
+        <str name="bq">
+            isHarvested:false^25000
+        </str>
+
       <!-- Default search field
          <str name="df">text</str> 
         -->
@@ -924,27 +966,6 @@
       <str name="spellcheck.collateExtendedResults">true</str>
       <str name="spellcheck.maxCollationTries">10</str>
       <str name="spellcheck.maxCollations">5</str>
-      <str name="defType">edismax</str>
-<str name="qf">
-dvName^170
-dvSubject^160
-dvDescription^150
-dvAffiliation^140
-title^130
-subject^120
-keyword^110
-topicClassValue^100
-dsDescriptionValue^90
-authorName^80
-authorAffiliation^70
-publicationCitation^60
-producerName^50
-fileName^40
-fileDescription^30
-variableLabel^20
-variableName^10
-text^1.0
-</str>
     </lst>
     <arr name="last-components">
       <str>spellcheck</str>

diff --git a/doc/release-notes/4690-custom-s3-url.md b/doc/release-notes/4690-custom-s3-url.md
@@ -0,0 +1,2 @@
+Configuration options for custom S3 URLs of Amazon S3 compatible storage available.
+See [configuration documentation](http://guides.dataverse.org/en/latest/installation/config.html#amazon-s3-storage-or-compatible) for details.
diff --git a/doc/release-notes/4980-upgrades-across-versions.md b/doc/release-notes/4980-upgrades-across-versions.md
@@ -0,0 +1,10 @@
+We now offer an *EXPERIMENTAL* database upgrade method allowing users
+to skip over a number of releases. E.g., it should be possible now to
+upgrade a Dataverse database from v4.8.6 directly to v4.10, without
+having to deploy the war files for the 5 releases between these 2
+versions and manually running the corresponding database upgrade
+scripts.
+
+The upgrade script, dbupgrade.sh is provided in the scripts/database
+directory of the Dataverse source tree. See the file
+README_upgrade_across_versions.txt for the instructions.
diff --git a/doc/release-notes/5011-search-by-dataverse-alias.md b/doc/release-notes/5011-search-by-dataverse-alias.md
@@ -0,0 +1 @@
+The Solr schema.xml file must be updated due to the addition of the "dvAlias" field.
diff --git a/doc/release-notes/5323-saved-original-size b/doc/release-notes/5323-saved-original-size
@@ -0,0 +1,15 @@
+Starting the release 4.10 the size of the saved original file (for an
+ingested tabular datafile) is stored in the database.  We provided the
+following API that retrieve and permanently store the sizes for any
+already existing saved originals:
+/api/admin/datafiles/integrity/fixmissingoriginalsizes (see the
+documentation note in the Native API guide, under "Datafile
+Integrity").
+
+While it's not strictly necessary to have these sizes in the database,
+having them makes certain operations more efficient (primary example
+is a user downloading the saved originals for multiple files/an entire
+dataset etc.) Also, if present in the database, the size will be added
+to the file information displayed in the output of the /api/datasets;
+which can be useful for some users.
+
diff --git a/doc/release-notes/README.md b/doc/release-notes/README.md
@@ -0,0 +1,12 @@
+# Dataverse Release Notes
+
+doc/sphinx-guides/source/developers/making-releases.rst documents the official process for making release notes but as indicated there, we are experimenting with a process with the following goals:
+
+- As a developer, I want to express in my pull request when an addition to the release notes will be necessary.
+- As a developer, I want to be aware of changes that should be made to my dev environment after a pull request has been merged. I already know to look in `scripts/database/upgrades` if I pull the latest code from the "develop" branch for updates as described in doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst but I want a place to look for non-SQL updates that are required. These could be Solr schema changes or curl commands to reload metadata blocks, for example.
+
+# release-notes directory process
+
+- Create a Markdown file named after your branch (assuming your branch starts with an issue number as requested in doc/sphinx-guides/source/developers/version-control.rst) such as "5053-apis-custom-homepage.md".
+- In the file you created, give instructions for non-SQL upgrade steps that must be taken to run the branch in your pull request. Examples include Solr schema updates or reloading metadata blocks.
+- At release time, gather all the files into final release notes and make a `git rm` commit to delete them to prevent clutter.