Merge pull request #5773 from IQSS/develop

v4.13 fixed

doc/sphinx-guides/source/developers/testing.rst

@@ -224,6 +224,17 @@ One way of generating load is by downloading many files. You can download :downl
 
 The script requires a file called ``files.txt`` to operate and database IDs for the files you want to download should each be on their own line.
 
+Continuous Integration
+~~~~~~~~~~~~~~~~~~~~~~
+
+The Dataverse Project currently makes use of two Continuous Integration platforms, Travis and Jenkins.
+
+Travis builds are configured via :download:`.travis.yml <../../../../.travis.yml>` and a `GitHub webhook <https://docs.travis-ci.com/user/notifications/#configuring-webhook-notifications>`; build output is viewable at https://travis-ci.org/IQSS/dataverse/builds
+
+Our Jenkins config is a work in progress and may be viewed at https://github.com/IQSS/dataverse-jenkins/ A corresponding GitHub webhook is required. Build output is viewable at https://jenkins.dataverse.org/
+
+As always, pull requests to improve our continuous integration configurations are welcome.
+
 The Phoenix Server
 ------------------
 

src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java

@@ -147,7 +147,8 @@ public enum License {
 
     @Size(min=0, max=ARCHIVE_NOTE_MAX_LENGTH)
     @Column(length = ARCHIVE_NOTE_MAX_LENGTH)
-    @ValidateURL()
+    //@ValidateURL() - this validation rule was making a bunch of older legacy datasets invalid;
+    // removed pending further investigation (v4.13)
     private String archiveNote;
 
     @Column(nullable=true, columnDefinition = "TEXT")

src/main/resources/db/migration/V4.12.0.1__4.13-re-sanitize-filemetadata.sql

@@ -0,0 +1,12 @@
+-- let's try again and fix the existing directoryLabels:
+-- (the script shipped with 4.12 was missing the most important line; bad copy-and-paste)
+-- replace any sequences of slashes and backslashes with a single slash:
+UPDATE filemetadata SET directoryLabel = regexp_replace(directoryLabel, '[/\\][/\\]+', '/', 'g');
+-- strip (and replace with a .) any characters that are no longer allowed in the directory labels:
+-- (this line was missing from the script released with 4.12!!)
+UPDATE filemetadata SET directoryLabel = regexp_replace(directoryLabel, '[^A-Za-z0-9_ ./-]+', '.', 'g');
+-- now replace any sequences of .s with a single .:
+UPDATE filemetadata SET directoryLabel = regexp_replace(directoryLabel, '\.\.+', '.', 'g');
+-- get rid of any leading or trailing slashes, spaces, '-'s and '.'s: 
+UPDATE filemetadata SET directoryLabel = regexp_replace(directoryLabel, '^[/ .\-]+', '', '');
+UPDATE filemetadata SET directoryLabel = regexp_replace(directoryLabel, '[/ \.\-]+$', '', '');

src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java

@@ -7,6 +7,7 @@
 import java.util.logging.Logger;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.junit.Ignore;
 import com.jayway.restassured.path.json.JsonPath;
 
 import java.util.List;
@@ -281,10 +282,18 @@ public void testCreatePublishDestroyDataset() {
         Response publishDataverse = UtilIT.publishDataverseViaSword(dataverseAlias, apiToken);
         assertEquals(200, publishDataverse.getStatusCode());
         Response attemptToPublishZeroDotOne = UtilIT.publishDatasetViaNativeApiDeprecated(datasetPersistentId, "minor", apiToken);
+        logger.info("Attempting to publish a minor (\"zero-dot-one\") version");
         attemptToPublishZeroDotOne.prettyPrint();
         attemptToPublishZeroDotOne.then().assertThat()
                 .body("message", equalTo("Cannot publish as minor version. Re-try as major release."))
                 .statusCode(403);
+
+        logger.info("Attempting to publish a major version");
+
+        // 3 second sleep, to allow the indexing to finish:
+        try {
+            Thread.sleep(3000l);
+        } catch (InterruptedException iex) {}
 
         Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetPersistentId, "major", apiToken);
         assertEquals(200, publishDataset.getStatusCode());
@@ -387,6 +396,7 @@ public void testCreatePublishDestroyDataset() {
      * This test requires the root dataverse to be published to pass.
      */
     @Test
+    @Ignore
     public void testExport() {
 
         Response createUser = UtilIT.createRandomUser();
@@ -421,6 +431,13 @@ public void testExport() {
                 .body("message", equalTo("Cannot publish as minor version. Re-try as major release."))
                 .statusCode(403);
 
+        logger.info("In testExport; attempting to publish, as major version");
+
+        // 3 second sleep, to allow the indexing to finish: 
+        try {
+            Thread.sleep(3000l);
+        } catch (InterruptedException iex) {}
+
         Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetPersistentId, "major", apiToken);
         assertEquals(200, publishDataset.getStatusCode());
 
@@ -545,6 +562,11 @@ public void testExcludeEmail() {
         setToExcludeEmailFromExport.then().assertThat()
                 .statusCode(OK.getStatusCode());
 
+        // 3 second sleep, to allow the indexing to finish:
+        try {
+            Thread.sleep(3000l);
+        } catch (InterruptedException iex) {}
+
         Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetPersistentId, "major", apiToken);
         assertEquals(200, publishDataset.getStatusCode());
 
@@ -1622,6 +1644,7 @@ public void testDatasetLocksApi() {
      * This test requires the root dataverse to be published to pass.
      */
     @Test
+    @Ignore
     public void testUpdatePIDMetadataAPI() {
 
         Response createUser = UtilIT.createRandomUser();
@@ -1650,6 +1673,10 @@ public void testUpdatePIDMetadataAPI() {
         Response publishDataverse = UtilIT.publishDataverseViaSword(dataverseAlias, apiToken);
         assertEquals(200, publishDataverse.getStatusCode());
 
+        try {
+            Thread.sleep(3000l);
+        } catch (InterruptedException iex){}
+
         Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetPersistentId, "major", apiToken);
         assertEquals(200, publishDataset.getStatusCode());
 

src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java

@@ -109,7 +109,7 @@ public void testSearchPermisions() throws InterruptedException {
         Response grantUser2AccessOnDataset = UtilIT.grantRoleOnDataverse(dataverseAlias, roleToAssign, "@" + username2, apiToken1);
         grantUser2AccessOnDataset.prettyPrint();
         assertEquals(200, grantUser2AccessOnDataset.getStatusCode());
-        sleep(500l);
+        sleep(3000l);
 
         Response shouldBeVisibleToUser2 = UtilIT.search("id:dataset_" + datasetId1 + "_draft", apiToken2);
         shouldBeVisibleToUser2.prettyPrint();
@@ -658,6 +658,13 @@ public void testIdentifier() {
         publishDataverse.then().assertThat()
                 .statusCode(OK.getStatusCode());
 
+        // 3 second sleep, to allow the indexing to finish:
+        try {
+            Thread.sleep(3000l);
+        } catch (InterruptedException iex) {
+            logger.info("WARNING: failed to execute 3 second sleep");
+        }
+
         Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetPid, "major", apiToken);
         publishDataset.then().assertThat()
                 .statusCode(OK.getStatusCode());