Edit Metadata API endpoint - support empty values

doc/release-notes/11243-editmetadata-api-extension.md

@@ -0,0 +1,5 @@
+### Edit Dataset Metadata API extension
+
+- This endpoint now allows removing fields (by sending empty values), as long as they are not required by the dataset.
+- New ``sourceInternalVersionNumber`` optional query parameter, which prevents inconsistencies by managing updates that
+  may occur from other users while a dataset is being edited.

doc/sphinx-guides/source/_static/api/dataset-edit-metadata-delete-fields-sample.json

@@ -0,0 +1,50 @@
+{
+  "fields": [
+    {
+      "typeName": "alternativeTitle",
+      "multiple": true,
+      "typeClass": "primitive",
+      "value": []
+    },
+    {
+      "typeName": "distributor",
+      "multiple": true,
+      "typeClass": "compound",
+      "value": [
+        {
+          "distributorName": {
+            "typeName": "distributorName",
+            "multiple": false,
+            "typeClass": "primitive",
+            "value": ""
+          },
+          "distributorAffiliation": {
+            "typeName": "distributorAffiliation",
+            "multiple": false,
+            "typeClass": "primitive",
+            "value": ""
+          }
+        }
+      ]
+    },
+    {
+      "fields": [
+        {
+          "typeName": "author",
+          "value": [
+            {
+              "authorName": {
+                "typeName": "authorName",
+                "value": "Belicheck, Bill"
+              },
+              "authorAffiliation": {
+                "typeName": "authorIdentifierScheme",
+                "value": ""
+              }
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

doc/sphinx-guides/source/api/native-api.rst

@@ -2124,6 +2124,30 @@ The fully expanded example above (without environment variables) looks like this
 
 For these edits your JSON file need only include those dataset fields which you would like to edit. A sample JSON file may be downloaded here: :download:`dataset-edit-metadata-sample.json <../_static/api/dataset-edit-metadata-sample.json>` 
 
+This endpoint also allows removing fields, as long as they are not required by the dataset. To remove a field, send an empty value (``""``) for individual fields. For multiple fields, send an empty array (``[]``). A sample JSON file for removing fields may be downloaded here: :download:`dataset-edit-metadata-delete-fields-sample.json <../_static/api/dataset-edit-metadata-delete-fields-sample.json>`
+
+If another user updates the dataset version metadata before you send the update request, data inconsistencies may occur. To prevent this, you can use the optional ``sourceInternalVersionNumber`` query parameter. This parameter must include the internal version number corresponding to the dataset version being updated. Note that internal version numbers increase sequentially with each version update.
+
+If this parameter is provided, the update will proceed only if the internal version number remains unchanged. Otherwise, the request will fail with an error.
+
+Example using ``sourceInternalVersionNumber``:
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/BCCP9Z
+  export SOURCE_INTERNAL_VERSION_NUMBER=5
+
+  curl -H "X-Dataverse-key: $API_TOKEN" -X PUT "$SERVER_URL/api/datasets/:persistentId/editMetadata?persistentId=$PERSISTENT_IDENTIFIER&replace=true&sourceInternalVersionNumber=$SOURCE_INTERNAL_VERSION_NUMBER" --upload-file dataset-update-metadata.json
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/datasets/:persistentId/editMetadata/?persistentId=doi:10.5072/FK2/BCCP9Z&replace=true&sourceInternalVersionNumber=5" --upload-file dataset-update-metadata.json
+
+
 Delete Dataset Metadata
 ~~~~~~~~~~~~~~~~~~~~~~~
 

src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java

@@ -923,6 +923,42 @@ public List<DatasetFieldType> findAllInMetadataBlockAndDataverse(MetadataBlock m
         return em.createQuery(criteriaQuery).getResultList();
     }
 
+    public boolean isFieldRequiredInDataverse(DatasetFieldType datasetFieldType, Dataverse dataverse) {
+        CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder();
+        CriteriaQuery<Long> criteriaQuery = criteriaBuilder.createQuery(Long.class);
+
+        Root<Dataverse> dataverseRoot = criteriaQuery.from(Dataverse.class);
+        Root<DatasetFieldType> datasetFieldTypeRoot = criteriaQuery.from(DatasetFieldType.class);
+
+        // Join Dataverse with DataverseFieldTypeInputLevel on the "dataverseFieldTypeInputLevels" attribute, using a LEFT JOIN.
+        Join<Dataverse, DataverseFieldTypeInputLevel> datasetFieldTypeInputLevelJoin = dataverseRoot.join("dataverseFieldTypeInputLevels", JoinType.LEFT);
+
+        // Define a predicate to include DatasetFieldTypes that are marked as required in the input level.
+        Predicate requiredAsInputLevelPredicate = criteriaBuilder.and(
+                criteriaBuilder.equal(datasetFieldTypeRoot, datasetFieldTypeInputLevelJoin.get("datasetFieldType")),
+                criteriaBuilder.isTrue(datasetFieldTypeInputLevelJoin.get("required"))
+        );
+
+        // Define a predicate to include the required fields in the installation.
+        Predicate requiredInTheInstallationPredicate = buildFieldRequiredInTheInstallationPredicate(criteriaBuilder, datasetFieldTypeRoot);
+
+        // Build the final WHERE clause by combining all the predicates.
+        criteriaQuery.where(
+                criteriaBuilder.equal(dataverseRoot.get("id"), dataverse.getId()),
+                criteriaBuilder.equal(datasetFieldTypeRoot.get("id"), datasetFieldType.getId()),
+                criteriaBuilder.or(
+                        requiredAsInputLevelPredicate,
+                        requiredInTheInstallationPredicate
+                )
+        );
+
+        criteriaQuery.select(criteriaBuilder.count(datasetFieldTypeRoot));
+
+        Long count = em.createQuery(criteriaQuery).getSingleResult();
+
+        return count != null && count > 0;
+    }
+
     private Predicate buildFieldPresentInDataversePredicate(Dataverse dataverse, boolean onlyDisplayedOnCreate, CriteriaQuery<DatasetFieldType> criteriaQuery, CriteriaBuilder criteriaBuilder, Root<DatasetFieldType> datasetFieldTypeRoot, Root<MetadataBlock> metadataBlockRoot) {
         Root<Dataverse> dataverseRoot = criteriaQuery.from(Dataverse.class);
 
@@ -960,7 +996,7 @@ private Predicate buildFieldPresentInDataversePredicate(Dataverse dataverse, boo
         // Define a predicate to exclude DatasetFieldTypes that have no associated input level (i.e., the subquery does not return a result).
         Predicate hasNoInputLevelPredicate = criteriaBuilder.not(criteriaBuilder.exists(subquery));
 
-        // Define a predicate to include the required fields in Dataverse.
+        // Define a predicate to include the required fields in the installation.
         Predicate fieldRequiredInTheInstallation = buildFieldRequiredInTheInstallationPredicate(criteriaBuilder, datasetFieldTypeRoot);
 
         // Define a predicate for displaying DatasetFieldTypes on create.

src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java

@@ -2,6 +2,7 @@
 
 import edu.harvard.iq.dataverse.actionlogging.ActionLogRecord;
 import edu.harvard.iq.dataverse.actionlogging.ActionLogServiceBean;
+import edu.harvard.iq.dataverse.dataset.DatasetFieldsValidator;
 import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
 import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean;
 import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItemServiceBean;
@@ -189,6 +190,9 @@ public class EjbDataverseEngine {
 
     @EJB
     DataverseFeaturedItemServiceBean dataverseFeaturedItemServiceBean;
+
+    @EJB
+    DatasetFieldsValidator datasetFieldsValidator;
 
     @EJB
     EjbDataverseEngineInner innerEngine;
@@ -531,6 +535,11 @@ public DataverseFeaturedItemServiceBean dataverseFeaturedItems() {
                     return dataverseFeaturedItemServiceBean;
                 }
 
+                @Override
+                public DatasetFieldsValidator datasetFieldsValidator() {
+                    return datasetFieldsValidator;
+                }
+
                 @Override
                 public StorageUseServiceBean storageUse() {
                     return storageUseService;

src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java

@@ -51,10 +51,7 @@
 
 import java.io.InputStream;
 import java.net.URI;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Map;
-import java.util.UUID;
+import java.util.*;
 import java.util.concurrent.Callable;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -446,6 +443,14 @@ public Command<DatasetVersion> handleLatestPublished() {
         return dsv;
     }
 
+    protected void validateInternalVersionNumberIsNotOutdated(Dataset dataset, int internalVersion) throws WrappedResponse {
+        if (dataset.getLatestVersion().getVersion() > internalVersion) {
+            throw new WrappedResponse(
+                    badRequest(BundleUtil.getStringFromBundle("abstractApiBean.error.datasetInternalVersionNumberIsOutdated", Collections.singletonList(Integer.toString(internalVersion))))
+            );
+        }
+    }
+
     protected DataFile findDataFileOrDie(String id) throws WrappedResponse {
         DataFile datafile;
         if (id.equals(PERSISTENT_ID_KEY)) {