Skip to content

Commit c25b5d0

Browse files
janvanmansumjanvanmansum
committed
Retrieving current dataset metadata more efficiently and less often.
1 parent 6aaf73a commit c25b5d0

File tree

8 files changed

+52
-34
lines changed

8 files changed

+52
-34
lines changed

‎pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@
6565
<dependency>
6666
<groupId>nl.knaw.dans</groupId>
6767
<artifactId>dans-dataverse-client-lib</artifactId>
68-
<version>1.0.2</version>
68+
<version>1.0.3-SNAPSHOT</version>
6969
</dependency>
7070
<dependency>
7171
<groupId>nl.knaw.dans</groupId>

‎src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingService.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import nl.knaw.dans.dvingest.core.yaml.EditPermissions;
2222
import nl.knaw.dans.lib.dataverse.DataverseException;
2323
import nl.knaw.dans.lib.dataverse.model.dataset.Dataset;
24+
import nl.knaw.dans.lib.dataverse.model.dataset.DatasetVersion;
2425

2526
import java.io.IOException;
2627
import java.nio.file.Path;
@@ -30,7 +31,7 @@ public interface DansBagMappingService {
3031

3132
DansBagDeposit readDansDeposit(Path depositDir) throws InvalidDepositException;
3233

33-
Dataset getDatasetMetadataFromDansDeposit(DansBagDeposit dansDeposit);
34+
Dataset getDatasetMetadataFromDansDeposit(DansBagDeposit dansDeposit, DatasetVersion updatesDataset);
3435

3536
EditFiles getEditFilesFromDansDeposit(DansBagDeposit dansDeposit, String updatesDataset);
3637

‎src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingServiceImpl.java

Lines changed: 28 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import nl.knaw.dans.lib.dataverse.DataverseException;
3434
import nl.knaw.dans.lib.dataverse.model.RoleAssignment;
3535
import nl.knaw.dans.lib.dataverse.model.dataset.Dataset;
36+
import nl.knaw.dans.lib.dataverse.model.dataset.DatasetVersion;
3637
import nl.knaw.dans.lib.dataverse.model.user.AuthenticatedUser;
3738
import nl.knaw.dans.lib.util.ZipUtil;
3839
import org.apache.commons.lang3.StringUtils;
@@ -88,37 +89,38 @@ public String getUpdatesDataset(Path depositDir) throws IOException, DataverseEx
8889
.findFirst().orElseThrow(() -> new IllegalArgumentException("No bag found in deposit"));
8990
var bagInfo = new LightweightBagInfo(bag.resolve("bag-info.txt"));
9091
var isVersionOf = bagInfo.get("Is-Version-Of");
91-
if (isVersionOf != null) {
92-
log.debug("Found Is-Version-Of in bag-info.txt, so this is an update-deposit: {}", isVersionOf);
93-
List<String> results;
94-
if (dansDepositProperties.getSwordToken() != null) {
95-
log.debug("Found sword token in deposit.properties, looking for target dataset by sword token");
96-
results = dataverseService.findDoiByMetadataField("dansSwordToken", dansDepositProperties.getSwordToken());
97-
}
98-
else if (depositToDvDatasetMetadataMapper.isMigration()) {
99-
log.debug("This is a migration deposit, looking for target dataset by dansBagIt. Note that this will only work for two versions of the same dataset");
100-
results = dataverseService.findDoiByMetadataField("dansBagId", isVersionOf);
101-
102-
}
103-
else {
104-
throw new IllegalArgumentException("Update deposit should have either a sword token or be a migration deposit");
105-
}
106-
if (results.size() == 1) {
107-
return results.get(0);
108-
}
109-
else {
110-
throw new IllegalArgumentException("Update deposit should update exactly one dataset, found " + results.size());
111-
}
112-
}
113-
else {
92+
93+
if (isVersionOf == null) {
11494
log.debug("No Is-Version-Of found in bag-info.txt, so this is a deposit of a new dataset");
11595
return null;
11696
}
97+
98+
log.debug("Found Is-Version-Of in bag-info.txt, so this is an update-deposit: {}", isVersionOf);
99+
List<String> results;
100+
101+
if (dansDepositProperties.getSwordToken() != null) {
102+
log.debug("Found sword token in deposit.properties, looking for target dataset by sword token");
103+
results = dataverseService.findDoiByMetadataField("dansSwordToken", dansDepositProperties.getSwordToken());
104+
}
105+
else if (depositToDvDatasetMetadataMapper.isMigration()) {
106+
log.debug("This is a migration deposit, looking for target dataset by dansBagId");
107+
results = dataverseService.findDoiByMetadataField("dansBagId", isVersionOf);
108+
}
109+
else {
110+
throw new IllegalArgumentException("Update deposit should have either a sword token or be a migration deposit");
111+
}
112+
113+
if (results.size() == 1) {
114+
return results.get(0);
115+
}
116+
else {
117+
throw new IllegalArgumentException("Update deposit should update exactly one dataset, found " + results.size());
118+
}
117119
}
118120
}
119121

120122
@Override
121-
public Dataset getDatasetMetadataFromDansDeposit(DansBagDeposit dansDeposit) {
123+
public Dataset getDatasetMetadataFromDansDeposit(DansBagDeposit dansDeposit, DatasetVersion currentMetadata) {
122124
var dataset = depositToDvDatasetMetadataMapper.toDataverseDataset(
123125
dansDeposit.getDdm(),
124126
dansDeposit.getOtherDoiId(),
@@ -136,6 +138,8 @@ public Dataset getDatasetMetadataFromDansDeposit(DansBagDeposit dansDeposit) {
136138
if (!dansDeposit.allowAccessRequests() && StringUtils.isBlank(version.getTermsOfAccess())) {
137139
version.setTermsOfAccess("N/a");
138140
}
141+
142+
139143
version.setLicense(supportedLicenses.getLicenseFromDansDeposit(dansDeposit));
140144
return dataset;
141145
}

‎src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositConverter.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,21 +21,23 @@
2121
import nl.knaw.dans.dvingest.core.yaml.EditFilesRoot;
2222
import nl.knaw.dans.dvingest.core.yaml.EditPermissionsRoot;
2323
import nl.knaw.dans.dvingest.core.yaml.UpdateState;
24+
import nl.knaw.dans.lib.dataverse.model.dataset.DatasetVersion;
2425

2526
import java.io.IOException;
2627

2728
@AllArgsConstructor
2829
public class DansDepositConverter {
2930
private final DansBagDeposit dansDeposit;
3031
private final String updatesDataset;
32+
private final DatasetVersion currentMetadata;
3133
private final DansBagMappingService mappingService;
3234
private final YamlService yamlService;
3335

3436
public void run() throws IOException {
3537
// TODO: pass to getEditFilesFromDansDeposit so that update-deposit can register it as a replaced file
3638
var originalMetadataPath = mappingService.packageOriginalMetadata(dansDeposit);
3739

38-
var dataset = mappingService.getDatasetMetadataFromDansDeposit(dansDeposit);
40+
var dataset = mappingService.getDatasetMetadataFromDansDeposit(dansDeposit, currentMetadata);
3941
yamlService.writeYaml(dataset, dansDeposit.getBagDir().resolve("dataset.yml"));
4042

4143
var editFiles = mappingService.getEditFilesFromDansDeposit(dansDeposit, updatesDataset);
@@ -48,5 +50,4 @@ public void run() throws IOException {
4850
updateState.setAction("publish-major");
4951
yamlService.writeYaml(updateState, dansDeposit.getBagDir().resolve("update-state.yml"));
5052
}
51-
5253
}

‎src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositSupport.java

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import nl.knaw.dans.dvingest.core.service.DataverseService;
3030
import nl.knaw.dans.dvingest.core.service.YamlService;
3131
import nl.knaw.dans.lib.dataverse.DataverseException;
32+
import nl.knaw.dans.lib.dataverse.model.dataset.DatasetVersion;
3233

3334
import java.io.IOException;
3435
import java.nio.file.Path;
@@ -49,7 +50,8 @@ public class DansDepositSupport implements Deposit {
4950

5051
private DansBagDeposit dansDeposit;
5152

52-
public DansDepositSupport(DataverseIngestDeposit dataverseIngestDeposit, ValidateDansBagService validateDansBagService, DansBagMappingService dansBagMappingService, DataverseService dataverseService, YamlService yamlService) {
53+
public DansDepositSupport(DataverseIngestDeposit dataverseIngestDeposit, ValidateDansBagService validateDansBagService, DansBagMappingService dansBagMappingService,
54+
DataverseService dataverseService, YamlService yamlService) {
5355
this.validateDansBagService = validateDansBagService;
5456
this.ingestDataverseIngestDeposit = dataverseIngestDeposit;
5557
this.dansBagMappingService = dansBagMappingService;
@@ -69,11 +71,13 @@ public boolean convertDansDepositIfNeeded() {
6971
log.info("Converting deposit to Dataverse ingest metadata");
7072
try {
7173
var updatesDataset = dansBagMappingService.getUpdatesDataset(ingestDataverseIngestDeposit.getLocation());
74+
DatasetVersion currentMetadata = null;
7275
if (updatesDataset != null) {
7376
ingestDataverseIngestDeposit.updateProperties(Map.of(UPDATES_DATASET_KEY, updatesDataset));
77+
currentMetadata = dataverseService.getDatasetMetadata(updatesDataset);
7478
}
7579
dansDeposit = dansBagMappingService.readDansDeposit(ingestDataverseIngestDeposit.getLocation());
76-
new DansDepositConverter(dansDeposit, updatesDataset, dansBagMappingService, yamlService).run();
80+
new DansDepositConverter(dansDeposit, updatesDataset, currentMetadata, dansBagMappingService, yamlService).run();
7781
log.info("Conversion successful");
7882
return true;
7983
}
@@ -119,7 +123,8 @@ public void onSuccess(@NonNull String pid, String message) {
119123
"identifier.urn", nbn
120124
)
121125
);
122-
} catch (IOException | DataverseException e) {
126+
}
127+
catch (IOException | DataverseException e) {
123128
throw new RuntimeException("Error getting URN:NBN", e); // Cancelling the "success"
124129
}
125130
}

‎src/main/java/nl/knaw/dans/dvingest/core/service/DataverseService.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,4 +72,6 @@ public interface DataverseService {
7272
void addEmbargo(String pid, Embargo embargo) throws IOException, DataverseException;
7373

7474
List<String> findDoiByMetadataField(String fieldName, String value) throws IOException, DataverseException;
75+
76+
DatasetVersion getDatasetMetadata(String pid) throws IOException, DataverseException;
7577
}

‎src/main/java/nl/knaw/dans/dvingest/core/service/DataverseServiceImpl.java

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,11 @@ public String getDatasetUrnNbn(String pid) throws IOException, DataverseExceptio
206206
.findFirst().orElseThrow(() -> new IllegalStateException("No URN:NBN found in dataset"));
207207
}
208208

209+
@Override
210+
public DatasetVersion getDatasetMetadata(String pid) throws IOException, DataverseException {
211+
return dataverseClient.dataset(pid).getVersion().getData();
212+
}
213+
209214
// TODO: move this to dans-dataverse-client-lib; it is similar to awaitLockState.
210215
public void waitForState(String datasetId, String expectedState) {
211216
var numberOfTimesTried = 0;
@@ -239,8 +244,8 @@ public void waitForState(String datasetId, String expectedState) {
239244
}
240245

241246
private String getDatasetState(String datasetId) throws IOException, DataverseException {
242-
var version = dataverseClient.dataset(datasetId).getLatestVersion();
243-
return version.getData().getLatestVersion().getVersionState();
247+
var version = dataverseClient.dataset(datasetId).getVersion(Version.LATEST.toString(), true);
248+
return version.getData().getVersionState();
244249

245250
}
246251
}

‎src/test/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositConverterTest.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ public void run_converts_dans_sword_all_mappings_example_to_dataverse_ingest_dep
4444
var deposit = dansBagDepositReader.readDeposit(depositDir);
4545

4646
// When
47-
new DansDepositConverter(deposit, null, mappingService, yamlService).run();
47+
new DansDepositConverter(deposit, null, null, mappingService, yamlService).run();
4848

4949
// Then
5050
assertThat(deposit.getBagDir().resolve("dataset.yml")).exists();

0 commit comments

Comments
 (0)