Skip to content

Commit bec3ed0

Browse files
slaurenzslaurenz
authored
Merge pull request #14 from eu-digital-green-certificates/feat/improve_download
Feat/improve download
2 parents e451476 + 7e24350 commit bec3ed0

File tree

18 files changed

+628
-48
lines changed

18 files changed

+628
-48
lines changed

‎pom.xml‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,14 +34,14 @@
3434
<lombok.version>1.18.22</lombok.version>
3535
<springdoc.version>1.6.0</springdoc.version>
3636
<mapstruct.version>1.4.2.Final</mapstruct.version>
37-
<junit.version>5.8.1</junit.version>
37+
<junit.jupiter.version>5.8.2</junit.jupiter.version>
3838
<mockito.version>4.1.0</mockito.version>
3939
<bcpkix.version>1.70</bcpkix.version>
4040
<okhttp.version>4.9.1</okhttp.version>
4141
<shedlock.version>4.30.0</shedlock.version>
4242
<nimbusds.version>9.9.2</nimbusds.version>
4343
<dgc.lib.version>1.1.13</dgc.lib.version>
44-
<dgc.bloomfilter.version>0.0.0-09cb38e</dgc.bloomfilter.version>
44+
<dgc.bloomfilter.version>0.0.0-29d785a</dgc.bloomfilter.version>
4545
<dgc.partialvarriablehashfilter.version>0.0.0-613ab9b</dgc.partialvarriablehashfilter.version>
4646
<sap.cloud.sdk.version>3.57.0</sap.cloud.sdk.version>
4747
<jjwt.version>0.11.2</jjwt.version>
@@ -144,6 +144,13 @@
144144
<type>pom</type>
145145
<scope>import</scope>
146146
</dependency>
147+
<dependency>
148+
<groupId>io.zonky.test.postgres</groupId>
149+
<artifactId>embedded-postgres-binaries-bom</artifactId>
150+
<version>9.6.16</version>
151+
<type>pom</type>
152+
<scope>import</scope>
153+
</dependency>
147154
</dependencies>
148155
</dependencyManagement>
149156

@@ -195,7 +202,7 @@
195202
<dependency>
196203
<groupId>org.junit.jupiter</groupId>
197204
<artifactId>junit-jupiter-api</artifactId>
198-
<version>${junit.version}</version>
205+
<version>${junit.jupiter.version}</version>
199206
<scope>test</scope>
200207
</dependency>
201208
<dependency>
@@ -301,6 +308,18 @@
301308
<version>0.11.2</version>
302309
<scope>runtime</scope>
303310
</dependency>
311+
<dependency>
312+
<groupId>io.zonky.test</groupId>
313+
<artifactId>embedded-database-spring-test</artifactId>
314+
<version>2.1.1</version>
315+
<scope>test</scope>
316+
</dependency>
317+
<dependency>
318+
<groupId>io.zonky.test</groupId>
319+
<artifactId>embedded-postgres</artifactId>
320+
<version>1.2.10</version>
321+
<scope>test</scope>
322+
</dependency>
304323
</dependencies>
305324

306325
<build>

‎src/main/java/eu/europa/ec/dgc/revocationdistribution/config/DgcConfigProperties.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ public class DgcConfigProperties {
4141
@Setter
4242
public static class GatewayDownload {
4343
private Integer timeInterval;
44+
private Integer downloadLimit;
4445
private Integer lockLimit;
4546
}
4647

‎src/main/java/eu/europa/ec/dgc/revocationdistribution/controller/RevocationListController.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ public class RevocationListController {
7979
@GetMapping(path = "lists", produces = MediaType.APPLICATION_JSON_VALUE)
8080
@Operation(
8181
summary = "Returns an overview about all available revocation lists.",
82-
description = "This method returns an over about available revocation lists for each KID. The response "
82+
description = "This method returns an overview about available revocation lists for each KID. The response "
8383
+ "contains for all available KIDs the last modification date, the used hash types etc.",
8484
tags = {"Revocation Lists"},
8585
parameters = {
@@ -763,7 +763,7 @@ private SliceType getSliceDataType(String sliceDataTypeHeader) {
763763
try {
764764
return SliceType.valueOf(sliceDataTypeHeader);
765765
} catch (IllegalArgumentException e) {
766-
log.info("Unkown slice data type requested {}", sliceDataTypeHeader);
766+
log.info("Unknown slice data type requested {}", sliceDataTypeHeader);
767767
throw new BadRequestException("Requested slice data type unknown: " + sliceDataTypeHeader);
768768
}
769769
}

‎src/main/java/eu/europa/ec/dgc/revocationdistribution/entity/HashesEntity.java‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,10 @@
2020

2121
package eu.europa.ec.dgc.revocationdistribution.entity;
2222

23+
import java.util.UUID;
2324
import javax.persistence.Column;
2425
import javax.persistence.Entity;
26+
import javax.persistence.GeneratedValue;
2527
import javax.persistence.Id;
2628
import javax.persistence.JoinColumn;
2729
import javax.persistence.OneToOne;
@@ -30,6 +32,8 @@
3032
import lombok.Getter;
3133
import lombok.NoArgsConstructor;
3234
import lombok.Setter;
35+
import org.hibernate.annotations.GenericGenerator;
36+
import org.springframework.data.domain.Persistable;
3337

3438

3539
@Getter
@@ -38,12 +42,17 @@
3842
@Table(name = "hashes")
3943
@AllArgsConstructor
4044
@NoArgsConstructor
41-
public class HashesEntity {
45+
public class HashesEntity implements Persistable<UUID> {
46+
47+
48+
@Id
49+
@GeneratedValue(generator = "uuid2")
50+
@GenericGenerator(name = "uuid2", strategy = "uuid2")
51+
private UUID id;
4252

4353
/**
4454
* The revoked hash.
4555
*/
46-
@Id
4756
@Column(name = "hash", nullable = false)
4857
private String hash;
4958

@@ -87,4 +96,13 @@ public class HashesEntity {
8796
@Column(name = "updated")
8897
private boolean updated;
8998

99+
@Override
100+
public UUID getId() {
101+
return id;
102+
}
103+
104+
@Override
105+
public boolean isNew() {
106+
return true;
107+
}
90108
}

‎src/main/java/eu/europa/ec/dgc/revocationdistribution/model/ChangeList.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,6 @@
3232
public class ChangeList {
3333
private List<ChangeListItem> created = new ArrayList<>();
3434
private List<ChangeListItem> updated = new ArrayList<>();
35-
private List<ChangeListItem> deleted = new ArrayList<>();
35+
private List<String> deletedKids = new ArrayList<>();
3636
}
3737

‎src/main/java/eu/europa/ec/dgc/revocationdistribution/model/ChangeListItem.java‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,20 +32,17 @@ public class ChangeListItem {
3232
/**
3333
* Constructor to create a new Change List item with the data from a kid view entity.
3434
* @param kve The kid view entity to get the data from.
35-
* @param oldStorageMode The old storage mode [POINT, VECTOR, COORDINATE] of the item, if present.
3635
*/
37-
public ChangeListItem(KidViewEntity kve, String oldStorageMode) {
36+
public ChangeListItem(KidViewEntity kve) {
3837
this.kidId = kve.getKid();
3938
this.lastUpdated = kve.getLastUpdated();
4039
this.expired = kve.getExpired();
4140
this.newStorageMode = kve.getStorageMode();
42-
this.oldStorageMode = oldStorageMode;
41+
4342
}
4443

4544
private String kidId;
4645

47-
private String oldStorageMode;
48-
4946
private String newStorageMode;
5047

5148
private ZonedDateTime lastUpdated;

‎src/main/java/eu/europa/ec/dgc/revocationdistribution/repository/HashesRepository.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,10 +38,10 @@ public interface HashesRepository extends JpaRepository<HashesEntity, String> {
3838
@Query("DELETE HashesEntity h WHERE h.batch = null")
3939
void deleteAllOrphanedHashes();
4040

41-
@Query("SELECT h.id FROM HashesEntity h WHERE h.id IN :hashes")
41+
@Query("SELECT h.hash FROM HashesEntity h WHERE h.hash IN :hashes")
4242
List<String> getHashesPresentInListAndDb(@Param("hashes") List<String> hashes);
4343

44-
@Query("SELECT h.id FROM HashesEntity h INNER JOIN h.batch b WHERE h.id IN :hashes AND b.expires > :checkTime")
44+
@Query("SELECT h.hash FROM HashesEntity h INNER JOIN h.batch b WHERE h.hash IN :hashes AND b.expires > :checkTime")
4545
List<String> getHashesPresentInListAndDbAndNotExpired(
4646
@Param("hashes") List<String> hashes,
4747
@Param("checkTime") ZonedDateTime checkTime);

‎src/main/java/eu/europa/ec/dgc/revocationdistribution/service/GeneratorService.java‎

Lines changed: 43 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import eu.europa.ec.dgc.revocationdistribution.repository.PointViewRepository;
3737
import eu.europa.ec.dgc.revocationdistribution.repository.SliceRepository;
3838
import eu.europa.ec.dgc.revocationdistribution.repository.VectorViewRepository;
39+
import eu.europa.ec.dgc.revocationdistribution.utils.HelperFunctions;
3940
import java.util.ArrayList;
4041
import java.util.List;
4142
import java.util.Map;
@@ -78,6 +79,8 @@ public class GeneratorService {
7879

7980
private final CoordinateViewMapper coordinateViewMapper;
8081

82+
private final HelperFunctions helperFunctions;
83+
8184
private String etag;
8285
private String oldEtag;
8386

@@ -128,52 +131,68 @@ private ChangeList generateList() {
128131
Map<String, RevocationListJsonResponseItemDto> itemsMap =
129132
items.stream().collect(Collectors.toMap(i -> i.getKid(), i -> i));
130133

134+
List<String> goneKids = new ArrayList<>(itemsMap.keySet());
135+
goneKids.removeAll(kidViewEntityList.stream().map(KidViewEntity::getKid).collect(Collectors.toList()));
136+
changeList.getDeletedKids().addAll(goneKids);
137+
log.debug("Gone kid entries: {}", goneKids);
138+
139+
140+
log.trace("Update items start");
131141
//Update Items
132142
kidViewEntityList.stream().forEach(kve -> {
133143

134144
if (kve.getTypes().isEmpty() && kve.getExpired() == null) {
135145
log.debug("Delete kid entry : {} ", kve.getKid());
136146
if (itemsMap.remove(kve.getKid()) != null) {
137-
changeList.getDeleted().add(new ChangeListItem(kve, null));
147+
changeList.getDeletedKids().add(kve.getKid());
138148
}
139149
} else {
140-
if (kve.isUpdated()) {
141-
RevocationListJsonResponseItemDto item;
142-
item = new RevocationListJsonResponseItemDto();
143-
item.setKid(kve.getKid());
144-
item.setLastUpdated(kve.getLastUpdated());
145-
item.setHashTypes(kve.getTypes());
146-
item.setMode(kve.getStorageMode());
147-
item.setExpires(kve.getExpired());
148150

151+
RevocationListJsonResponseItemDto item = getRevocationListJsonItem(kve);
152+
153+
if (kve.isUpdated() || (!itemsMap.containsKey(kve.getKid()))) {
154+
155+
itemsMap.put(item.getKid(), item);
156+
changeList.getUpdated().add(new ChangeListItem(kve));
157+
158+
} else {
149159
RevocationListJsonResponseItemDto oldItem;
150-
oldItem = itemsMap.put(item.getKid(), item);
160+
oldItem = itemsMap.get(kve.getKid());
151161

152-
if (oldItem != null) {
153-
changeList.getUpdated().add(new ChangeListItem(kve, oldItem.getMode()));
154-
} else {
155-
changeList.getCreated().add(new ChangeListItem(kve, null));
162+
if (!helperFunctions.compareRevocationListItems(item, oldItem)) {
163+
itemsMap.put(item.getKid(), item);
164+
changeList.getUpdated().add(new ChangeListItem(kve));
156165
}
157166
}
158167
}
159168
});
160-
169+
log.trace("update items stop");
161170
RevocationListJsonEntity revocationListJsonEntity = new RevocationListJsonEntity();
162171
revocationListJsonEntity.setEtag(etag);
163172
revocationListJsonEntity.setJsonData(new ArrayList<>(itemsMap.values()));
164-
173+
log.trace("before save");
165174
revocationListService.saveRevocationListJson(revocationListJsonEntity);
166-
167-
log.trace(itemsMap.values().toString());
175+
log.trace("create list finished");
176+
//log.trace(itemsMap.values().toString());
168177
return changeList;
169178
}
170179

180+
private RevocationListJsonResponseItemDto getRevocationListJsonItem(KidViewEntity kve) {
181+
RevocationListJsonResponseItemDto item = new RevocationListJsonResponseItemDto();
182+
183+
item.setKid(kve.getKid());
184+
item.setLastUpdated(kve.getLastUpdated());
185+
item.setHashTypes(kve.getTypes());
186+
item.setMode(kve.getStorageMode());
187+
item.setExpires(kve.getExpired());
188+
189+
return item;
190+
}
191+
192+
171193
private void handleChangeList(ChangeList changeList) {
172194
//handle deleted kIds
173-
List<String> deletedKids =
174-
changeList.getDeleted().stream().map(ChangeListItem::getKidId).collect(Collectors.toList());
175-
176-
markDataForRemoval(deletedKids);
195+
markDataForRemoval(changeList.getDeletedKids());
177196

178197
//handle updated kIds
179198
List<String> updatedKids =
@@ -238,7 +257,7 @@ private void generatePartitionsForKidInVectorMode(ChangeListItem changeItem) {
238257
//get all ids for kId
239258
List<String> partitionIds = vectorViewRepository.findDistinctIdsByKid(changeItem.getKidId());
240259

241-
log.debug("PartionIds {}", partitionIds);
260+
log.debug("PartitionIds {}", partitionIds);
242261

243262
for (String partitionId : partitionIds) {
244263
List<ChunkMetaViewDto> entities =
@@ -257,7 +276,7 @@ private void generatePartitionsForKidInCoordinateMode(ChangeListItem changeItem)
257276
//get all ids for kId
258277
List<String> partitionIds = coordinateViewRepository.findDistinctIdsByKid(changeItem.getKidId());
259278

260-
log.debug("PartionIds {}", partitionIds);
279+
log.debug("PartitionIds {}", partitionIds);
261280

262281
for (String partitionId : partitionIds) {
263282
List<ChunkMetaViewDto> entities =

‎src/main/java/eu/europa/ec/dgc/revocationdistribution/service/PartitionGeneratorService.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ public class PartitionGeneratorService {
6666

6767
public void generatePartition(String etag, List<ChunkMetaViewDto> entities,
6868
String kid, String id, String storageMode) {
69-
69+
log.info("Generate Partition of entities: {}, kId: {}, ID: {}, etag: {}", entities.size(), kid, id, etag);
7070
if (sliceCalculationBloomFilter.isPresent()) {
7171
generatePartition(etag, entities, kid, id, sliceCalculationBloomFilter.get(), storageMode);
7272
}
@@ -98,7 +98,7 @@ private void generatePartition(String etag, List<ChunkMetaViewDto> entities,
9898
if (!Objects.equals(mve.getKid(), kid) || !Objects.equals(mve.getId(), id)) {
9999
log.error("Kid and/or id does not match: kid: {} , {} id {}, {}", kid, mve.getKid(), id, mve.getId());
100100
} else {
101-
101+
log.info("Number of hashes per slice: {}",mve.getHashes().length);
102102
SliceDataDto sliceDataDto = sliceCalculation.calculateSlice(mve.getHashes(), storageMode);
103103
if (sliceDataDto != null) {
104104
Map<String, PartitionChunksJsonItemDto> chunkItemsMap;

‎src/main/java/eu/europa/ec/dgc/revocationdistribution/service/RevocationListDownloadServiceGatewayImpl.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -99,9 +99,9 @@ public void downloadRevocationList() {
9999

100100
boolean needsCalculation = getNeedsCalculation();
101101

102-
int timeInterval = properties.getRevocationListDownload().getTimeInterval();
102+
int downloadLimit = properties.getRevocationListDownload().getDownloadLimit();
103103

104-
ZonedDateTime abortTime = ZonedDateTime.now().plusSeconds((timeInterval / 1000) / 2);
104+
ZonedDateTime abortTime = ZonedDateTime.now().plusSeconds(downloadLimit / 1000);
105105

106106
DgcGatewayRevocationListDownloadIterator revocationListIterator;
107107

@@ -133,7 +133,7 @@ public void downloadRevocationList() {
133133
RevocationBatchDto revocationBatchDto =
134134
dgcGatewayDownloadConnector.getRevocationListBatchById(batchListItem.getBatchId());
135135

136-
log.trace(revocationBatchDto.toString());
136+
//log.trace(revocationBatchDto.toString());
137137

138138
revocationListservice.updateRevocationListBatch(batchListItem.getBatchId(), revocationBatchDto);
139139
log.info("Downloaded batch: {}", batchListItem.getBatchId());

0 commit comments

Comments
 (0)