Moved partial variable hash filter calculation to external lib

slaurenz · slaurenz · commit 1427f511e534 · 2022-03-15T10:00:45.000+01:00
diff --git a/pom.xml b/pom.xml
@@ -42,6 +42,7 @@
 		<nimbusds.version>9.9.2</nimbusds.version>
 		<dgc.lib.version>1.1.13</dgc.lib.version>
 		<dgc.bloomfilter.version>0.0.0-09cb38e</dgc.bloomfilter.version>
+		<dgc.partialvarriablehashfilter.version>0.0.0-613ab9b</dgc.partialvarriablehashfilter.version>
 		<sap.cloud.sdk.version>3.57.0</sap.cloud.sdk.version>
 		<jjwt.version>0.11.2</jjwt.version>
 		<!-- plugins -->
@@ -157,6 +158,11 @@
 			<artifactId>bloomfilter</artifactId>
 			<version>${dgc.bloomfilter.version}</version>
 		</dependency>
+		<dependency>
+			<groupId>eu.europa.ec.dgc</groupId>
+			<artifactId>partialvarriablehashfilter</artifactId>
+			<version>${dgc.partialvarriablehashfilter.version}</version>
+		</dependency>
 		<dependency>
 			<groupId>org.springframework.boot</groupId>
 			<artifactId>spring-boot-starter</artifactId>
diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/config/DgcConfigProperties.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/config/DgcConfigProperties.java
@@ -60,10 +60,7 @@ public static class VarHashListConfig {
         private String type = "hash_list";
         private String version;
         private float probRate;
-        private int minByteCount = 4;
+        private byte minByteCount = 4;
 
-        public int getMinByteCount() {
-            return minByteCount > 255 ? 255 : minByteCount;
-        }
     }
 }
diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/GeneratorService.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/GeneratorService.java
@@ -227,7 +227,8 @@ private void generatePartitionsForKidInPointMode(ChangeListItem changeItem) {
         List<ChunkMetaViewDto> entities = pointViewRepository.findAllByKid(changeItem.getKidId()).stream()
             .map(pointViewMapper::map).collect(Collectors.toList());
 
-        partitionGeneratorService.generatePartition(etag, entities, changeItem.getKidId(), null);
+        partitionGeneratorService.generatePartition(
+            etag, entities, changeItem.getKidId(), null, changeItem.getNewStorageMode());
 
 
     }
@@ -244,7 +245,8 @@ private void generatePartitionsForKidInVectorMode(ChangeListItem changeItem) {
                 vectorViewRepository.findAllByKidAndId(changeItem.getKidId(), partitionId).stream()
                     .map(vectorViewMapper::map).collect(Collectors.toList());
 
-            partitionGeneratorService.generatePartition(etag, entities, changeItem.getKidId(), partitionId);
+            partitionGeneratorService.generatePartition(
+                etag, entities, changeItem.getKidId(), partitionId, changeItem.getNewStorageMode());
 
         }
 
@@ -262,7 +264,8 @@ private void generatePartitionsForKidInCoordinateMode(ChangeListItem changeItem)
                 coordinateViewRepository.findAllByKidAndId(changeItem.getKidId(), partitionId).stream()
                     .map(coordinateViewMapper::map).collect(Collectors.toList());
 
-            partitionGeneratorService.generatePartition(etag, entities, changeItem.getKidId(), partitionId);
+            partitionGeneratorService.generatePartition(
+                etag, entities, changeItem.getKidId(), partitionId, changeItem.getNewStorageMode());
 
         }
 
diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/PartitionGeneratorService.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/PartitionGeneratorService.java
@@ -65,21 +65,21 @@ public class PartitionGeneratorService {
      */
 
     public void generatePartition(String etag, List<ChunkMetaViewDto> entities,
-                                   String kid, String id) {
+                                   String kid, String id, String storageMode) {
 
         if (sliceCalculationBloomFilter.isPresent()) {
-            generatePartition(etag, entities, kid, id, sliceCalculationBloomFilter.get());
+            generatePartition(etag, entities, kid, id, sliceCalculationBloomFilter.get(), storageMode);
         }
 
         if (sliceCalculationHashList.isPresent()) {
-            generatePartition(etag, entities, kid, id, sliceCalculationHashList.get());
+            generatePartition(etag, entities, kid, id, sliceCalculationHashList.get(), storageMode);
         }
 
     }
 
 
     private void generatePartition(String etag, List<ChunkMetaViewDto> entities,
-                                   String kid, String id, SliceCalculation sliceCalculation) {
+                                   String kid, String id, SliceCalculation sliceCalculation, String storageMode) {
 
         String x = null;
         String y = null;
@@ -99,7 +99,7 @@ private void generatePartition(String etag, List<ChunkMetaViewDto> entities,
                 log.error("Kid and/or id does not match: kid: {} , {} id {}, {}", kid, mve.getKid(), id, mve.getId());
             } else {
 
-                SliceDataDto sliceDataDto = sliceCalculation.calculateSlice(mve.getHashes());
+                SliceDataDto sliceDataDto = sliceCalculation.calculateSlice(mve.getHashes(), storageMode);
                 if (sliceDataDto != null) {
                     Map<String, PartitionChunksJsonItemDto> chunkItemsMap;
 
diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/SliceCalculation.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/SliceCalculation.java
@@ -27,7 +27,7 @@ public interface SliceCalculation {
 
     SliceType getSliceType();
 
-    SliceDataDto calculateSlice(String[] hashes);
+    SliceDataDto calculateSlice(String[] hashes, String storageMode);
 
 
 }
diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/SliceCalculationBloomFilterImpl.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/SliceCalculationBloomFilterImpl.java
@@ -52,7 +52,7 @@ public SliceType getSliceType() {
     }
 
     @Override
-    public SliceDataDto calculateSlice(String[] hashes) {
+    public SliceDataDto calculateSlice(String[] hashes, String storageMode) {
         if (hashes.length <= 0) {
             return null;
         }
diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/SliceCalculationVarHashListImpl.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/SliceCalculationVarHashListImpl.java
@@ -20,14 +20,14 @@
 
 package eu.europa.ec.dgc.revocationdistribution.service;
 
-
+import eu.europa.ec.dgc.partialvariablehashfilter.PartialVariableHashFilter;
+import eu.europa.ec.dgc.partialvariablehashfilter.PartitionOffset;
 import eu.europa.ec.dgc.revocationdistribution.config.DgcConfigProperties;
 import eu.europa.ec.dgc.revocationdistribution.dto.SliceDataDto;
 import eu.europa.ec.dgc.revocationdistribution.model.SliceType;
 import eu.europa.ec.dgc.revocationdistribution.utils.HelperFunctions;
-import java.io.ByteArrayOutputStream;
+import java.io.IOException;
 import java.security.NoSuchAlgorithmException;
-import java.util.Arrays;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.bouncycastle.util.encoders.DecoderException;
@@ -50,35 +50,38 @@ public SliceType getSliceType() {
     }
 
     @Override
-    public SliceDataDto calculateSlice(String[] hashes) {
+    public SliceDataDto calculateSlice(String[] hashes, String storageMode) {
         if (hashes.length <= 0) {
             return null;
         }
 
-        Arrays.sort(hashes);
-
-        int numBytesToStore = properties.getVarHashList().getMinByteCount();
+        byte minByteCount = properties.getVarHashList().getMinByteCount();
 
         SliceDataDto sliceDataDto = new SliceDataDto();
 
         sliceDataDto.getMetaData().setType(SliceType.VARHASHLIST.name());
         sliceDataDto.getMetaData().setVersion(properties.getVarHashList().getVersion());
 
-        ByteArrayOutputStream baos = new ByteArrayOutputStream();
-        baos.write(numBytesToStore);
+        PartialVariableHashFilter filter =
+            new PartialVariableHashFilter(minByteCount, getPartitionOffset(storageMode), hashes.length,
+                properties.getVarHashList().getProbRate());
+
+
         for (String hash : hashes) {
             try {
                 byte[] hashBytes = helperFunctions.getBytesFromHexString(hash);
-                baos.write(hashBytes, 0, numBytesToStore);
+                filter.add(hashBytes);
             } catch (DecoderException e) {
                 log.error("Could not add hash to hash list: {} , {}", hash, e.getMessage());
             }
         }
 
-        sliceDataDto.setBinaryData(baos.toByteArray());
-
         try {
+            sliceDataDto.setBinaryData(filter.writeTo());
             sliceDataDto.getMetaData().setHash(helperFunctions.calculateHash(sliceDataDto.getBinaryData()));
+        } catch (IOException e) {
+            log.error("Could not set binary data.");
+            return null;
         } catch (NoSuchAlgorithmException e) {
             log.error("Could not calculate hash for binary data.");
             return null;
@@ -87,5 +90,19 @@ public SliceDataDto calculateSlice(String[] hashes) {
         return sliceDataDto;
     }
 
+    private PartitionOffset getPartitionOffset(String storageMode) {
+        switch (storageMode) {
+            case "POINT": {
+                return PartitionOffset.POINT;
+            }
+            case "VECTOR": {
+                return PartitionOffset.VECTOR;
+            }
+            case "COORDINATE":
+            default: {
+                return PartitionOffset.COORDINATE;
+            }
+        }
+    }
 
 }
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
@@ -25,6 +25,7 @@ spring:
     scheduling:
       pool:
         size: 5
+
 management:
   server:
     ssl:
@@ -56,6 +57,7 @@ springdoc:
     enabled: true
   swagger-ui:
     path: /swagger
+    tagsSorter=alpha:
 
 dgc:
   revocationListDownload:

Original file line number	Diff line number	Diff line change
`@@ -60,10 +60,7 @@ public static class VarHashListConfig {`
`60`	`60`	`private String type = "hash_list";`
`61`	`61`	`private String version;`
`62`	`62`	`private float probRate;`
`63`		`- private int minByteCount = 4;`
	`63`	`+ private byte minByteCount = 4;`
`64`	`64`
`65`		`- public int getMinByteCount() {`
`66`		`- return minByteCount > 255 ? 255 : minByteCount;`
`67`		`- }`
`68`	`65`	`}`
`69`	`66`	`}`
Original file line number	Diff line number	Diff line change
`@@ -227,7 +227,8 @@ private void generatePartitionsForKidInPointMode(ChangeListItem changeItem) {`
`227`	`227`	`List<ChunkMetaViewDto> entities = pointViewRepository.findAllByKid(changeItem.getKidId()).stream()`
`228`	`228`	`.map(pointViewMapper::map).collect(Collectors.toList());`
`229`	`229`
`230`		`- partitionGeneratorService.generatePartition(etag, entities, changeItem.getKidId(), null);`
	`230`	`+ partitionGeneratorService.generatePartition(`
	`231`	`+ etag, entities, changeItem.getKidId(), null, changeItem.getNewStorageMode());`
`231`	`232`
`232`	`233`
`233`	`234`	`}`
`@@ -244,7 +245,8 @@ private void generatePartitionsForKidInVectorMode(ChangeListItem changeItem) {`
`244`	`245`	`vectorViewRepository.findAllByKidAndId(changeItem.getKidId(), partitionId).stream()`
`245`	`246`	`.map(vectorViewMapper::map).collect(Collectors.toList());`
`246`	`247`
`247`		`- partitionGeneratorService.generatePartition(etag, entities, changeItem.getKidId(), partitionId);`
	`248`	`+ partitionGeneratorService.generatePartition(`
	`249`	`+ etag, entities, changeItem.getKidId(), partitionId, changeItem.getNewStorageMode());`
`248`	`250`
`249`	`251`	`}`
`250`	`252`
`@@ -262,7 +264,8 @@ private void generatePartitionsForKidInCoordinateMode(ChangeListItem changeItem)`
`262`	`264`	`coordinateViewRepository.findAllByKidAndId(changeItem.getKidId(), partitionId).stream()`
`263`	`265`	`.map(coordinateViewMapper::map).collect(Collectors.toList());`
`264`	`266`
`265`		`- partitionGeneratorService.generatePartition(etag, entities, changeItem.getKidId(), partitionId);`
	`267`	`+ partitionGeneratorService.generatePartition(`
	`268`	`+ etag, entities, changeItem.getKidId(), partitionId, changeItem.getNewStorageMode());`
`266`	`269`
`267`	`270`	`}`
`268`	`271`
Original file line number	Diff line number	Diff line change
`@@ -27,7 +27,7 @@ public interface SliceCalculation {`
`27`	`27`
`28`	`28`	`SliceType getSliceType();`
`29`	`29`
`30`		`- SliceDataDto calculateSlice(String[] hashes);`
	`30`	`+ SliceDataDto calculateSlice(String[] hashes, String storageMode);`
`31`	`31`
`32`	`32`
`33`	`33`	`}`
Original file line number	Diff line number	Diff line change
`@@ -52,7 +52,7 @@ public SliceType getSliceType() {`
`52`	`52`	`}`
`53`	`53`
`54`	`54`	`@Override`
`55`		`- public SliceDataDto calculateSlice(String[] hashes) {`
	`55`	`+ public SliceDataDto calculateSlice(String[] hashes, String storageMode) {`
`56`	`56`	`if (hashes.length <= 0) {`
`57`	`57`	`return null;`
`58`	`58`	`}`