[fix] Data synchronization from another ES node : UTF8 error - fix #10

duniter · May 3, 2017 · 1eef325 · 1eef325
1 parent f3fb4a3
commit 1eef325
Show file tree

Hide file tree

Showing 10 changed files with 199 additions and 71 deletions.
diff --git a/...r4j-core-client/src/main/java/org/duniter/core/client/model/bma/jackson/JacksonUtils.java b/...r4j-core-client/src/main/java/org/duniter/core/client/model/bma/jackson/JacksonUtils.java
@@ -22,6 +22,7 @@
  * #L%
  */
 
+import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.core.util.MinimalPrettyPrinter;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationConfig;
@@ -58,6 +59,9 @@ public static ObjectMapper newObjectMapper() {
 
         objectMapper.registerModule(module);
 
+        // Adding features
+        //objectMapper.getFactory().configure(JsonGenerator.Feature., true);
+
         return objectMapper;
     }
 

diff --git a/...nt/model/elasticsearch/MessageRecord.java → ...e/client/model/elasticsearch/Message.java b/...nt/model/elasticsearch/MessageRecord.java → ...e/client/model/elasticsearch/Message.java
@@ -22,17 +22,31 @@
  * #L%
  */
 
+import com.fasterxml.jackson.annotation.JsonGetter;
+import com.fasterxml.jackson.annotation.JsonSetter;
+
 /**
  * Created by blavenie on 01/03/16.
  */
-public class MessageRecord extends Record {
+public class Message extends Record {
 
+    public static final String PROPERTY_TITLE="title";
     public static final String PROPERTY_CONTENT="content";
     public static final String PROPERTY_RECIPIENT="recipient";
+    public static final String PROPERTY_READ_SIGNATURE="read_signature";
 
-
+    private String title;
     private String content;
     private String recipient;
+    private String readSignature;
+
+    public String getTitle() {
+        return title;
+    }
+
+    public void setTitle(String title) {
+        this.title = title;
+    }
 
     public String getContent() {
         return content;
@@ -49,4 +63,15 @@ public String getRecipient() {
     public void setRecipient(String recipient) {
         this.recipient = recipient;
     }
+
+    @JsonGetter(PROPERTY_READ_SIGNATURE)
+    public String getReadSignature() {
+        return readSignature;
+    }
+
+    @JsonSetter(PROPERTY_READ_SIGNATURE)
+    public void setReadSignature(String readSignature) {
+        this.readSignature = readSignature;
+    }
+
 }
diff --git a/duniter4j-core-client/src/main/java/org/duniter/core/client/model/elasticsearch/Records.java b/duniter4j-core-client/src/main/java/org/duniter/core/client/model/elasticsearch/Records.java
@@ -0,0 +1,38 @@
+package org.duniter.core.client.model.elasticsearch;
+
+/*
+ * #%L
+ * Duniter4j :: Core Client API
+ * %%
+ * Copyright (C) 2014 - 2016 EIS
+ * %%
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation, either version 3 of the 
+ * License, or (at your option) any later version.
+ * 
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public 
+ * License along with this program.  If not, see
+ * <http://www.gnu.org/licenses/gpl-3.0.html>.
+ * #L%
+ */
+
+/**
+ * Helper class
+ * Created by blavenie on 01/03/16.
+ */
+public final class Records {
+
+    public static final String PROPERTY_ISSUER="issuer";
+    public static final String PROPERTY_HASH="hash";
+    public static final String PROPERTY_SIGNATURE="signature";
+    public static final String PROPERTY_TIME="time";
+    public static final String PROPERTY_READ_SIGNATURE="read_signature";
+
+
+}
diff --git a/duniter4j-core-client/src/main/java/org/duniter/core/client/service/HttpServiceImpl.java b/duniter4j-core-client/src/main/java/org/duniter/core/client/service/HttpServiceImpl.java
@@ -24,6 +24,7 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Charsets;
 import com.google.common.base.Joiner;
 import org.apache.http.HttpResponse;
 import org.apache.http.HttpStatus;
@@ -517,7 +518,7 @@ public <T> T readValue(byte[] json, Class<T> clazz) throws IOException {
     }
 
     public <T> T readValue(InputStream json, Class<T> clazz) throws IOException {
-        return objectMapper.readValue(json, clazz);
+        return objectMapper.readValue(new InputStreamReader(json, Charsets.UTF_8.name()), clazz);
     }
 
 }
diff --git a/duniter4j-es-core/src/main/java/org/duniter/elasticsearch/model/SynchroResult.java b/duniter4j-es-core/src/main/java/org/duniter/elasticsearch/model/SynchroResult.java
@@ -33,9 +33,11 @@ public class SynchroResult {
     private long insertTotal = 0;
     private long updateTotal = 0;
     private long deleteTotal = 0;
+    private long invalidSignatureTotal = 0;
     private Map<String, Long> insertHits = new HashMap<>();
     private Map<String, Long> updateHits = new HashMap<>();
     private Map<String, Long> deleteHits = new HashMap<>();
+    private Map<String, Long> invalidSignatureHits = new HashMap<>();
 
     public void addInserts(String index, String type, long nbHits) {
         insertHits.put(index + "/" + type, getInserts(index, type) + nbHits);
@@ -52,6 +54,11 @@ public void addDeletes(String index, String type, long nbHits) {
         deleteTotal += nbHits;
     }
 
+    public void addInvalidSignatures(String index, String type, long nbHits) {
+        invalidSignatureHits.put(index + "/" + type, getDeletes(index, type) + nbHits);
+        invalidSignatureTotal += nbHits;
+    }
+
     public long getInserts(String index, String type) {
         return insertHits.getOrDefault(index + "/" + type, 0l);
     }
@@ -60,6 +67,10 @@ public long getUpdates(String index, String type) {
         return updateHits.getOrDefault(index + "/" + type, 0l);
     }
 
+    public long getInvalidSignatures(String index, String type) {
+        return invalidSignatureHits.getOrDefault(index + "/" + type, 0l);
+    }
+
     public long getDeletes(String index, String type) {
         return deleteHits.getOrDefault(index + "/" + type, 0l);
     }
@@ -76,6 +87,10 @@ public long getDeletes() {
         return deleteTotal;
     }
 
+    public long getInvalidSignatures() {
+        return invalidSignatureTotal;
+    }
+
     public long getTotal() {
         return insertTotal + updateTotal + deleteTotal;
     }

diff --git a/duniter4j-es-core/src/main/java/org/duniter/elasticsearch/service/AbstractService.java b/duniter4j-es-core/src/main/java/org/duniter/elasticsearch/service/AbstractService.java
@@ -23,20 +23,21 @@
  */
 
 
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableSet;
 import org.duniter.core.beans.Bean;
 import org.duniter.core.client.model.bma.jackson.JacksonUtils;
 import org.duniter.core.client.model.elasticsearch.Record;
+import org.duniter.core.client.model.elasticsearch.Records;
 import org.duniter.core.exception.TechnicalException;
 import org.duniter.core.service.CryptoService;
 import org.duniter.elasticsearch.PluginSettings;
 import org.duniter.elasticsearch.client.Duniter4jClient;
 import org.duniter.elasticsearch.exception.InvalidFormatException;
 import org.duniter.elasticsearch.exception.InvalidSignatureException;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.client.Client;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.nuiton.i18n.I18n;
@@ -111,41 +112,30 @@ protected <T> T executeWithRetry(RetryFunction<T> retryFunction) throws Technica
     }
 
     protected JsonNode readAndVerifyIssuerSignature(String recordJson) throws ElasticsearchException {
+       return readAndVerifyIssuerSignature(recordJson, Records.PROPERTY_ISSUER);
+    }
+
+    protected JsonNode readAndVerifyIssuerSignature(String recordJson, String issuerFieldName) throws ElasticsearchException {
 
         try {
-            JsonNode actualObj = objectMapper.readTree(recordJson);
-            readAndVerifyIssuerSignature(recordJson, actualObj);
-            return actualObj;
+            JsonNode recordObj = objectMapper.readTree(recordJson);
+            readAndVerifyIssuerSignature(recordJson, recordObj, issuerFieldName);
+            return recordObj;
         }
         catch(IOException e) {
             throw new InvalidFormatException("Invalid record JSON: " + e.getMessage(), e);
         }
     }
 
-    protected void readAndVerifyIssuerSignature(String recordJson, JsonNode actualObj) throws ElasticsearchException {
-
-        Set<String> fieldNames = ImmutableSet.copyOf(actualObj.fieldNames());
-        if (!fieldNames.contains(Record.PROPERTY_ISSUER)
-                || !fieldNames.contains(Record.PROPERTY_SIGNATURE)) {
-            throw new InvalidFormatException(String.format("Invalid record JSON format. Required fields [%s,%s]", Record.PROPERTY_ISSUER, Record.PROPERTY_SIGNATURE));
-        }
-        String issuer = getMandatoryField(actualObj, Record.PROPERTY_ISSUER).asText();
-        String signature = getMandatoryField(actualObj, Record.PROPERTY_SIGNATURE).asText();
 
+    protected void readAndVerifyIssuerSignature(JsonNode actualObj, String issuerFieldName) throws ElasticsearchException, JsonProcessingException {
         // Remove hash and signature
-        recordJson = JacksonUtils.removeAttribute(recordJson, Record.PROPERTY_SIGNATURE);
-        recordJson = JacksonUtils.removeAttribute(recordJson, Record.PROPERTY_HASH);
-
-        if (!cryptoService.verify(recordJson, signature, issuer)) {
-            throw new InvalidSignatureException("Invalid signature of JSON string");
-        }
-
-        // TODO: check issuer is in the WOT ?
+        String recordJson = objectMapper.writeValueAsString(actualObj);
+        readAndVerifyIssuerSignature(recordJson, actualObj, issuerFieldName);
     }
 
-
     protected String getIssuer(JsonNode actualObj) {
-        return  getMandatoryField(actualObj, Record.PROPERTY_ISSUER).asText();
+        return  getMandatoryField(actualObj, Records.PROPERTY_ISSUER).asText();
     }
 
     protected JsonNode getMandatoryField(JsonNode actualObj, String fieldName) {
@@ -160,4 +150,32 @@ public interface RetryFunction<T> {
 
         T execute() throws TechnicalException;
     }
+
+    /* -- internal methods -- */
+
+    private void readAndVerifyIssuerSignature(String recordJson, JsonNode recordObj, String issuerFieldName) throws ElasticsearchException {
+
+        Set<String> fieldNames = ImmutableSet.copyOf(recordObj.fieldNames());
+        if (!fieldNames.contains(issuerFieldName)
+                || !fieldNames.contains(Records.PROPERTY_SIGNATURE)) {
+            throw new InvalidFormatException(String.format("Invalid record JSON format. Required fields [%s,%s]", Records.PROPERTY_ISSUER, Records.PROPERTY_SIGNATURE));
+        }
+        String issuer = getMandatoryField(recordObj, issuerFieldName).asText();
+        String signature = getMandatoryField(recordObj, Records.PROPERTY_SIGNATURE).asText();
+
+        // Remove hash and signature
+        recordJson = JacksonUtils.removeAttribute(recordJson, Records.PROPERTY_SIGNATURE);
+        recordJson = JacksonUtils.removeAttribute(recordJson, Records.PROPERTY_HASH);
+
+        // Remove 'read_signature' attribute if exists (added AFTER signature)
+        if (fieldNames.contains(Records.PROPERTY_READ_SIGNATURE)) {
+            recordJson = JacksonUtils.removeAttribute(recordJson, Records.PROPERTY_READ_SIGNATURE);
+        }
+
+        if (!cryptoService.verify(recordJson, signature, issuer)) {
+            throw new InvalidSignatureException("Invalid signature of JSON string");
+        }
+
+        // TODO: check issuer is in the WOT ?
+    }
 }