#37 Support binary format

wanglingsong · May 16, 2018 · 486e1b8 · 486e1b8
1 parent 93ad14c
commit 486e1b8
Show file tree

Hide file tree

Showing 16 changed files with 538 additions and 23 deletions.
diff --git a/jsurfer-all/pom.xml b/jsurfer-all/pom.xml
@@ -36,6 +36,31 @@
             <artifactId>jsurfer-jackson</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.dataformat</groupId>
+            <artifactId>jackson-dataformat-cbor</artifactId>
+            <version>${jackson.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.dataformat</groupId>
+            <artifactId>jackson-dataformat-smile</artifactId>
+            <version>${jackson.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.dataformat</groupId>
+            <artifactId>jackson-dataformat-ion</artifactId>
+            <version>${jackson.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.dataformat</groupId>
+            <artifactId>jackson-dataformat-protobuf</artifactId>
+            <version>${jackson.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.dataformat</groupId>
+            <artifactId>jackson-dataformat-avro</artifactId>
+            <version>${jackson.version}</version>
+        </dependency>
         <dependency>
             <groupId>com.github.jsurfer</groupId>
             <artifactId>jsurfer-jsonsimple</artifactId>

diff --git a/jsurfer-all/src/test/java/org/jsfr/json/Employee.java b/jsurfer-all/src/test/java/org/jsfr/json/Employee.java
@@ -0,0 +1,8 @@
+package org.jsfr.json;
+
+public class Employee {
+    public String name;
+    public int age;
+    public String[] emails;
+    public Employee boss;
+}
diff --git a/jsurfer-all/src/test/java/org/jsfr/json/JacksonCBORParserTest.java b/jsurfer-all/src/test/java/org/jsfr/json/JacksonCBORParserTest.java
@@ -0,0 +1,46 @@
+package org.jsfr.json;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.cbor.CBORFactory;
+import org.jsfr.json.provider.JacksonProvider;
+import org.junit.Before;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+public class JacksonCBORParserTest extends JsonSurferTest {
+
+    @Before
+    public void setUp() throws Exception {
+        provider = new JacksonProvider();
+        surfer = new JsonSurfer(new JacksonParser(new CBORFactory()), provider);
+    }
+
+    @Override
+    protected InputStream read(String resourceName) throws IOException {
+        ObjectMapper om = new ObjectMapper();
+        JsonNode node = om.readTree(this.readAsString(resourceName));
+        CBORFactory f = new CBORFactory();
+        ObjectMapper cborMapper = new ObjectMapper(f);
+        byte[] cborData = cborMapper.writeValueAsBytes(node);
+        return new ByteArrayInputStream(cborData);
+    }
+
+    @Override
+    public void testCollectAllFromString() throws Exception {
+        // skip non-byte-based source
+    }
+
+    @Override
+    public void testCollectOneFromString() throws Exception {
+        // skip non-byte-based source
+    }
+
+    @Override
+    public void testWildcardAtRoot() throws Exception {
+        // skip non-byte-based source
+    }
+
+}
diff --git a/jsurfer-all/src/test/java/org/jsfr/json/JacksonIonParserTest.java b/jsurfer-all/src/test/java/org/jsfr/json/JacksonIonParserTest.java
@@ -0,0 +1,46 @@
+package org.jsfr.json;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.ion.IonFactory;
+import org.jsfr.json.provider.JacksonProvider;
+import org.junit.Before;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+public class JacksonIonParserTest extends JsonSurferTest {
+
+    @Before
+    public void setUp() throws Exception {
+        provider = new JacksonProvider();
+        surfer = new JsonSurfer(new JacksonParser(new IonFactory()), provider);
+    }
+
+    @Override
+    protected InputStream read(String resourceName) throws IOException {
+        ObjectMapper om = new ObjectMapper();
+        JsonNode node = om.readTree(this.readAsString(resourceName));
+        IonFactory f = new IonFactory();
+        ObjectMapper cborMapper = new ObjectMapper(f);
+        byte[] ionData = cborMapper.writeValueAsBytes(node);
+        return new ByteArrayInputStream(ionData);
+    }
+
+    @Override
+    public void testCollectAllFromString() throws Exception {
+        // skip non-byte-based source
+    }
+
+    @Override
+    public void testCollectOneFromString() throws Exception {
+        // skip non-byte-based source
+    }
+
+    @Override
+    public void testWildcardAtRoot() throws Exception {
+        // skip non-byte-based source
+    }
+
+}
diff --git a/jsurfer-all/src/test/java/org/jsfr/json/JacksonParserTest.java b/jsurfer-all/src/test/java/org/jsfr/json/JacksonParserTest.java
@@ -24,10 +24,22 @@
 
 package org.jsfr.json;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.avro.AvroFactory;
+import com.fasterxml.jackson.dataformat.avro.AvroMapper;
+import com.fasterxml.jackson.dataformat.avro.AvroSchema;
+import com.fasterxml.jackson.dataformat.protobuf.ProtobufFactory;
+import com.fasterxml.jackson.dataformat.protobuf.ProtobufMapper;
+import com.fasterxml.jackson.dataformat.protobuf.schema.ProtobufSchema;
+import com.fasterxml.jackson.dataformat.protobuf.schema.ProtobufSchemaLoader;
+import org.apache.avro.Schema;
 import org.jsfr.json.provider.JacksonProvider;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
 
+import java.io.ByteArrayInputStream;
+
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.Matchers.any;
@@ -64,4 +76,70 @@ public void testNonBlockingParser() throws Exception {
         verify(mockListener).onValue(eq(provider.primitive("abcd")), any(ParsingContext.class));
     }
 
+    @Ignore
+    @Test
+    public void testProtobufParser() throws Exception {
+        JsonPathListener mockListener = mock(JsonPathListener.class);
+
+        ObjectMapper mapper = new ProtobufMapper();
+        String protobuf_str = "message Employee {\n"
+                + " required string name = 1;\n"
+                + " required int32 age = 2;\n"
+                + " repeated string emails = 3;\n"
+                + " optional Employee boss = 4;\n"
+                + "}\n";
+        final ProtobufSchema schema = ProtobufSchemaLoader.std.parse(protobuf_str);
+
+//        Employee boss = new Employee();
+//        boss.age = 30;
+//        boss.emails = new String[]{"[email protected]"};
+//        boss.name = "bar";
+
+        Employee empl = new Employee();
+        empl.age = 30;
+        empl.emails = new String[]{"[email protected]"};
+        empl.name = "foo";
+//        empl.boss = boss;
+
+        byte[] protobufData = mapper.writer(schema)
+                .writeValueAsBytes(empl);
+
+        // TODO Jackson's bug
+        JsonSurfer protobufSurfer = new JsonSurfer(new JacksonParser(new ProtobufFactory(), schema), provider);
+        SurfingConfiguration config = protobufSurfer.configBuilder().bind("$.name", mockListener).build();
+        protobufSurfer.surf(new ByteArrayInputStream(protobufData), config);
+        verify(mockListener).onValue(eq(provider.primitive("foo")), any(ParsingContext.class));
+    }
+
+    @Test
+    public void testAvroParser() throws Exception {
+        JsonPathListener mockListener = mock(JsonPathListener.class);
+
+        String SCHEMA_JSON = "{\n"
+                + "\"type\": \"record\",\n"
+                + "\"name\": \"Employee\",\n"
+                + "\"fields\": [\n"
+                + " {\"name\": \"name\", \"type\": \"string\"},\n"
+                + " {\"name\": \"age\", \"type\": \"int\"},\n"
+                + " {\"name\": \"emails\", \"type\": {\"type\": \"array\", \"items\": \"string\"}},\n"
+                + " {\"name\": \"boss\", \"type\": [\"Employee\",\"null\"]}\n"
+                + "]}";
+        Schema raw = new Schema.Parser().setValidate(true).parse(SCHEMA_JSON);
+        final AvroSchema schema = new AvroSchema(raw);
+
+        Employee empl = new Employee();
+        empl.age = 30;
+        empl.emails = new String[]{"[email protected]"};
+        empl.name = "foo";
+
+        AvroMapper mapper = new AvroMapper();
+        byte[] avroData = mapper.writer(schema)
+                .writeValueAsBytes(empl);
+
+        JsonSurfer avroSurfer = new JsonSurfer(new JacksonParser(new AvroFactory(), schema), provider);
+        SurfingConfiguration config = avroSurfer.configBuilder().bind("$.name", mockListener).build();
+        avroSurfer.surf(new ByteArrayInputStream(avroData), config);
+        verify(mockListener).onValue(eq(provider.primitive("foo")), any(ParsingContext.class));
+    }
+
 }
diff --git a/jsurfer-all/src/test/java/org/jsfr/json/JacksonSmileParserTest.java b/jsurfer-all/src/test/java/org/jsfr/json/JacksonSmileParserTest.java
@@ -0,0 +1,46 @@
+package org.jsfr.json;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import org.jsfr.json.provider.JacksonProvider;
+import org.junit.Before;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+public class JacksonSmileParserTest extends JsonSurferTest {
+
+    @Before
+    public void setUp() throws Exception {
+        provider = new JacksonProvider();
+        surfer = new JsonSurfer(new JacksonParser(new SmileFactory()), provider);
+    }
+
+    @Override
+    protected InputStream read(String resourceName) throws IOException {
+        ObjectMapper om = new ObjectMapper();
+        JsonNode node = om.readTree(this.readAsString(resourceName));
+        SmileFactory f = new SmileFactory();
+        ObjectMapper cborMapper = new ObjectMapper(f);
+        byte[] smileData = cborMapper.writeValueAsBytes(node);
+        return new ByteArrayInputStream(smileData);
+    }
+
+    @Override
+    public void testCollectAllFromString() throws Exception {
+        // skip non-byte-based source
+    }
+
+    @Override
+    public void testCollectOneFromString() throws Exception {
+        // skip non-byte-based source
+    }
+
+    @Override
+    public void testWildcardAtRoot() throws Exception {
+        // skip non-byte-based source
+    }
+
+}
diff --git a/jsurfer-all/src/test/java/org/jsfr/json/JsonSurferTest.java b/jsurfer-all/src/test/java/org/jsfr/json/JsonSurferTest.java
@@ -36,6 +36,7 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.Reader;
 import java.nio.charset.Charset;
@@ -64,6 +65,14 @@ public void onValue(Object value, ParsingContext context) {
         }
     };
 
+    protected InputStream read(String resourceName) throws IOException {
+        return Resources.getResource(resourceName).openStream();
+    }
+
+    protected String readAsString(String resourceName) throws IOException {
+        return Resources.toString(Resources.getResource(resourceName), surfer.getParserCharset());
+    }
+
     @Test
     public void testTypeCasting() throws Exception {
         surfer.configBuilder()
@@ -111,15 +120,13 @@ public void testWildcardAtRoot() throws Exception {
 
     @Test
     public void testTypeBindingOne() throws Exception {
-        Reader reader = read("sample.json");
-        Book book = surfer.collectOne(reader, Book.class, JsonPathCompiler.compile("$..book[1]"));
+        Book book = surfer.collectOne(read("sample.json"), Book.class, JsonPathCompiler.compile("$..book[1]"));
         assertEquals("Evelyn Waugh", book.getAuthor());
     }
 
     @Test
     public void testTypeBindingCollection() throws Exception {
-        Reader reader = read("sample.json");
-        Collection<Book> book = surfer.collectAll(reader, Book.class, JsonPathCompiler.compile("$..book[*]"));
+        Collection<Book> book = surfer.collectAll(read("sample.json"), Book.class, JsonPathCompiler.compile("$..book[*]"));
         assertEquals(4, book.size());
         assertEquals("Nigel Rees", book.iterator().next().getAuthor());
     }
@@ -445,14 +452,6 @@ public void testAnyIndex() throws Exception {
                 .onValue(anyObject(), any(ParsingContext.class));
     }
 
-    protected Reader read(String resourceName) throws IOException {
-        return new InputStreamReader(Resources.getResource(resourceName).openStream(), Charset.forName("UTF-8"));
-    }
-
-    protected String readAsString(String resourceName) throws IOException {
-        return Resources.toString(Resources.getResource(resourceName), Charset.forName("UTF-8"));
-    }
-
     @Test
     public void testWildcardCombination() throws Exception {
         JsonPathListener mockListener = mock(JsonPathListener.class);

diff --git a/jsurfer-core/src/main/java/org/jsfr/json/JsonParserAdapter.java b/jsurfer-core/src/main/java/org/jsfr/json/JsonParserAdapter.java
@@ -24,6 +24,7 @@
 
 package org.jsfr.json;
 
+import java.io.InputStream;
 import java.io.Reader;
 
 /**
@@ -36,7 +37,9 @@ public interface JsonParserAdapter {
      *
      * @param reader  reader
      * @param context SurfingContext
+     * @deprecated use {@link #parse(InputStream, SurfingContext)} instead
      */
+    @Deprecated
     void parse(Reader reader, SurfingContext context);
 
     /**
@@ -47,13 +50,23 @@ public interface JsonParserAdapter {
      */
     void parse(String json, SurfingContext context);
 
+    /**
+     * Create and start a resumable parser
+     *
+     * @param inputStream inputStream
+     * @param context     SurfingContext
+     */
+    void parse(InputStream inputStream, SurfingContext context);
+
     /**
      * Create a resumable parser
      *
      * @param reader  Json source
      * @param context Surfing context
      * @return Resumable Parser
+     * @deprecated use {@link #createResumableParser(InputStream, SurfingContext)} instead
      */
+    @Deprecated
     ResumableParser createResumableParser(Reader reader, SurfingContext context);
 
     /**
@@ -65,6 +78,15 @@ public interface JsonParserAdapter {
      */
     ResumableParser createResumableParser(String json, SurfingContext context);
 
+    /**
+     * Create a resumable parser
+     *
+     * @param json    Json source
+     * @param context Surfing context
+     * @return Resumable Parser
+     */
+    ResumableParser createResumableParser(InputStream json, SurfingContext context);
+
     /**
      * Create a NonBlockingParser
      *