Skip to content

Commit

Permalink
Make avro serde accept clj maps with underscores,add option to skip m…
Browse files Browse the repository at this point in the history 
…angling names.
  • Loading branch information
@sathyavijayan
sathyavijayan committed Apr 18, 2019
1 parent 4bbb6de commit 6307689
Show file tree
Hide file tree
Showing 2 changed files with 207 additions and 5 deletions.
19 changes: 14 additions & 5 deletions src/jackdaw/serdes/avro.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,11 +86,20 @@
(when schema-str
(.parse (Schema$Parser.) ^String schema-str)))))

(def ^:dynamic *mangle-names*
"When true, record field names will be mangled during schema parse and
record de/serialization. Default value is `true`."
true)

(defn- ^String mangle [^String n]
(str/replace n #"-" "_"))
(if *mangle-names* (str/replace n #"-" "_") n))

(defn- ^String unmangle [^String n]
(str/replace n #"_" "-"))
(if *mangle-names* (str/replace n #"_" "-") n))

(defn- ->field-key
[field-name]
(keyword (unmangle field-name)))

(defn- dispatch-on-type-fields
[^Schema schema]
Expand Down Expand Up @@ -398,7 +407,7 @@
(comp (map first)
(map (fn [^Schema$Field field]
(let [field-name (.name field)
field-key (keyword (unmangle field-name))
field-key (->field-key field-name)
[_ field-coercion :as entry] (get field->schema+coercion field-key)
value (.get ^GenericData$Record avro-record field-name)]
(when-not field-coercion
Expand All @@ -425,7 +434,7 @@
new-k
(.getName schema))
{:path path, :clj-data clj-map})))
(let [[_ field-coercion] (get field->schema+coercion k)
(let [[_ field-coercion] (get field->schema+coercion (->field-key new-k))
new-v (clj->avro field-coercion v (conj path k))]
(.set record-builder new-k new-v))))

Expand All @@ -442,7 +451,7 @@
[schema->coercion ^Schema schema]
(let [fields (into {}
(map (fn [^Schema$Field field]
[(keyword (unmangle (.name field)))
[(->field-key (.name field))
[field (schema->coercion (.schema field))]]))
(.getFields schema))]
(RecordType. schema fields)))
Expand Down
193 changes: 193 additions & 0 deletions test/jackdaw/serdes/avro_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -457,6 +457,199 @@
"topic"
(uuid/to-string uuid/+null+))))))))

(deftest accept-unmangled-input
(let [schema {:name "testRecord"
:type "record"
:fields [{:name "string_field"
:type "string"}
{:name "long_field"
:type "long"}
{:name "optional_field"
:type ["null" "int"]
:default nil}
{:name "nil_field"
:type "null"}
{:name "default_field"
:type "long"
:default 1}
{:name "bytes_field"
:type "bytes"}
{:name "enum_field"
:type {:type "enum"
:name "weird_values"
:symbols ["a_1" "B3"]}}
{:name "map_field"
:type ["null" {:type "map"
:values bananas-schema}]}
{:name "array_field"
:type ["null" {:name "subrecords"
:type "array"
:items "banana"}]}
{:name "uuid_field"
:type {:type "string",
:logicalType "uuid"}}]}
schema-str (json/write-str schema)
serde (->serde schema-str)]

(is (= (round-trip serde "bananas"
{:string_field "hello"
:long_field 3
:optional_field 3
:nil_field nil
:default_field 1
:bytes_field (ByteBuffer/wrap (.getBytes "hello"))
:enum_field "a_1"
:map_field {"banana" {:color "yellow"}
"ripe b4nana$" {:color "yellow-green"}}

:array_field [{:color "yellow"}]
:uuid_field uuid/+null+})
{:string-field "hello"
:long-field 3
:default-field 1
:nil-field nil
:bytes-field (ByteBuffer/wrap (.getBytes "hello"))
:map-field {"banana" {:color "yellow"}
"ripe b4nana$" {:color "yellow-green"}}
:enum-field :a-1
:optional-field 3
:array-field [{:color "yellow"}]
:uuid-field uuid/+null+}))

(is (= (round-trip serde "bananas"
{:string_field "hello"
:long_field 3
:optional_field 3
:nil_field nil
:default_field 1
:bytes_field (ByteBuffer/wrap (.getBytes "hello"))
:enum_field :a_1
:map_field {"banana" {:color "yellow"}
"ripe b4nana$" {:color "yellow-green"}}

:array_field [{:color "yellow"}]
:uuid_field uuid/+null+})
{:string-field "hello"
:long-field 3
:default-field 1
:nil-field nil
:bytes-field (ByteBuffer/wrap (.getBytes "hello"))
:map-field {"banana" {:color "yellow"}
"ripe b4nana$" {:color "yellow-green"}}
:enum-field :a-1
:optional-field 3
:array-field [{:color "yellow"}]
:uuid-field uuid/+null+}))


(is (= (round-trip serde "bananas"
{"string_field" "hello"
"long_field" 3
"optional_field" 3
"nil_field" nil
"default_field" 1
"bytes_field" (ByteBuffer/wrap (.getBytes "hello"))
"enum_field" "a_1"
"map_field" {"banana" {"color" "yellow"}
"ripe b4nana$" {"color" "yellow-green"}}
"array_field" [{"color" "yellow"}]
"uuid_field" uuid/+null+})
{:string-field "hello"
:long-field 3
:default-field 1
:nil-field nil
:bytes-field (ByteBuffer/wrap (.getBytes "hello"))
:map-field {"banana" {:color "yellow"}
"ripe b4nana$" {:color "yellow-green"}}
:enum-field :a-1
:optional-field 3
:array-field [{:color "yellow"}]
:uuid-field uuid/+null+}))))

(deftest no-mangling-test
(binding [jackdaw.serdes.avro/*mangle-names* false]
(let [schema {:name "testRecord"
:type "record"
:fields [{:name "string_field"
:type "string"}
{:name "long_field"
:type "long"}
{:name "optional_field"
:type ["null" "int"]
:default nil}
{:name "nil_field"
:type "null"}
{:name "default_field"
:type "long"
:default 1}
{:name "bytes_field"
:type "bytes"}
{:name "enum_field"
:type {:type "enum"
:name "weird_values"
:symbols ["a_1" "B3"]}}
{:name "map_field"
:type ["null" {:type "map"
:values bananas-schema}]}
{:name "array_field"
:type ["null" {:name "subrecords"
:type "array"
:items "banana"}]}
{:name "uuid_field"
:type {:type "string",
:logicalType "uuid"}}]}
schema-str (json/write-str schema)
serde (->serde schema-str)]

(is (= (round-trip serde "bananas"
{:string_field "hello"
:long_field 3
:optional_field 3
:nil_field nil
:default_field 1
:bytes_field (ByteBuffer/wrap (.getBytes "hello"))
:enum_field :a_1
:map_field {"banana" {:color "yellow"}
"ripe b4nana$" {:color "yellow-green"}}

:array_field [{:color "yellow"}]
:uuid_field uuid/+null+})
{:string_field "hello"
:long_field 3
:default_field 1
:nil_field nil
:bytes_field (ByteBuffer/wrap (.getBytes "hello"))
:map_field {"banana" {:color "yellow"}
"ripe b4nana$" {:color "yellow-green"}}
:enum_field :a_1
:optional_field 3
:array_field [{:color "yellow"}]
:uuid_field uuid/+null+}))

(is (= (round-trip serde "bananas"
{"string_field" "hello"
"long_field" 3
"optional_field" 3
"nil_field" nil
"default_field" 1
"bytes_field" (ByteBuffer/wrap (.getBytes "hello"))
"enum_field" "a_1"
"map_field" {"banana" {"color" "yellow"}
"ripe b4nana$" {"color" "yellow-green"}}
"array_field" [{"color" "yellow"}]
"uuid_field" uuid/+null+})
{:string_field "hello"
:long_field 3
:default_field 1
:nil_field nil
:bytes_field (ByteBuffer/wrap (.getBytes "hello"))
:map_field {"banana" {:color "yellow"}
"ripe b4nana$" {:color "yellow-green"}}
:enum_field :a_1
:optional_field 3
:array_field [{:color "yellow"}]
:uuid_field uuid/+null+})))))

(deftest schemaless-test
(let [serde (->serde nil)]
(is (= (round-trip serde "bananas" "hello")
Expand Down

0 comments on commit 6307689

Please sign in to comment.