Skip to content

Commit

Permalink
fix array flattening
Browse files Browse the repository at this point in the history
  • Loading branch information
@minenwerfer
minenwerfer committed Aug 21, 2023
1 parent eafa33b commit 96d2f92
Show file tree
Hide file tree
Showing 4 changed files with 105 additions and 17 deletions.
29 changes: 15 additions & 14 deletions src/normalize_json/normalize.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,21 +69,27 @@ def unserialize(raw: typing.Any, mime: AcceptedMime = 'application/json'):

raise TypeError('invalid mime')

def flatten(target: RawObject, separator: str = '.', preserve_arrays: bool = False):
def flatten(target: typing.Any, separator: str = '.', preserve_arrays: bool = False):
if not isinstance(target, dict) and not isinstance(target, list):
return target

ret: RawObject = {}

def internal_flatten(obj: typing.Any, parent: str = '') -> typing.Any:
def internal_flatten(obj: typing.Any, parent: str = '', depth: int = 0) -> typing.Any:
if isinstance(obj, dict):
for k, v in typing.cast(RawObject, obj).items():
internal_flatten(v, '%s%s%s' % (parent, separator, k))
internal_flatten(v, '%s%s%s' % (parent, separator, k), depth + 1)

elif isinstance(obj, list):
if preserve_arrays:
ret[parent] = obj
if preserve_arrays and depth > 0:
ret[parent] = [
flatten(elem, separator, preserve_arrays)
for elem in typing.cast(list[typing.Any], obj)
]
return

for i, v in enumerate(typing.cast(list[RawObject], obj)):
internal_flatten(v, '%s[%d]' % (parent, i))
internal_flatten(v, '%s[%d]' % (parent, i), depth + 1)

else:
ret[parent] = obj
Expand All @@ -108,7 +114,7 @@ def check_types(node: Node, value: typing.Any, modifiers: list[Modifier]):

return actual, node['type']

def handle_modifiers(node: Node, modifiers: list[Modifier], old_value: typing.Any):
def handle_modifiers(node: Node, mapped_name: str, modifiers: list[Modifier], old_value: typing.Any):
value = old_value

if not value:
Expand All @@ -117,7 +123,7 @@ def handle_modifiers(node: Node, modifiers: list[Modifier], old_value: typing.An
elif 'default_null' in modifiers:
return None
else:
raise ValueError('value for %s wasnt provided' % node.get('map'))
raise ValueError('value for %s wasnt provided' % mapped_name)

if trim := node.get('trim_start'):
value = value[trim*-1:]
Expand Down Expand Up @@ -176,9 +182,6 @@ def translate(target: T | tuple[T, int], mapping: Mapping, acc: RawObject = {},
if 'reverse' in modifiers:
mapped_name, original_name = original_name, mapped_name

if not mapped_name:
continue

for n in mapped_name if isinstance(mapped_name, list) else mapped_name.split('|'):
mapped_name = n.strip()

Expand All @@ -205,7 +208,6 @@ def translate(target: T | tuple[T, int], mapping: Mapping, acc: RawObject = {},
else:
child: typing.Any = initial_value or target[original_name]
value = translate(child, node, acc, modifiers, (flat_obj, flat_obj_arr))

if node.get('array') and not isinstance(value, list):
value = [value]

Expand All @@ -215,7 +217,7 @@ def translate(target: T | tuple[T, int], mapping: Mapping, acc: RawObject = {},
if not initial_value:
initial_value = get_initial_value(target, mapped_name, flat_obj)

value = handle_modifiers(node, modifiers, initial_value)
value = handle_modifiers(node, mapped_name, modifiers, initial_value)
if node.get('array') and not isinstance(value, list):
value = [value]

Expand All @@ -235,7 +237,6 @@ def translate(target: T | tuple[T, int], mapping: Mapping, acc: RawObject = {},

return typing.cast(T, result)


return typing.cast(T, ret)


Expand Down
35 changes: 35 additions & 0 deletions tests/flatten.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
# pyright: basic

from unittest import TestCase
from src.normalize_json.normalize import flatten

class TestFlatten(TestCase):
def test_flatten_array(self):
result = flatten([
{
'person': {
'name': 'joao',
'details': {
'dogs': [
'thor',
'bobby'
]
}
}
},
{
'person': {
'name': 'also joao',
'details': {
'dogs': [
'spike'
]
}
}
}
], preserve_arrays=True)

self.assertEqual(result['[0].person.name'], 'joao')
self.assertEqual(result['[0].person.details.dogs'], ['thor', 'bobby'])
self.assertEqual(result['[1].person.name'], 'also joao')
self.assertEqual(result['[1].person.details.dogs'], ['spike'])
55 changes: 55 additions & 0 deletions tests/translate.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,51 @@
}
}

mapping2: Mapping = {
'__fields': {
'items': {
'map': 'items',
'type': 'object',
'__fields': {
'meta': {
'map': 'meta',
'type': 'object',
'__fields': {
'page': {
'type': 'integer'
}
}
},
'data': {
'map': 'data',
'array': True,
'type': 'object',
'__fields': {
'name': {
'type': 'string'
},
'age': {
'type': 'integer'
}
}
}
}
}
}
}

sample2 = {
'items': {
'meta': {
'page': 1,
},
'data': [
{ 'name': 'João', 'age': 23 },
{ 'name': 'Thor', 'age': 15 }
]
}
}

class TestTranslate(TestCase):
def test_translate_output(self):
unserialized = unserialize(sample1)
Expand All @@ -137,3 +182,13 @@ def test_translate_output(self):
self.assertEqual(result['trim_me_end'], 'abc')
self.assertEqual(result['pick_until'], 'Terry')


def test_translate_inloco(self):
unserialized = unserialize(sample2)
result = translate(unserialized, mapping2)

self.assertEqual(result['items']['meta']['page'], 1)
self.assertEqual(result['items']['data'][0]['name'], 'João')
self.assertEqual(result['items']['data'][0]['age'], 23)
self.assertEqual(result['items']['data'][1]['name'], 'Thor')
self.assertEqual(result['items']['data'][1]['age'], 15)
3 changes: 0 additions & 3 deletions tests/translate_array_entry.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,3 @@ class TestTranslateArrayEntry(TestCase):
def test_translate_output(self):
unserialized = unserialize(sample1)
result = translate(unserialized, mapping1)

import json
print(json.dumps(result, indent=2))

0 comments on commit 96d2f92

Please sign in to comment.