diff --git a/TODO.md b/TODO.md index 8331158..3abc42f 100644 --- a/TODO.md +++ b/TODO.md @@ -9,13 +9,11 @@ Schema.resolve_field(entity_type, field_spec) -> list of fields for entity_type Schema.resolve_one_field(...) -> one field or ValueError - - Create a standard-ish set of tags and aliases: $parent pointing to typical parent - Include inverse_association (from private schema) - - *type -> explicit prefix matching - basic key/value metadata store diff --git a/docs/python_api.rst b/docs/python_api.rst index d5d6ef2..bb4bee2 100644 --- a/docs/python_api.rst +++ b/docs/python_api.rst @@ -6,3 +6,13 @@ Python API .. autoclass:: sgschema.schema.Schema :members: +.. automodule:: sgschema.entity + +..autoclass:: sgschema.entity.Entity + :members: + +.. automodule:: sgschema.field + +..autoclass:: sgschema.field.Field + :members: + diff --git a/sgschema/entity.py b/sgschema/entity.py index 4690e57..687d0d4 100644 --- a/sgschema/entity.py +++ b/sgschema/entity.py @@ -10,44 +10,8 @@ def __init__(self, schema, name): self.name = name self.fields = {} - - self._aliases = set() - self._tags = set() - - self._field_aliases = {} - self._field_tags = {} - - @cached_property - def field_aliases(self): - field_aliases = dict(self._field_aliases) - for field in self.fields.itervalues(): - for alias in field._aliases: - field_aliases[alias] = field.name - return field_aliases - - @cached_property - def field_tags(self): - field_tags = {k: set(v) for k, v in self._field_tags.iteritems()} - for field in self.fields.itervalues(): - for tag in field._tags: - field_tags.setdefault(tag, set()).add(field.name) - return field_tags - - @cached_property - def aliases(self): - aliases = set(self._aliases) - for k, v in self.schema._entity_aliases.iteritems(): - if v == self.name: - aliases.add(k) - return aliases - - @cached_property - def tags(self): - tags = set(self._tags) - for k, v in self.schema._entity_tags.iteritems(): - if self.name in v: - tags.add(k) - return tags + self.field_aliases = {} + self.field_tags = {} def _get_or_make_field(self, name): try: @@ -58,22 +22,17 @@ def _get_or_make_field(self, name): def _reduce_raw(self, schema, raw_entity): pass - def _load(self, raw): - for name, value in raw.pop('fields', {}).iteritems(): - self._get_or_make_field(name)._load(value) - - self._field_aliases.update(raw.pop('field_aliases', {})) - self._field_tags.update(raw.pop('field_tags', {})) - - self._aliases.update(raw.pop('aliases', ())) - self._tags.update(raw.pop('tags', ())) + def __getstate__(self): + return dict((k, v) for k, v in ( + ('fields', dict((n, f.__getstate__()) for n, f in self.fields.iteritems())), + ('field_aliases', self.field_aliases), + ('field_tags', self.field_tags), + ) if v) + def __setstate__(self, raw): + for name, value in raw.pop('fields', {}).iteritems(): + self._get_or_make_field(name).__setstate__(value) + self.field_aliases.update(raw.pop('field_aliases', {})) + self.field_tags.update(raw.pop('field_tags', {})) if raw: raise ValueError('unknown entity keys: %s' % ', '.join(sorted(raw))) - - def _dump(self): - return {k: v for k, v in ( - ('fields', {field.name: field._dump() for field in self.fields.itervalues()}), - ('tags', sorted(self.tags)), - ('aliases', sorted(self.aliases)), - ) if v} diff --git a/sgschema/field.py b/sgschema/field.py index 7176827..d11652e 100644 --- a/sgschema/field.py +++ b/sgschema/field.py @@ -10,25 +10,6 @@ def __init__(self, entity, name): self.allowed_entity_types = set() self.data_type = None - self._aliases = set() - self._tags = set() - - @cached_property - def aliases(self): - aliases = set(self._aliases) - for k, v in self.entity._field_aliases.iteritems(): - if v == self.name: - aliases.add(k) - return aliases - - @cached_property - def tags(self): - tags = set(self._tags) - for k, v in self.entity._field_tags.iteritems(): - if self.name in v: - tags.add(k) - return tags - def _reduce_raw(self, schema, raw_field): self.data_type = raw_field['data_type']['value'] @@ -38,25 +19,21 @@ def _reduce_raw(self, schema, raw_field): if raw_private.get('identifier_column'): # It would be nice to add a "name" alias, but that might be # a little too magical. - self._aliases.add('shotgun:name') + self.entity.field_aliases['shotgun:name'] = self.name if self.data_type in ('entity', 'multi_entity'): types_ = raw_private['allowed_entity_types'] or [] self.allowed_entity_types = set(types_[:]) - def _load(self, raw): + def __setstate__(self, raw): self.allowed_entity_types.update(raw.pop('allowed_entity_types', ())) self.data_type = raw.pop('data_type', self.data_type) - self._aliases.update(raw.pop('aliases', ())) - self._tags.update(raw.pop('tags', ())) if raw: - raise ValueError('unknown field tags: %s' % ', '.join(sorted(raw))) + raise ValueError('unknown field keys: %s' % ', '.join(sorted(raw))) - def _dump(self): - return {k: v for k, v in ( - ('aliases', sorted(self.aliases)), + def __getstate__(self): + return dict((k, v) for k, v in ( ('allowed_entity_types', sorted(self.allowed_entity_types)), ('data_type', self.data_type), - ('tags', sorted(self.tags)), - ) if v} + ) if v) diff --git a/sgschema/schema.py b/sgschema/schema.py index 2840c04..56676af 100644 --- a/sgschema/schema.py +++ b/sgschema/schema.py @@ -61,27 +61,8 @@ def __init__(self): self._raw_private = None self.entities = {} - - self._entity_aliases = {} - self._entity_tags = {} - - @cached_property - def entity_aliases(self): - """Mapping of entity aliases to entity names.""" - entity_aliases = dict(self._entity_aliases) - for entity in self.entities.itervalues(): - for alias in entity._aliases: - entity_aliases[alias] = entity.name - return entity_aliases - - @cached_property - def entity_tags(self): - """Mapping of entity tags to lists of entity names.""" - entity_tags = {k: set(v) for k, v in self._entity_tags.iteritems()} - for entity in self.entities.itervalues(): - for tag in entity._tags: - entity_tags.setdefault(tag, set()).add(entity.name) - return entity_tags + self.entity_aliases = {} + self.entity_tags = {} def _get_or_make_entity(self, name): try: @@ -141,25 +122,29 @@ def _reduce_raw(self): field = entity._get_or_make_field(field_name) field._reduce_raw(self, raw_field) - def dump(self, path, raw=False): + def dump_raw(self, path): + with open(path, 'w') as fh: + fh.write(json.dumps({ + 'raw_fields': self._raw_fields, + 'raw_entities': self._raw_entities, + 'raw_private': self._raw_private, + }, indent=4, sort_keys=True)) + + def __getstate__(self): + return dict((k, v) for k, v in ( + ('entities', dict((n, e.__getstate__()) for n, e in self.entities.iteritems())), + ('entity_aliases', self.entity_aliases), + ('entity_tags', self.entity_tags), + ) if v) + + def dump(self, path): """Save the schema as JSON to the given path. :param str path: The path to save to. - :param bool raw: Save the raw schema, or the reduced version? """ - - if raw: - with open(path, 'w') as fh: - fh.write(json.dumps({ - 'raw_fields': self._raw_fields, - 'raw_entities': self._raw_entities, - 'raw_private': self._raw_private, - }, indent=4, sort_keys=True)) - else: - data = {'entities': {entity.name: entity._dump() for entity in self.entities.itervalues()}} - with open(path, 'w') as fh: - fh.write(json.dumps(data, indent=4, sort_keys=True)) + with open(path, 'w') as fh: + fh.write(json.dumps(self, indent=4, sort_keys=True, default=lambda x: x.__getstate__())) def load_directory(self, dir_path): """Load all ``.json`` files in the given directory.""" @@ -202,6 +187,10 @@ def load(self, input_): else: raise TypeError('require str path or dict schema') + self.__setstate__(raw_schema) + + def __setstate__(self, raw_schema): + # If it is a dictionary of entity types, pretend it is in an "entities" key. title_cased = sum(int(k[:1].isupper()) for k in raw_schema) if title_cased: @@ -210,13 +199,13 @@ def load(self, input_): raw_schema = {'entities': raw_schema} for type_name, value in raw_schema.pop('entities', {}).iteritems(): - self._get_or_make_entity(type_name)._load(value) + self._get_or_make_entity(type_name).__setstate__(value) - merge_update(self._entity_aliases, raw_schema.pop('entity_aliases', {})) - merge_update(self._entity_tags , raw_schema.pop('entity_tags', {})) + merge_update(self.entity_aliases, raw_schema.pop('entity_aliases', {})) + merge_update(self.entity_tags , raw_schema.pop('entity_tags', {})) if raw_schema: - raise ValueError('unknown keys: %s' % ', '.join(sorted(raw_schema))) + raise ValueError('unknown schema keys: %s' % ', '.join(sorted(raw_schema))) def resolve_entity(self, entity_spec, implicit_aliases=True, strict=False): diff --git a/tests/test_deep_fields.py b/tests/test_deep_fields.py index 589b222..0d6f9c3 100644 --- a/tests/test_deep_fields.py +++ b/tests/test_deep_fields.py @@ -57,7 +57,7 @@ def test_explicit_aliases(self): self.assertEqual(self.s.resolve_field('Task', 'entity.Shot.$status'), ['entity.Shot.sg_status_list']) def test_explicit_tags(self): - self.assertEqual(self.s.resolve_field('Task', '$parent.Shot.#core'), ['entity.Shot.sg_sequence', 'entity.Shot.code', 'entity.Shot.description']) + self.assertEqual(self.s.resolve_field('Task', '$parent.Shot.#core'), ['entity.Shot.code', 'entity.Shot.description', 'entity.Shot.sg_sequence']) diff --git a/tests/test_entities.py b/tests/test_entities.py index fe1bf99..11f90f4 100644 --- a/tests/test_entities.py +++ b/tests/test_entities.py @@ -7,35 +7,31 @@ def setUp(self): self.s = s = Schema() s.load({ 'entities': { - 'Entity': { - 'aliases': ['A', 'with:Namespace'], - 'tags': ['X'], - }, - 'Another': {} + 'Entity': {}, + 'Another': {}, }, 'entity_aliases': { - 'B': 'Entity', + 'Alias': 'Entity', + 'with:Namespace': 'Entity', }, 'entity_tags': { - 'Y': ['Entity'], - 'Multiple': ['Entity', 'Another'], + 'TagOne': ['Entity'], + 'TagTwo': ['Entity', 'Another'], } }) def test_explicit(self): self.assertEqual(self.s.resolve_entity('!Entity'), ['Entity']) - self.assertEqual(self.s.resolve_entity('$A'), ['Entity']) - self.assertEqual(self.s.resolve_entity('$B'), ['Entity']) - self.assertEqual(self.s.resolve_entity('#X'), ['Entity']) - self.assertEqual(self.s.resolve_entity('#Y'), ['Entity']) + self.assertEqual(self.s.resolve_entity('$Alias'), ['Entity']) + self.assertEqual(self.s.resolve_entity('#TagOne'), ['Entity']) + self.assertEqual(self.s.resolve_entity('#TagTwo'), ['Entity', 'Another']) def test_namespace(self): self.assertEqual(self.s.resolve_entity('$with:Namespace'), ['Entity']) def test_implicit(self): self.assertEqual(self.s.resolve_entity('Entity'), ['Entity']) - self.assertEqual(self.s.resolve_entity('A'), ['Entity']) - self.assertEqual(self.s.resolve_entity('B'), ['Entity']) + self.assertEqual(self.s.resolve_entity('Alias'), ['Entity']) def test_missing(self): self.assertEqual(self.s.resolve_entity('#Missing'), []) @@ -47,7 +43,7 @@ def test_missing(self): def test_one(self): self.assertEqual(self.s.resolve_one_entity('Entity'), 'Entity') self.assertEqual(self.s.resolve_one_entity('!Entity'), 'Entity') - self.assertEqual(self.s.resolve_one_entity('$A'), 'Entity') - self.assertEqual(self.s.resolve_one_entity('#X'), 'Entity') - self.assertRaises(ValueError, self.s.resolve_one_entity, '#Missing') - self.assertRaises(ValueError, self.s.resolve_one_entity, '#Multiple') \ No newline at end of file + self.assertEqual(self.s.resolve_one_entity('$Alias'), 'Entity') + self.assertEqual(self.s.resolve_one_entity('#TagOne'), 'Entity') + self.assertRaises(ValueError, self.s.resolve_one_entity, '#TagNone') + self.assertRaises(ValueError, self.s.resolve_one_entity, '#TagTwo') \ No newline at end of file diff --git a/tests/test_fields.py b/tests/test_fields.py index a0e0e15..97d63be 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -9,22 +9,19 @@ def setUp(self): 'entities': { 'Entity': { 'fields': { - 'attr': { - 'aliases': ['a', 'with:namespace'], - 'tags': ['x'], - - }, + 'attr': {}, 'sg_version': {}, 'sg_type': {}, 'name': {}, 'sg_name': {}, }, 'field_aliases': { - 'b': 'attr', + 'alias': 'attr', + 'with:namespace': 'attr', }, 'field_tags': { - 'y': ['attr'], - 'multi': ['multi_a', 'multi_b'], + 'tagone': ['attr'], + 'tagtwo': ['multi_a', 'multi_b'], } } }, @@ -32,10 +29,9 @@ def setUp(self): def test_explicit(self): self.assertEqual(self.s.resolve_field('Entity', '!attr'), ['attr']) - self.assertEqual(self.s.resolve_field('Entity', '$a'), ['attr']) - self.assertEqual(self.s.resolve_field('Entity', '$b'), ['attr']) - self.assertEqual(self.s.resolve_field('Entity', '#x'), ['attr']) - self.assertEqual(self.s.resolve_field('Entity', '#y'), ['attr']) + self.assertEqual(self.s.resolve_field('Entity', '$alias'), ['attr']) + self.assertEqual(self.s.resolve_field('Entity', '#tagone'), ['attr']) + self.assertEqual(self.s.resolve_field('Entity', '#tagtwo'), ['multi_a', 'multi_b']) def test_namespace(self): self.assertEqual(self.s.resolve_field('Entity', '$with:namespace'), ['attr']) @@ -43,8 +39,7 @@ def test_namespace(self): def test_implicit(self): self.assertEqual(self.s.resolve_field('Entity', 'attr'), ['attr']) - self.assertEqual(self.s.resolve_field('Entity', 'a'), ['attr']) - self.assertEqual(self.s.resolve_field('Entity', 'b'), ['attr']) + self.assertEqual(self.s.resolve_field('Entity', 'alias'), ['attr']) def test_prefix(self): self.assertEqual(self.s.resolve_field('Entity', 'sg_version'), ['sg_version']) @@ -71,13 +66,13 @@ def test_missing(self): def test_one(self): self.assertEqual(self.s.resolve_one_field('Entity', 'sg_type'), 'sg_type') - self.assertEqual(self.s.resolve_one_field('Entity', '$a'), 'attr') - self.assertEqual(self.s.resolve_one_field('Entity', '#x'), 'attr') - self.assertRaises(ValueError, self.s.resolve_one_field, 'Entity', '#missing') - self.assertRaises(ValueError, self.s.resolve_one_field, 'Entity', '#multi') + self.assertEqual(self.s.resolve_one_field('Entity', '$alias'), 'attr') + self.assertEqual(self.s.resolve_one_field('Entity', '#tagone'), 'attr') + self.assertRaises(ValueError, self.s.resolve_one_field, 'Entity', '#tagnone') + self.assertRaises(ValueError, self.s.resolve_one_field, 'Entity', '#tagtwo') def test_many(self): - self.assertEqual(self.s.resolve_field('Entity', ['sg_type', 'version', '#x', '#multi']), [ + self.assertEqual(self.s.resolve_field('Entity', ['sg_type', 'version', '$alias', '#tagtwo']), [ 'sg_type', 'sg_version', 'attr', 'multi_a', 'multi_b', ]) diff --git a/tests/test_load.py b/tests/test_load.py index 1d494ff..3bdf75a 100644 --- a/tests/test_load.py +++ b/tests/test_load.py @@ -1,3 +1,5 @@ +import json + from . import * @@ -6,71 +8,78 @@ class TestLoading(TestCase): def test_load_entity_tags(self): s = Schema() s.load({ - 'entities': {'Entity': { - 'tags': ['a'], - }}, - 'entity_tags': {'b': ['Entity']}, + 'entities': { + 'Entity': {}, + }, + 'entity_tags': { + 'X': ['Entity'], + }, }) - - self.assertIn('a', s.entities['Entity'].tags) - self.assertIn('b', s.entities['Entity'].tags) - self.assertIn('Entity', s.entity_tags['a']) - self.assertIn('Entity', s.entity_tags['b']) + self.assertIn('Entity', s.entity_tags['X']) def test_load_field_tags(self): - s = Schema() s.load({ 'Entity': { 'fields': { - 'sg_type': { - 'tags': ['a'], - }, + 'sg_type': {} }, 'field_tags': { - 'b': ['sg_type'], + 'x': ['sg_type'], }, }, }) - - self.assertIn('a', s.entities['Entity'].fields['sg_type'].tags) - self.assertIn('b', s.entities['Entity'].fields['sg_type'].tags) - self.assertIn('sg_type', s.entities['Entity'].field_tags['a']) - self.assertIn('sg_type', s.entities['Entity'].field_tags['b']) + self.assertIn('sg_type', s.entities['Entity'].field_tags['x']) def test_load_entity_aliases(self): - s = Schema() s.load({ - 'entities': {'Entity': { - 'aliases': ['A'], - }}, - 'entity_aliases': {'B': 'Entity'}, + 'entities': { + 'Entity': {} + }, + 'entity_aliases': {'X': 'Entity'}, }) - - self.assertIn('A', s.entities['Entity'].aliases) - self.assertIn('B', s.entities['Entity'].aliases) - self.assertEqual('Entity', s.entity_aliases['A']) - self.assertEqual('Entity', s.entity_aliases['B']) + self.assertEqual('Entity', s.entity_aliases['X']) def test_load_field_aliases(self): - s = Schema() s.load({ 'Entity': { 'fields': { - 'sg_type': { - 'aliases': ['a'], - }, + 'sg_type': {}, }, 'field_aliases': { - 'b': 'sg_type', + 'x': 'sg_type', }, }, }) + self.assertEqual('sg_type', s.entities['Entity'].field_aliases['x']) + + def test_serialize(self): + + raw = { + 'entities': { + 'Entity': { + 'fields': { + 'sg_type': {}, + }, + 'field_aliases': { + 'x': 'sg_type', + }, + }, + }, + 'entity_aliases': { + 'Alias': 'Entity', + }, + 'entity_tags': { + 'Tag': ['Entity'], + }, + } + + schema = Schema() + schema.load(raw) - self.assertIn('a', s.entities['Entity'].fields['sg_type'].aliases) - self.assertIn('b', s.entities['Entity'].fields['sg_type'].aliases) - self.assertEqual('sg_type', s.entities['Entity'].field_aliases['a']) - self.assertEqual('sg_type', s.entities['Entity'].field_aliases['b']) + raw2 = json.loads(json.dumps(raw)) + self.assertEqual(raw, raw2) + \ No newline at end of file diff --git a/tests/test_structures.py b/tests/test_structures.py index 6e4d8c5..e1d055c 100644 --- a/tests/test_structures.py +++ b/tests/test_structures.py @@ -4,27 +4,24 @@ class TestResolveStructures(TestCase): def setUp(self): - self.s = s = Schema() - s.load({ + self.s = Schema() + self.s.load({ 'entities': { 'Entity': { 'fields': { - 'attr': { - 'aliases': ['a', 'with:namespace'], - 'tags': ['x'], - - }, + 'attr': {}, 'sg_version': {}, 'sg_type': {}, 'name': {}, 'sg_name': {}, }, 'field_aliases': { - 'b': 'attr', + 'alias': 'attr', + 'with:namespace': 'attr', }, 'field_tags': { - 'y': ['attr'], - 'multi': ['multi_a', 'multi_b'] + 'tagone': ['attr'], + 'tagtwo': ['multi_a', 'multi_b'], } } }, @@ -47,8 +44,8 @@ def test_entity_list(self): }, { 'type': 'Entity', - '$b': 'attr_value', - '#multi': 'xxx', + '$alias': 'attr_value', + '#tagtwo': 'xxx', } ]), [ {