diff --git a/.gitignore b/.gitignore index b5a6589..5cef73c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ *.pyc *~ .tox +.idea +*.egg-info +dist diff --git a/django_elasticsearch/managers.py b/django_elasticsearch/managers.py index edf2718..c0dfe14 100644 --- a/django_elasticsearch/managers.py +++ b/django_elasticsearch/managers.py @@ -14,6 +14,8 @@ from django_elasticsearch.query import EsQueryset from django_elasticsearch.client import es_client +import inspect + # Note: we use long/double because different db backends # could store different sizes of numerics ? @@ -65,7 +67,7 @@ def __init__(self, k): self.model = k self.serializer = None - self._mapping = None + self._full_mapping = None def get_index(self): return self.model.Elasticsearch.index @@ -121,11 +123,26 @@ def deserialize(self, source): @needs_instance def do_index(self): - body = self.serialize() - es_client.index(index=self.index, - doc_type=self.doc_type, - id=self.instance.id, - body=body) + kwargs = { + 'index': self.index, + 'doc_type': self.doc_type, + 'id': self.instance.id, + 'body': self.serialize() + } + + parent = self.model.Elasticsearch.parent_model + if parent: + parent.es.create_index() + parent_instance = None + for member in inspect.getmembers(self.instance): + value = member[1] + if isinstance(value, parent): + parent_instance = value + break + parent_instance.es.do_index() + kwargs.update({'parent': parent_instance.id}) + + es_client.index(**kwargs) @needs_instance def delete(self): @@ -253,6 +270,8 @@ def make_mapping(self): """ mappings = {} + sort_fields = self.model.Elasticsearch.sort_fields + for field_name in self.get_fields(): try: field = self.model._meta.get_field(field_name) @@ -274,6 +293,16 @@ def make_mapping(self): mapping.update(self.model.Elasticsearch.mappings[field_name]) except (AttributeError, KeyError, TypeError): pass + + if sort_fields is not None and field_name in sort_fields: + if 'type' in mapping and mapping.get('type') == 'string': + mapping['fields'] = { + 'raw': { + 'type': 'string', + 'index': 'not_analyzed' + } + } + mappings[field_name] = mapping # add a completion mapping for every auto completable field @@ -282,20 +311,29 @@ def make_mapping(self): complete_name = "{0}_complete".format(field_name) mappings[complete_name] = {"type": "completion"} - return { + es_mapping = { self.doc_type: { "properties": mappings } } - def get_mapping(self): - if self._mapping is None: - # TODO: could be done once for every index/doc_type ? - full_mapping = es_client.indices.get_mapping(index=self.index, - doc_type=self.doc_type) - self._mapping = full_mapping[self.index]['mappings'][self.doc_type]['properties'] + parent = self.model.Elasticsearch.parent_model + if parent: + parent.es.create_index() + es_mapping[self.doc_type]['_parent'] = { + 'type': parent.es.doc_type + } + + return es_mapping + + def get_full_mapping(self): + if self._full_mapping is None: + self._full_mapping = es_client.indices.get_mapping(index=self.index, doc_type=self.doc_type) - return self._mapping + return self._full_mapping + + def get_mapping(self): + return self.get_full_mapping()[self.index]['mappings'][self.doc_type]['properties'] def get_settings(self): """ @@ -332,7 +370,6 @@ def create_index(self, ignore=True): body = {} if hasattr(settings, 'ELASTICSEARCH_SETTINGS'): body['settings'] = settings.ELASTICSEARCH_SETTINGS - es_client.indices.create(self.index, body=body, ignore=ignore and 400) diff --git a/django_elasticsearch/models.py b/django_elasticsearch/models.py index 39ab19f..e2a2606 100644 --- a/django_elasticsearch/models.py +++ b/django_elasticsearch/models.py @@ -31,6 +31,8 @@ class Elasticsearch: mapping = None serializer_class = EsJsonSerializer fields = None + sort_fields = None + parent_model = None facets_limit = 10 facets_fields = None # http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-suggesters-term.html diff --git a/django_elasticsearch/tests/test_indexable.py b/django_elasticsearch/tests/test_indexable.py index 1d0d9b2..b165907 100644 --- a/django_elasticsearch/tests/test_indexable.py +++ b/django_elasticsearch/tests/test_indexable.py @@ -1,16 +1,13 @@ # -*- coding: utf-8 -*- -from elasticsearch import NotFoundError - +from django import get_version from django.test import TestCase from django.test.utils import override_settings +from elasticsearch import NotFoundError +from test_app.models import TestModel, Test2Model from django_elasticsearch.managers import es_client from django_elasticsearch.tests.utils import withattrs -from test_app.models import TestModel - -from django import get_version - class EsIndexableTestCase(TestCase): def setUp(self): @@ -95,8 +92,8 @@ def test_fuzziness(self): "default": "test_analyzer", "analyzer": { "test_analyzer": { - "type": "custom", - "tokenizer": "standard", + "type": "custom", + "tokenizer": "standard", } } } @@ -131,7 +128,7 @@ def test_auto_completion(self): @withattrs(TestModel.Elasticsearch, 'fields', ['username', 'date_joined']) def test_get_mapping(self): - TestModel.es._mapping = None + TestModel.es._full_mapping = None TestModel.es.flush() TestModel.es.do_update() @@ -140,7 +137,51 @@ def test_get_mapping(self): # Reset the eventual cache on the Model mapping mapping = TestModel.es.get_mapping() - TestModel.es._mapping = None + TestModel.es._full_mapping = None + self.assertEqual(expected, mapping) + + @withattrs(TestModel.Elasticsearch, 'fields', ['username', 'date_joined']) + def test_get_full_mapping(self): + TestModel.es._full_mapping = None + TestModel.es.flush() + TestModel.es.do_update() + + expected = {u'django-test': {u'mappings': {u'test-doc-type': {u'properties': { + u'date_joined': {u'format': u'dateOptionalTime', u'type': u'date'}, + u'username': {u'index': u'not_analyzed', u'type': u'string'} + }}}}} + + # Reset the eventual cache on the Model mapping + mapping = TestModel.es.get_full_mapping() + TestModel.es._full_mapping = None + self.assertEqual(expected, mapping) + + @withattrs(TestModel.Elasticsearch, 'fields', ['username', 'date_joined']) + @withattrs(Test2Model.Elasticsearch, 'doc_type', 'test-2-doc-type') + @withattrs(Test2Model.Elasticsearch, 'fields', ['text', 'email']) + @withattrs(Test2Model.Elasticsearch, 'parent_model', TestModel) + def test_get_parent_mapping(self): + self.maxDiff = None + Test2Model.es._full_mapping = None + Test2Model.es.flush() + Test2Model.es.do_update() + + expected = {u'django-test': {u'mappings': {u'test-2-doc-type': { + u'properties': { + u'text': {u'type': u'string'}, + u'email': {u'type': u'string'} + }, + u'_routing': { + u'required': True + }, + u'_parent': { + u'type': u'test-doc-type' + } + }}}} + + # Reset the eventual cache on the Model mapping + mapping = Test2Model.es.get_full_mapping() + Test2Model.es._full_mapping = None self.assertEqual(expected, mapping) def test_get_settings(self): @@ -171,8 +212,8 @@ def test_diff(self): expected = { u'first_name': { - 'es': u'woot', - 'db': u'pouet' + 'es': u'woot', + 'db': u'pouet' } } @@ -209,7 +250,7 @@ def setUp(self): post_save.connect(es_save_callback) post_delete.connect(es_delete_callback) post_migrate.connect(es_syncdb_callback) - + if int(get_version()[2]) >= 6: sender = app else: diff --git a/django_elasticsearch/tests/test_qs.py b/django_elasticsearch/tests/test_qs.py index 60f8c12..08bd3cb 100644 --- a/django_elasticsearch/tests/test_qs.py +++ b/django_elasticsearch/tests/test_qs.py @@ -113,7 +113,7 @@ def test_suggestions(self): u'last_name': [ {u'length': 5, u'offset': 0, - u'options': [{u'freq': 3, + u'options': [{u'freq': 6, u'score': 0.8, u'text': u'smith'}], u'text': u'smath'}]} @@ -220,7 +220,7 @@ def test_isnull_lookup(self): @withattrs(TestModel.Elasticsearch, 'fields', ['id', 'date_joined_exp']) def test_sub_object_lookup(self): TestModel.es._fields = None - TestModel.es._mapping = None + TestModel.es._full_mapping = None TestModel.es.flush() # update the mapping time.sleep(2) @@ -232,14 +232,14 @@ def test_sub_object_lookup(self): self.assertEqual(qs.count(), 4) def test_nested_filter(self): - TestModel.es._mapping = None + TestModel.es._full_mapping = None qs = TestModel.es.filter(groups=self.group) self.assertEqual(qs.count(), 1) @withattrs(TestModel.Elasticsearch, 'fields', ['id', 'date_joined_exp']) def test_filter_date_range(self): TestModel.es._fields = None - TestModel.es._mapping = None + TestModel.es._full_mapping = None TestModel.es.flush() # update the mapping time.sleep(2) @@ -303,7 +303,7 @@ def test_chain_filter_exclude(self): @withattrs(TestModel.Elasticsearch, 'mappings', {}) def test_contains(self): TestModel.es._fields = None - TestModel.es._mapping = None + TestModel.es._full_mapping = None TestModel.es.flush() # update the mapping, username is now analyzed time.sleep(2) # TODO: flushing is not immediate, find a better way contents = TestModel.es.filter(username__contains='woot').deserialize() diff --git a/readme.md b/readme.md index 55b19ba..227c577 100644 --- a/readme.md +++ b/readme.md @@ -154,6 +154,10 @@ Each EsIndexable model receive an Elasticsearch class that contains its options Defaults to None The fields on which to activate auto-completion (needs a specific mapping). +* **sort_fields** + Defaults to None + A list of fields that will receive a specific mapping for sorting purposes. (See [this](https://www.elastic.co/guide/en/elasticsearch/guide/1.x/multi-fields.html)) + API === diff --git a/setup.py b/setup.py index aa8eebe..ee6e267 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="django-elasticsearch", - version="0.5", + version="0.5.2", description="Simple wrapper around py-elasticsearch to index/search a django Model.", author="Robin Tissot", url="https://github.com/liberation/django_elasticsearch", diff --git a/test_project/tox.ini b/test_project/tox.ini index 547e86b..9d8ed4f 100644 --- a/test_project/tox.ini +++ b/test_project/tox.ini @@ -15,7 +15,7 @@ deps = django16: django>=1.6, <1.7 django17: django>=1.7, <1.8 django18: django>=1.8, <1.9 - django19: django>=1.9, <2.0 + django19: django>=1.9, <1.10 django{14,16,17}: djangorestframework>=2.4, <3.0 django{18,19}: djangorestframework>3.0, <3.2 -r../requirements.txt