Merge remote-tracking branch 'insomnialab/release-0.3'

.travis.yml

@@ -7,7 +7,7 @@ services:
   - elasticsearch
 
 install:
-  - "pip install flake8"
+  - "pip install flake8 sphinx"
   - "python setup.py install"
 
 before_script:
@@ -16,3 +16,4 @@ before_script:
 script:
   - 'flake8'
   - python setup.py test
+  - "python setup.py build_sphinx"

CHANGELOG.rst

@@ -0,0 +1,35 @@
+===================
+Libreant changelog
+===================
+
+0.3
++++
+
+Major changes:
+--------------
+- Implemented a role-based access control layer.
+  This means that libreant now support the common ``login`` procedure.
+  This functionality isn't documented yet, anyway you can use the brand new ``libreant-users`` command to manage users, groups and capabilities,
+  and enable this feature at runtime with the ``--users-db`` parameter.
+  The default user is (user: admin, password: admin)
+
+Web interface:
+--------------
+- Added possibility to delete a volume through a button on the single-volume-view page.
+- New user menu (only in users-mode)
+- New login/logut pages.
+- Improoved error messages/pages
+
+Deployment:
+-----------
+- Removed elasticsearch strong dependecy.
+  Now libreant can be started with elasticsearch still not ready or not running.
+- Bugfix: make libreant command exits with code 1 on exception.
+- Fixed ``elasticsearch-py`` version dependency. Now the version must be ``>=1`` and ``<2``.
+- Reloader is used only in debug mode (``--debug``).
+- More uniform logs.
+
+Documentation:
+--------------
+- The suggested version for elasticsearch installation has been updated: ``1.4`` -> ``1.7``
+- A lot of packages have been inserted in the official docs.

MANIFEST.in

@@ -3,3 +3,4 @@ recursive-include webant/templates *
 recursive-include webant/translations *.po
 include msgfmt.py
 include README.rst
+include CHANGELOG.rst

Vagrantfile

@@ -14,7 +14,7 @@ end
 
 $script = <<SCRIPT
 wget -qO - https://packages.elasticsearch.org/GPG-KEY-elasticsearch | apt-key add -
-echo "deb http://packages.elasticsearch.org/elasticsearch/1.4/debian stable main" >> /etc/apt/sources.list
+echo "deb http://packages.elasticsearch.org/elasticsearch/1.7/debian stable main" >> /etc/apt/sources.list
 apt-get update && apt-get install -y python python-dev python-virtualenv openjdk-7-jre-headless elasticsearch
 update-rc.d elasticsearch defaults
 service elasticsearch start

archivant/__init__.py

@@ -1,4 +1,4 @@
 from archivant import Archivant
 
 
-__all__ = [Archivant]
+__all__ = ['Archivant']

archivant/archivant.py

@@ -19,12 +19,13 @@
 class Archivant():
     ''' Implementation of a Data Access Layer
 
-        Archivant handling both an fsdb instance and
-        a libreantdb one exposes an API to operate on 'volumes'.
+        Archivant handles both an fsdb instance and
+        a libreantdb one and exposes an high-level API to operate on 'volumes'.
 
         A 'volume' represents a physical/digital object stored within archivant.
-        Volumes are structured as described in `Archivant.normalize_volume`,
-        these have metadata and a list of attachments.
+        Volumes are structured as described in :meth:`~Archivant.normalize_volume`;
+        shortly, they have language, metadata and attachments.
+        An attachment is an URL plus some metadata.
 
         If you won't configure the FSDB_PATH parameter, fsdb will not be initialized
         and archivant will start in metadata-only mode.
@@ -50,10 +51,16 @@ def __init__(self, conf={}):
         # initialize elasticsearch
         if not self._config['ES_INDEXNAME']:
             raise ValueError('ES_INDEXNAME cannot be empty')
-        db = DB(Elasticsearch(hosts=self._config['ES_HOSTS']),
-                index_name=self._config['ES_INDEXNAME'])
-        db.setup_db()
-        self._db = db
+        self.__db = None
+
+    @property
+    def _db(self):
+        if self.__db is None:
+            db = DB(Elasticsearch(hosts=self._config['ES_HOSTS']),
+                    index_name=self._config['ES_INDEXNAME'])
+            db.setup_db()
+            self.__db = db
+        return self.__db
 
     @property
     def _fsdb(self):
@@ -76,25 +83,27 @@ def normalize_volume(volume):
 
            This function makes side effect on input volume
 
-           output example:
-           {
-            'id': 'AU0paPZOMZchuDv1iDv8',
-            'type': 'volume',
-            'metadata': {'_language': 'en',
-                         'key1': 'value1',
-                         'key2': 'value2',
-                         'key3': 'value3'},
-            'attachments': [{'id': 'a910e1kjdo2d192d1dko1p2kd1209d',
-                             'type' : 'attachment',
-                             'url': 'fsdb:///624bffa8a6f90813b7982d0e5b4c1475ebec40e3',
-                             'metadata': {'download_count': 0,
-                                          'mime': 'application/json',
-                                          'name': 'tmp9fyat_',
-                                          'notes': 'this file is awsome',
-                                          'sha1': '624bffa8a6f90813b7982d0e5b4c1475ebec40e3',
-                                          'size': 10}
-                           }]
-           }
+           output example::
+
+            {
+                'id': 'AU0paPZOMZchuDv1iDv8',
+                'type': 'volume',
+                'metadata': {'_language': 'en',
+                            'key1': 'value1',
+                            'key2': 'value2',
+                            'key3': 'value3'},
+                'attachments': [{'id': 'a910e1kjdo2d192d1dko1p2kd1209d',
+                                'type' : 'attachment',
+                                'url': 'fsdb:///624bffa8a6f90813b7982d0e5b4c1475ebec40e3',
+                                'metadata': {'download_count': 0,
+                                            'mime': 'application/json',
+                                            'name': 'tmp9fyat_',
+                                            'notes': 'this file is awsome',
+                                            'sha1': '624bffa8a6f90813b7982d0e5b4c1475ebec40e3',
+                                            'size': 10}
+                            }]
+            }
+
         '''
         res = dict()
         res['type'] = 'volume'
@@ -223,25 +232,30 @@ def insert_attachments(self, volumeID, attachments):
     def insert_volume(self, metadata, attachments=[]):
         '''Insert a new volume
 
-           Returns the ID of the added volume
+        Returns the ID of the added volume
 
-           `metadata` must be a dict containg metadata of the volume.
-           The only required key is `_language`
-           {
-             "_language" : "it",  # language of the metadata
-             "key1" : "value1",   # attribute
-             "key2" : "value2",
+        `metadata` must be a dict containg metadata of the volume::
+
+            {
+              "_language" : "it",  # language of the metadata
+              "key1" : "value1",   # attribute
+              "key2" : "value2",
               ...
-             "keyN" : "valueN"
-           }
-
-           `attachments` must be an array of dict
-           {
-               "file"  : "/prova/una/path/a/caso" # path or fp
-               "name"  : "nome_buffo.ext"         # name of the file (extension included) [optional if a path was given]
-               "mime"  : "application/json"       # mime type of the file [optional]
-               "notes" : "this file is awesome"   # notes that will be attached to this file [optional]
-           }
+              "keyN" : "valueN"
+            }
+
+            The only required key is `_language`
+
+
+        `attachments` must be an array of dict::
+
+            {
+              "file"  : "/prova/una/path/a/caso" # path or fp
+              "name"  : "nome_buffo.ext"         # name of the file (extension included) [optional if a path was given]
+              "mime"  : "application/json"       # mime type of the file [optional]
+              "notes" : "this file is awesome"   # notes that will be attached to this file [optional]
+            }
+
         '''
 
         log.debug("adding new volume:\n\tdata: {}\n\tfiles: {}".format(metadata, attachments))
@@ -328,9 +342,9 @@ def update_volume(self, volumeID, metadata):
     def update_attachment(self, volumeID, attachmentID, metadata):
         '''update an existing attachment
 
-           the given metadata dict will be merged with the old one.
-           only the following fields could be updated:
-             [name, mime, notes, download_count]
+        the given metadata dict will be merged with the old one.
+        only the following fields could be updated:
+        [name, mime, notes, download_count]
         '''
         log.debug('updating metadata of attachment {} from volume {}'.format(attachmentID, volumeID))
         modifiable_fields = ['name', 'mime', 'notes', 'download_count']

archivant/test/class_template.py

@@ -27,12 +27,15 @@ def setUp(self):
         self.arc = Archivant(conf)
 
     def tearDown(self):
-        self.es.delete_by_query(index=self.TEST_ES_INDEX,
-                                body={'query': {'match_all': {}}})
+        self.arc._db.es.delete_by_query(index=self.TEST_ES_INDEX,
+                                        body={'query': {'match_all': {}}})
         rmtree(self.tmpDir)
         self.tmpDir = None
         self.arc = None
 
+    def refresh_index(self):
+        self.arc._db.es.indices.refresh(index=self.TEST_ES_INDEX)
+
     @staticmethod
     def generate_volume_metadata():
         return {'_language': 'en',

archivant/test/test_get_attachment.py

@@ -11,7 +11,6 @@ def test_get_attachment(self):
         attachments = self.generate_attachments(1)
         id = self.arc.insert_volume(volume_metadata, attachments=attachments)
         added_volume = self.arc.get_volume(id)
-        print added_volume['attachments'][0]['id']
         self.arc.get_attachment(id, added_volume['attachments'][0]['id'])
 
     def test_get_file(self):

archivant/test/test_get_volume.py

@@ -20,7 +20,7 @@ def test_get_all_volumes(self):
         for _ in range(n):
             id = self.arc.insert_volume(self.generate_volume_metadata())
             ids.append(id)
-        self.arc._db.es.indices.refresh()
+        self.refresh_index()
         volumes = [vol for vol in self.arc.get_all_volumes()]
         for vol in volumes:
             ok_(vol['id'] in ids)

archivant/test/test_instantiation.py

@@ -2,9 +2,8 @@
 from archivant.exceptions import FileOpNotSupported
 from tempfile import mkdtemp
 from shutil import rmtree
-from elasticsearch import Elasticsearch
+from elasticsearch import Elasticsearch, ElasticsearchException
 from nose.tools import raises
-from urllib3.exceptions import LocationValueError
 
 
 TEST_ES_INDEX = 'test-archivant'
@@ -14,7 +13,8 @@
 def cleanup(esIndex=None, tmpDir=None):
     if esIndex is not None:
         es = Elasticsearch()
-        es.indices.delete(esIndex)
+        if es.indices.exists(esIndex):
+            es.indices.delete(esIndex)
 
     if tmpDir is not None:
         rmtree(tmpDir)
@@ -45,13 +45,27 @@ def test_instantiation_no_indexname():
         cleanup(tmpDir=tmpDir)
 
 
-@raises(LocationValueError)
 def test_instantiation_hosts_error():
+    '''this should not raise errors'''
     tmpDir = mkdtemp(prefix=FSDB_PATH_PREFIX)
     conf = {'ES_INDEXNAME': TEST_ES_INDEX,
-            'ES_HOSTS': "",
+            'ES_HOSTS': "127.0.0.1:12345",
             'FSDB_PATH': tmpDir}
     try:
         Archivant(conf)
     finally:
         cleanup(tmpDir=tmpDir)
+
+
+@raises(ElasticsearchException)
+def test_instantiation_hosts_error_on_query():
+    '''explicitly doing queries will raise error'''
+    tmpDir = mkdtemp(prefix=FSDB_PATH_PREFIX)
+    conf = {'ES_INDEXNAME': TEST_ES_INDEX,
+            'ES_HOSTS': "127.0.0.1:12345",
+            'FSDB_PATH': tmpDir}
+    try:
+        arc = Archivant(conf)
+        arc.get_volume('whatever')
+    finally:
+        cleanup(tmpDir=tmpDir)

archivant/test/test_shrink.py

@@ -15,7 +15,7 @@ def test_dangling_files_empty(self):
         volume_metadata = self.generate_volume_metadata()
         attachments = self.generate_attachments(n)
         self.arc.insert_volume(volume_metadata, attachments=attachments)
-        self.arc._db.es.indices.refresh()
+        self.refresh_index()
         eq_(len([fid for fid in self.arc.dangling_files()]), 0)
 
     def test_dangling_files(self):
@@ -24,7 +24,7 @@ def test_dangling_files(self):
         attachments = self.generate_attachments(n)
         id = self.arc.insert_volume(volume_metadata, attachments=attachments)
         self.arc.delete_volume(id)
-        self.arc._db.es.indices.refresh()
+        self.refresh_index()
         eq_(len([fid for fid in self.arc.dangling_files()]), n)
 
     def test_shrink_dangling(self):
@@ -33,7 +33,7 @@ def test_shrink_dangling(self):
         attachments = self.generate_attachments(n)
         id = self.arc.insert_volume(volume_metadata, attachments=attachments)
         self.arc.delete_volume(id)
-        self.arc._db.es.indices.refresh()
+        self.refresh_index()
         eq_(self.arc.shrink_local_fsdb(dangling=True), n)
         eq_(len(self.arc._fsdb), 0)
 
@@ -43,6 +43,6 @@ def test_shrink_dryrun(self):
         attachments = self.generate_attachments(n)
         id = self.arc.insert_volume(volume_metadata, attachments=attachments)
         self.arc.delete_volume(id)
-        self.arc._db.es.indices.refresh()
+        self.refresh_index()
         eq_(self.arc.shrink_local_fsdb(dangling=True, dryrun=True), n)
         eq_(len(self.arc._fsdb), n)

archivant/test/test_update_volume.py

@@ -22,8 +22,6 @@ def test_update_volume_add(self):
         volume_metadata = {'_language': 'en',
                            'old_1': 'old_value_1'}
         id = self.arc.insert_volume(volume_metadata)
-        from pprint import pprint
-        pprint(volume_metadata)
         volume_metadata['new_1'] = 'new_value_1'
         self.arc.update_volume(id, volume_metadata)
         vol_metadata = (self.arc.get_volume(id))['metadata']