Merge pull request #152 from ael-code/archivant_shrink

Archivant shrink

archivant/archivant.py

@@ -327,3 +327,30 @@ def _resolve_url(self, url):
             return self._fsdb[os.path.basename(parseResult.path)]
         else:
             raise Exception("url scheme '{}' not supported".format(parseResult.scheme))
+
+    def dangling_files(self):
+        '''iterate over fsdb files no more attached to any volume'''
+        for fid in self._fsdb:
+            if not self._db.file_is_attached('fsdb:///'+fid):
+                yield fid
+
+    def shrink_local_fsdb(self, dangling=True, corrupted=True, dryrun=False):
+        '''shrink local fsdb by removing dangling and/or corrupted files
+
+           return number of deleted files
+        '''
+        log.debug('shrinking local fsdb [danglings={}, corrupted={}]'.format(dangling, corrupted))
+        count = 0
+        if dangling:
+            for fid in self.dangling_files():
+                log.info("shrinking: removing dangling  '{}'".format(fid))
+                if not dryrun:
+                    self._fsdb.remove(fid)
+                count += 1
+        if corrupted:
+            for fid in self._fsdb.corrupted():
+                log.info("shrinking: removing corrupted '{}'".format(fid))
+                if not dryrun:
+                    self._fsdb.remove(fid)
+                count += 1
+        return count

archivant/test/test_shrink.py

@@ -0,0 +1,48 @@
+from archivant.test import TestArchivant
+
+from nose.tools import raises, ok_, eq_
+
+
+class TestArchivantShrink(TestArchivant):
+
+    def test_dangling_files_db_empty(self):
+        '''no dangling files with db empty'''
+        eq_(len([fid for fid in self.arc.dangling_files()]), 0)
+
+    def test_dangling_files_empty(self):
+        '''no dangling files without previous deletion'''
+        n = 3
+        volume_metadata = self.generate_volume_metadata()
+        attachments = self.generate_attachments(n)
+        id = self.arc.insert_volume(volume_metadata, attachments=attachments)
+        self.arc._db.es.indices.refresh()
+        eq_(len([fid for fid in self.arc.dangling_files()]), 0)
+
+    def test_dangling_files(self):
+        n = 3
+        volume_metadata = self.generate_volume_metadata()
+        attachments = self.generate_attachments(n)
+        id = self.arc.insert_volume(volume_metadata, attachments=attachments)
+        self.arc.delete_volume(id)
+        self.arc._db.es.indices.refresh()
+        eq_(len([fid for fid in self.arc.dangling_files()]), n)
+
+    def test_shrink_dangling(self):
+        n = 3
+        volume_metadata = self.generate_volume_metadata()
+        attachments = self.generate_attachments(n)
+        id = self.arc.insert_volume(volume_metadata, attachments=attachments)
+        self.arc.delete_volume(id)
+        self.arc._db.es.indices.refresh()
+        eq_(self.arc.shrink_local_fsdb(dangling=True), n)
+        eq_(len(self.arc._fsdb), 0)
+
+    def test_shrink_dryrun(self):
+        n = 3
+        volume_metadata = self.generate_volume_metadata()
+        attachments = self.generate_attachments(n)
+        id = self.arc.insert_volume(volume_metadata, attachments=attachments)
+        self.arc.delete_volume(id)
+        self.arc._db.es.indices.refresh()
+        eq_(self.arc.shrink_local_fsdb(dangling=True, dryrun=True), n)
+        eq_(len(self.arc._fsdb), n)

libreantdb/api.py

@@ -190,6 +190,13 @@ def user_search(self, query):
         '''
         return self.get_books_querystring(query)
 
+    def file_is_attached(self, url):
+        '''return true if at least one book has
+           file with the given url as attachment
+        '''
+        body = self._get_search_field('_attachments.url', url)
+        return self.es.count(index=self.index_name, body=body)['count'] > 0
+
     def autocomplete(self, fieldname, start):
         raise NotImplementedError()
     # End queries }}}