individual file locked shelf

permacache/locked_shelf.py

@@ -1,9 +1,13 @@
+import json
 import os
 import shelve
 import time
+import uuid
 
 from filelock import FileLock
 
+from permacache.hash import stable_hash
+
 
 class Lock:
     def __init__(self, lock_path, time_path):
@@ -147,3 +151,64 @@ def __exit__(self, *args, **kwargs):
     def close(self):
         if self.shelf is not None:
             self.shelf.close()
+
+
+class IndividualFileLockedStore:
+    """
+    Like LockedShelf, but stores each key in a separate file. Should be
+    broadly multiprocess safe, but you can enhance this by using the
+    multiprocess_safe flag.
+    """
+
+    def __init__(
+        self, path, read_from_shelf_context_manager=None, multiprocess_safe=False
+    ):
+        try:
+            os.makedirs(path)
+        except FileExistsError:
+            pass
+        self.path = path
+        self.lock = Lock(self.path + "/lock", self.path + "/time")
+        self.cache = None
+        self.read_from_shelf_context_manager = read_from_shelf_context_manager
+        self.multi_process_safe = multiprocess_safe
+
+    def _path_for_key(self, key):
+        if len(key) < 40 and all(c.isalnum() or c in "-_.,[](){} " for c in key):
+            return os.path.join(self.path, "." + key)
+        key = stable_hash(key)[:20]
+        return os.path.join(self.path, key)
+
+    def __getitem__(self, key):
+        with open(self._path_for_key(key), "r") as f:
+            result = json.load(f)
+        return result[key]
+
+    def __contains__(self, key):
+        return os.path.exists(self._path_for_key(key))
+
+    def __setitem__(self, key, value):
+        temporary_path = self._path_for_key(key) + "." + uuid.uuid4().hex[:10]
+        with open(temporary_path, "w") as f:
+            json.dump({key: value}, f)
+        os.rename(temporary_path, self._path_for_key(key))
+
+    def __delitem__(self, key):
+        os.remove(self._path_for_key(key))
+
+    def items(self):
+        for filename in os.listdir(self.path):
+            with open(os.path.join(self.path, filename), "r") as f:
+                yield json.load(f)
+
+    def __enter__(self):
+        if self.multi_process_safe:
+            self.lock.__enter__()
+        return self
+
+    def __exit__(self, *args, **kwargs):
+        if self.multi_process_safe:
+            self.lock.__exit__(*args, **kwargs)
+
+    def close(self):
+        self.__exit__()

tests/locked_shelf_test.py

@@ -1,8 +1,11 @@
+import json
+import os
 import random
 import shutil
 import unittest
 
-from permacache.locked_shelf import LockedShelf
+from permacache.hash import stable_hash
+from permacache.locked_shelf import IndividualFileLockedStore, LockedShelf
 
 
 class LockedShelfTest(unittest.TestCase):
@@ -46,7 +49,37 @@ def test_several_accesses(self):
                     else:
                         self.assertFalse(k in s)
 
+    def test_large_key(self):
+        with self.shelf as s:
+            s["a" * 100] = "b"
+            self.assertEqual(s["a" * 100], "b")
+            self.assertTrue("a" * 100 in s)
+            self.assertFalse(stable_hash("a" * 100)[:20] in s)
+
 
 class LockedShelfTestLargeObjects(LockedShelfTest):
     def setUp(self):
         self.shelf = LockedShelf("temp/tempshelf", allow_large_values=True)
+
+
+class IndividualFileLockedStoreTest(LockedShelfTest):
+    def setUp(self):
+        self.shelf = IndividualFileLockedStore("temp/tempshelf")
+
+    def test_put_and_access(self):
+        super().test_put_and_access()
+        self.assertEqual(os.listdir("temp/tempshelf"), [".a"])
+        with open("temp/tempshelf/.a") as f:
+            self.assertEqual(json.load(f), {"a": "b"})
+
+    def test_several_accesses(self):
+        super().test_several_accesses()
+        for p in os.listdir("temp/tempshelf"):
+            self.assertIn(p, [f".{x}" for x in range(100)])
+
+    def test_large_key(self):
+        super().test_large_key()
+        h = stable_hash("a" * 100)[:20]
+        self.assertEqual(os.listdir("temp/tempshelf"), [h])
+        with open("temp/tempshelf/" + h) as f:
+            self.assertEqual(json.load(f), {"a" * 100: "b"})