From 57b5fae643e42b2b1464e9a4609b2bf2665171d0 Mon Sep 17 00:00:00 2001 From: Adrien Barbaresi Date: Mon, 24 Apr 2023 17:37:44 +0200 Subject: [PATCH] urlstore fix: compression and total_url_number() --- courlan/urlstore.py | 2 +- tests/urlstore_tests.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/courlan/urlstore.py b/courlan/urlstore.py index 3706783..da6a69e 100644 --- a/courlan/urlstore.py +++ b/courlan/urlstore.py @@ -392,7 +392,7 @@ def get_all_counts(self) -> List[int]: def total_url_number(self) -> int: "Find number of all URLs in store." - return sum(len(self.urldict[d].tuples) for d in self.urldict) + return sum(len(self._load_urls(d)) for d in self.urldict) def download_threshold_reached(self, threshold: float) -> bool: "Find out if the download limit (in seconds) has been reached for one of the websites in store." diff --git a/tests/urlstore_tests.py b/tests/urlstore_tests.py index 32a1bd2..3a60f94 100644 --- a/tests/urlstore_tests.py +++ b/tests/urlstore_tests.py @@ -101,6 +101,7 @@ def test_urlstore(): my_urls = UrlStore(compressed=True) url_buffer = UrlStore()._buffer_urls(example_urls) my_urls.add_urls(example_urls) + assert my_urls.total_url_number() == len(example_urls) # necessary to pickle my_urls._lock = None assert len(pickle.dumps(my_urls)) < len(pickle.dumps(url_buffer)) @@ -109,6 +110,7 @@ def test_urlstore(): my_urls = UrlStore(compressed=True) url_buffer = UrlStore()._buffer_urls(test_urls) my_urls.add_urls(test_urls) + assert my_urls.total_url_number() == len(test_urls) # necessary to pickle my_urls._lock = None assert len(pickle.dumps(my_urls)) < len(pickle.dumps(url_buffer))