Further improve pagination on Gitea & Pagure

ricardobranco777 · Oct 21, 2023 · dc1a608 · dc1a608
1 parent c55bf44
commit dc1a608
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 4 deletions.
diff --git a/services/gitea.py b/services/gitea.py
@@ -63,8 +63,8 @@ def _get_paginated2(
         entries: list[dict] = []
 
         def get_page(page: int) -> list[dict]:
+            params["page"] = str(page)
             try:
-                params["page"] = str(page)
                 got = self.session.get(url, params=params)
                 got.raise_for_status()
                 return got.json()
@@ -74,7 +74,7 @@ def get_page(page: int) -> list[dict]:
                 )
             return []
 
-        with ThreadPoolExecutor(max_workers=10) as executor:
+        with ThreadPoolExecutor(max_workers=min(10, last_page - 1)) as executor:
             pages_to_fetch = range(2, last_page + 1)
             results = executor.map(get_page, pages_to_fetch)
             for result in results:

diff --git a/services/pagure.py b/services/pagure.py
@@ -72,8 +72,8 @@ def _get_paginated2(
         entries: list[dict] = []
 
         def get_page(page: int) -> list[dict]:
+            params["page"] = str(page)
             try:
-                params["page"] = str(page)
                 got = self.session.get(url, params=params)
                 got.raise_for_status()
                 data = got.json()
@@ -84,7 +84,7 @@ def get_page(page: int) -> list[dict]:
                 )
             return []
 
-        with ThreadPoolExecutor(max_workers=10) as executor:
+        with ThreadPoolExecutor(max_workers=min(10, last_page - 1)) as executor:
             pages_to_fetch = range(2, last_page + 1)
             results = executor.map(get_page, pages_to_fetch)
             for result in results: