From dc1a608ddd6bfa5d19c62bd5601333ba9fe96239 Mon Sep 17 00:00:00 2001 From: Ricardo Branco Date: Sat, 21 Oct 2023 13:49:59 +0200 Subject: [PATCH] Further improve pagination on Gitea & Pagure --- services/gitea.py | 4 ++-- services/pagure.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/gitea.py b/services/gitea.py index c2f7f1f..63b98a2 100644 --- a/services/gitea.py +++ b/services/gitea.py @@ -63,8 +63,8 @@ def _get_paginated2( entries: list[dict] = [] def get_page(page: int) -> list[dict]: + params["page"] = str(page) try: - params["page"] = str(page) got = self.session.get(url, params=params) got.raise_for_status() return got.json() @@ -74,7 +74,7 @@ def get_page(page: int) -> list[dict]: ) return [] - with ThreadPoolExecutor(max_workers=10) as executor: + with ThreadPoolExecutor(max_workers=min(10, last_page - 1)) as executor: pages_to_fetch = range(2, last_page + 1) results = executor.map(get_page, pages_to_fetch) for result in results: diff --git a/services/pagure.py b/services/pagure.py index 2d22020..9dc0b04 100644 --- a/services/pagure.py +++ b/services/pagure.py @@ -72,8 +72,8 @@ def _get_paginated2( entries: list[dict] = [] def get_page(page: int) -> list[dict]: + params["page"] = str(page) try: - params["page"] = str(page) got = self.session.get(url, params=params) got.raise_for_status() data = got.json() @@ -84,7 +84,7 @@ def get_page(page: int) -> list[dict]: ) return [] - with ThreadPoolExecutor(max_workers=10) as executor: + with ThreadPoolExecutor(max_workers=min(10, last_page - 1)) as executor: pages_to_fetch = range(2, last_page + 1) results = executor.map(get_page, pages_to_fetch) for result in results: