Extend module support up to 10,000 releases

changelog.rst

@@ -1,6 +1,10 @@
 Commit History
 ==============
 
+0.5.5 (12/01/2023)
+------------------
+- Extend module support up to 10,000 releases
+
 0.5.4 (12/01/2023)
 ------------------
 - Reduce redundancy, improve runtime

docs/index.html

@@ -76,14 +76,11 @@ <h1>Welcome to GitVerse’s documentation!<a class="headerlink" href="#welcome-t
 
 <dl class="py function">
 <dt class="sig sig-object py" id="gitverse.releases.get_api_releases">
-<span class="sig-prename descclassname"><span class="pre">gitverse.releases.</span></span><span class="sig-name descname"><span class="pre">get_api_releases</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><span class="pre">Dict</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">List</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#gitverse.releases.get_api_releases" title="Permalink to this definition">¶</a></dt>
+<span class="sig-prename descclassname"><span class="pre">gitverse.releases.</span></span><span class="sig-name descname"><span class="pre">get_api_releases</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><span class="pre">Generator</span><span class="p"><span class="pre">[</span></span><span class="pre">Dict</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">List</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#gitverse.releases.get_api_releases" title="Permalink to this definition">¶</a></dt>
 <dd><p>Get release notes via git api.</p>
 <dl class="field-list simple">
-<dt class="field-odd">Returns<span class="colon">:</span></dt>
-<dd class="field-odd"><p>Returns release notes in the form of release version and description as key-value pairs gathered via GitHub API.</p>
-</dd>
-<dt class="field-even">Return type<span class="colon">:</span></dt>
-<dd class="field-even"><p>Dict[str, List[str]]</p>
+<dt class="field-odd">Yields<span class="colon">:</span></dt>
+<dd class="field-odd"><p><em>Generator[Dict[str, List[str]]]</em> – Yields release notes in the form of release version and description as key-value pairs gathered via GitHub API.</p>
 </dd>
 </dl>
 </dd></dl>

gitverse/__init__.py

@@ -1 +1 @@
-version = "3.1"
+version = "3.2"

gitverse/releases.py

@@ -15,12 +15,12 @@
 from gitverse.models.callables import md_link_pattern, options
 
 
-def get_api_releases() -> Dict[str, List[str]]:
+def get_api_releases() -> Generator[Dict[str, List[str]]]:
     """Get release notes via git api.
 
-    Returns:
-        Dict[str, List[str]]:
-        Returns release notes in the form of release version and description as key-value pairs gathered via GitHub API.
+    Yields:
+        Generator[Dict[str, List[str]]]:
+        Yields release notes in the form of release version and description as key-value pairs gathered via GitHub API.
     """
     gh_token = os.getenv('GIT_TOKEN') or os.getenv('git_token')
     session = requests.Session()
@@ -31,15 +31,25 @@ def get_api_releases() -> Dict[str, List[str]]:
             session.auth = BearerAuth(token=gh_token)
         else:
             debugger.warning("Trying to collect release notes without github token")
-        response = session.get(url=f'https://api.github.com/repos/{owner}/{repo_name}/releases')
-        if response.ok:
-            debugger.info("Collected release notes via GitHub API")
-            try:
-                return {resp['name']: resp['body'].splitlines() for resp in response.json()}
-            except requests.JSONDecodeError as error:
-                debugger.error(error)
-        else:
-            debugger.error(f"{response.status_code} - {response.text}")
+        page_num = 0
+        # 100 pages with 100 releases per page, should cover up to 10_000 releases
+        while page_num <= 100:
+            page_num += 1
+            response = session.get(url=f'https://api.github.com/repos/{owner}/{repo_name}/releases',
+                                   params={'per_page': 100, 'page': page_num})
+            if response.ok:
+                response_json = response.json()
+                if not response_json:
+                    debugger.debug(f"Page {page_num} returned {response_json}, assuming end of releases.")
+                    break
+                debugger.info(f"Collected release notes on page {page_num} via GitHub API")
+                try:
+                    yield {resp['name']: resp['body'].splitlines() for resp in response_json}
+                except requests.JSONDecodeError as error:
+                    debugger.error(error)
+            else:
+                debugger.error(f"{response.status_code} - {response.text}")
+                break
 
 
 def run_git_cmd(cmd: str) -> str:
@@ -103,13 +113,15 @@ def get_releases() -> Union[List[Dict[str, Union[str, List[str], int, str]]], No
         return
     debugger.info(f"Git tags gathered: {len(tags)}")
     # Update release notes for each version, if available via GitHub API
-    if release_api := get_api_releases():
-        debugger.info(f"Release notes gathered: {len(release_api)}")
+    n = 0
+    for release_api in get_api_releases():
+        n = len(release_api)
         for tag in tags:
             if api_description := release_api.get(tag['version']):
                 tag['description'] = api_description
             else:
-                debugger.warning(f"{tag['version']} could not be found in releases")
+                debugger.warning(f"{tag['version']} is either missing or doesn't have release notes")
+    debugger.info(f"Release notes gathered: {n}")
     if options['reverse']:
         debugger.warning('Converting snippets to reverse order')
         version_updates = sorted(tags, key=lambda x: x['timestamp'], reverse=True)

release_notes.rst

@@ -1,6 +1,10 @@
 Release Notes
 =============
 
+v3.2 (12/01/2023)
+-----------------
+- Includes support up to 10,000 releases (previously 30)
+
 v3.1 (12/01/2023)
 -----------------
 - Reduces redundant code and improves runtime