fix: retry .taskcluster.yml fetches on 404

We get this intermittently for whatever reason, and it's constantly busting deployments.

Author: bhearsum

src/ciadmin/generate/in_tree_actions.py

@@ -10,6 +10,7 @@
 import hashlib
 import textwrap
 
+import aiohttp
 import iso8601
 import yaml
 from taskcluster import optionsFromEnvironment
@@ -66,10 +67,17 @@ def hash(val):
             # TODO: perhaps we should do this for partly globbed branches,
             # eg: release* ?
             # we'd have to fetch that list from the server to do that
-            if "*" not in b:
+            if "*" not in b and "*" not in p.repo:
 
                 def process(project, branch_name, task):
-                    tcy = task.result()
+                    try:
+                        tcy = task.result()
+                    except aiohttp.ClientResponseError as e:
+                        # .taskcluster.yml doesn't exist. This can happen if
+                        # a project owner moves it away to disable Taskcluster.
+                        if e.status == 404:
+                            return
+                        raise e
 
                     # some ancient projects have no .taskcluster.yml
                     if not tcy:
@@ -102,7 +110,7 @@ def process(project, branch_name, task):
                 future.add_done_callback(functools.partial(process, p, b))
                 futures.append(future)
 
-    await asyncio.gather(*futures)
+    await asyncio.gather(*futures, return_exceptions=True)
     return rv
 
 

src/ciadmin/generate/tcyml.py

@@ -6,7 +6,7 @@
 
 from asyncio import Lock
 
-import aiohttp
+from aiohttp_retry import ExponentialRetry, RetryClient
 from tcadmin.util.sessions import aiohttp_session
 
 _cache = {}
@@ -49,15 +49,16 @@ async def get(repo_path, repo_type="hg", revision=None, default_branch=None):
         if repo_path in _cache:
             return _cache[repo_path]
 
-        try:
-            async with aiohttp_session().get(url) as response:
-                response.raise_for_status()
-                result = await response.read()
-        except aiohttp.ClientResponseError as e:
-            if e.status == 404:
-                result = None
-            else:
-                raise e
+        client = RetryClient(
+            client_session=aiohttp_session(),
+            # Despite only setting 404 here, 5xx statuses will still be retried
+            # for. See https://github.com/inyutin/aiohttp_retry?tab=readme-ov-file
+            # for details.
+            retry_options=ExponentialRetry(attempts=5, statuses={404}),
+        )
+        async with client.get(url) as response:
+            response.raise_for_status()
+            result = await response.read()
 
         _cache[repo_path] = result
         return _cache[repo_path]