Merge pull request #95 from metabrainz/feedback-rate-limiting

Data fetching rate limiting

Author: mayhem

troi/listenbrainz/feedback.py

@@ -1,6 +1,7 @@
 from collections import defaultdict
 from datetime import datetime, timedelta
 from typing import Optional
+from time import sleep
 
 import requests
 
@@ -40,10 +41,17 @@ def read(self, inputs):
         for idx in range(0, len(mbids), batch_size):
             recording_mbids = mbids[idx: idx + batch_size]
 
-            response = requests.get(
-                f"https://api.listenbrainz.org/1/feedback/user/{self.user_name}/get-feedback-for-recordings",
-                params={"recording_mbids": ",".join(recording_mbids)}
-            )
+            while True:
+                response = requests.get(
+                    f"https://api.listenbrainz.org/1/feedback/user/{self.user_name}/get-feedback-for-recordings",
+                    params={"recording_mbids": ",".join(recording_mbids)}
+                )
+                if response.status_code == 429:
+                    sleep(2)
+                    continue
+
+                break
+
             response.raise_for_status()
             data = response.json()["feedback"]
             if len(data) == 0:

troi/listenbrainz/listens.py

@@ -1,6 +1,7 @@
 from collections import defaultdict
 from datetime import datetime, timedelta
 from typing import Optional
+from time import sleep
 
 import requests
 
@@ -45,6 +46,10 @@ def _fetch_recent_listens_index(self):
                 f"https://api.listenbrainz.org/1/user/{self.user_name}/listens",
                 params={"min_ts": min_ts, "count": 100}
             )
+            if response.status_code == 429:
+                sleep(2)
+                continue
+
             response.raise_for_status()
             data = response.json()["payload"]
             if len(data["listens"]) == 0:

troi/musicbrainz/recording_lookup.py

@@ -1,3 +1,5 @@
+from time import sleep
+
 import requests
 import ujson
 
@@ -40,9 +42,16 @@ def read(self, inputs):
 
         self.debug("- debug %d recordings" % len(recordings))
 
-        r = requests.post(self.SERVER_URL % len(recordings), json=data)
-        if r.status_code != 200:
-            raise PipelineError("Cannot fetch recordings from ListenBrainz: HTTP code %d" % r.status_code)
+        while True:
+            r = requests.post(self.SERVER_URL % len(recordings), json=data)
+            if r.status_code == 429:
+                sleep(2)
+                continue
+
+            if r.status_code != 200:
+                raise PipelineError("Cannot fetch recordings from ListenBrainz: HTTP code %d" % r.status_code)
+
+            break
 
         try:
             rows = ujson.loads(r.text)