fix: more robust progress_retrieve (#1361)

scripts/_helpers.py

@@ -389,17 +389,28 @@ def aggregate_costs(n, flatten=False, opts=None, existing_only=False):
 
 
 def progress_retrieve(url, file, disable=False):
+    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"}
+
     if disable:
-        urllib.request.urlretrieve(url, file)
+        response = requests.get(url, headers=headers, stream=True)
+        with open(file, "wb") as f:
+            f.write(response.content)
     else:
-        with tqdm(unit="B", unit_scale=True, unit_divisor=1024, miniters=1) as t:
-
-            def update_to(b=1, bsize=1, tsize=None):
-                if tsize is not None:
-                    t.total = tsize
-                t.update(b * bsize - t.n)
-
-            urllib.request.urlretrieve(url, file, reporthook=update_to)
+        response = requests.get(url, headers=headers, stream=True)
+        total_size = int(response.headers.get("content-length", 0))
+        chunk_size = 1024
+
+        with tqdm(
+            total=total_size,
+            unit="B",
+            unit_scale=True,
+            unit_divisor=1024,
+            desc=file,
+        ) as t:
+            with open(file, "wb") as f:
+                for data in response.iter_content(chunk_size=chunk_size):
+                    f.write(data)
+                    t.update(len(data))
 
 
 def mock_snakemake(

scripts/retrieve_jrc_idees.py

@@ -35,7 +35,7 @@
     to_fn_zp = to_fn + ".zip"
 
     # download .zip file
-    logger.info(f"Downloading JRC IDEES from {url_jrc}.")
+    logger.info(f"Downloading JRC IDEES from '{url_jrc}'.")
     progress_retrieve(url_jrc, to_fn_zp, disable=disable_progress)
 
     # extract