diff --git a/hail/python/benchmark/hail/fixtures.py b/hail/python/benchmark/hail/fixtures.py index 2686f8fd62a..e74c58d060b 100644 --- a/hail/python/benchmark/hail/fixtures.py +++ b/hail/python/benchmark/hail/fixtures.py @@ -6,6 +6,7 @@ import pytest import hail as hl +from hailtop.utils import async_to_blocking, retry_transient_errors @pytest.fixture(scope='session') @@ -23,12 +24,9 @@ def resource_dir(request, tmpdir_factory): gs_curl_root = 'https://storage.googleapis.com/hail-common/benchmark' -def __download(data_dir, filename): +async def __download(data_dir, filename): url = os.path.join(gs_curl_root, filename) logging.info(f'downloading: {filename}') - # Note: the below does not work on batch due to docker/ssl problems - # dest = os.path.join(data_dir, filename) - # urlretrieve(url, dest) subprocess.check_call(['curl', url, '-Lfs', '--output', f'{data_dir / filename}']) logging.info(f'done: {filename}') @@ -36,7 +34,13 @@ def __download(data_dir, filename): def localize(path: Path): if not path.exists(): path.parent.mkdir(parents=True, exist_ok=True) - __download(path.parent, path.name) + async_to_blocking( + retry_transient_errors( + __download, + path.parent, + path.name, + ) + ) return path