Skip to content

Commit

Permalink
Refactor parallel download to use multiprocessing instead of dask.
Browse files Browse the repository at this point in the history
  • Loading branch information
michaelaye committed Nov 27, 2024
1 parent 3abc13c commit 06582fd
Showing 1 changed file with 5 additions and 12 deletions.
17 changes: 5 additions & 12 deletions src/planetarypy/spice/kernels.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@
import requests
import spiceypy as spice
from astropy.time import Time
from dask.distributed import Client
from tqdm.auto import tqdm
from tqdm.contrib.concurrent import process_map
from yarl import URL
Expand Down Expand Up @@ -237,17 +236,11 @@ def get_local_path(self, url) -> Path:
return basepath / u.parent.name / u.name

def _non_blocking_download(self, overwrite: bool = False):
"Use dask for a parallel download."
with Client() as client:
futures = []
for url in tqdm(self.kernel_urls, desc="Kernels downloaded"):
local_path = self.get_local_path(url)
if local_path.exists() and not overwrite:
print(local_path.parent.name, local_path.name, "locally available.")
continue
local_path.parent.mkdir(exist_ok=True, parents=True)
futures.append(client.submit(url_retrieve, url, local_path))
return [f.result() for f in futures]
"Use multiprocessing for parallel download."
paths = [self.get_local_path(url) for url in self.kernel_urls]
args = zip(self.kernel_urls, paths, repeat(overwrite))
_ = process_map(download_one_url, args, max_workers=cpu_count() - 2,
desc="Kernels downloaded")

def _concurrent_download(self, overwrite: bool = False):
paths = [self.get_local_path(url) for url in self.kernel_urls]
Expand Down

0 comments on commit 06582fd

Please sign in to comment.