From a44a1f9d6837b8e1e90d8fedf6c39b4b0a3eb228 Mon Sep 17 00:00:00 2001 From: Edmund Higham Date: Wed, 30 Oct 2024 17:30:33 -0400 Subject: [PATCH] fixes for running benchmarks with no arguments --- hail/python/hail/backend/backend.py | 10 ++++++++ hail/python/hail/backend/py4j_backend.py | 4 +-- hail/python/hail/backend/service_backend.py | 23 ++++++++++++++--- hail/python/hail/context.py | 28 ++++++++++----------- 4 files changed, 44 insertions(+), 21 deletions(-) diff --git a/hail/python/hail/backend/backend.py b/hail/python/hail/backend/backend.py index dd35b670e660..8d8e00926299 100644 --- a/hail/python/hail/backend/backend.py +++ b/hail/python/hail/backend/backend.py @@ -398,7 +398,17 @@ def requires_lowering(self): def local_tmpdir(self) -> str: pass + @local_tmpdir.setter + @abc.abstractmethod + def local_tmpdir(self, dir: str) -> None: + pass + @property @abc.abstractmethod def remote_tmpdir(self) -> str: pass + + @remote_tmpdir.setter + @abc.abstractmethod + def remote_tmpdir(self, dir: str) -> None: + pass diff --git a/hail/python/hail/backend/py4j_backend.py b/hail/python/hail/backend/py4j_backend.py index c34ca1d6b986..dfdcf5a685a8 100644 --- a/hail/python/hail/backend/py4j_backend.py +++ b/hail/python/hail/backend/py4j_backend.py @@ -332,7 +332,7 @@ def local_tmpdir(self) -> str: return self._local_tmpdir @local_tmpdir.setter - def local_tmpdir(self, tmpdir) -> str: + def local_tmpdir(self, tmpdir: str) -> None: self._local_tmpdir = tmpdir self._jbackend.pySetLocalTmp(tmpdir) @@ -341,6 +341,6 @@ def remote_tmpdir(self) -> str: return self._remote_tmpdir @remote_tmpdir.setter - def remote_tmpdir(self, tmpdir) -> str: + def remote_tmpdir(self, tmpdir: str) -> None: self._remote_tmpdir = tmpdir self._jbackend.pySetRemoteTmp(tmpdir) diff --git a/hail/python/hail/backend/service_backend.py b/hail/python/hail/backend/service_backend.py index 89490ec03d3b..941d1d43bae8 100644 --- a/hail/python/hail/backend/service_backend.py +++ b/hail/python/hail/backend/service_backend.py @@ -5,12 +5,12 @@ import warnings from contextlib import AsyncExitStack from dataclasses import dataclass -from typing import Any, Awaitable, Dict, List, Mapping, Optional, Set, Tuple, TypeVar, Union +from typing import Any, Awaitable, Dict, List, Mapping, NoReturn, Optional, Set, Tuple, TypeVar, Union import orjson import hailtop.aiotools.fs as afs -from hail.context import TemporaryDirectory, TemporaryFilename, revision, tmp_dir, version +from hail.context import TemporaryDirectory, TemporaryFilename, revision, version from hail.experimental import read_expression, write_expression from hail.utils import FatalError from hailtop import yamlx @@ -239,8 +239,7 @@ def __init__( self._batch_was_submitted: bool = False self.disable_progress_bar = disable_progress_bar self.batch_attributes = batch_attributes - self.local_tmpdir = tmp_dir() - self.remote_tmpdir = remote_tmpdir + self._remote_tmpdir = remote_tmpdir self.flags: Dict[str, str] = {} self._registered_ir_function_names: Set[str] = set() self.driver_cores = driver_cores @@ -520,3 +519,19 @@ def get_flags(self, *flags: str) -> Mapping[str, str]: @property def requires_lowering(self): return True + + @property + def local_tmpdir(self) -> NoReturn: + raise AttributeError('local tmp folders are not supported on the batch backend') + + @local_tmpdir.setter + def local_tmpdir(self, tmpdir: str) -> NoReturn: + raise AttributeError('local tmp folders are not supported on the batch backend') + + @property + def remote_tmpdir(self) -> str: + return self._remote_tmpdir + + @remote_tmpdir.setter + def remote_tmpdir(self, tmpdir: str) -> None: + self._remote_tmpdir = tmpdir diff --git a/hail/python/hail/context.py b/hail/python/hail/context.py index 4ffc0f8b9ede..49914ac6e5fb 100644 --- a/hail/python/hail/context.py +++ b/hail/python/hail/context.py @@ -67,8 +67,6 @@ def create( log: str, quiet: bool, append: bool, - tmpdir: str, - local_tmpdir: str, default_reference: str, global_seed: Optional[int], backend: Backend, @@ -77,25 +75,17 @@ def create( log=log, quiet=quiet, append=append, - tmpdir=tmpdir, - local_tmpdir=local_tmpdir, global_seed=global_seed, backend=backend, ) hc.initialize_references(default_reference) return hc - @typecheck_method( - log=str, quiet=bool, append=bool, tmpdir=str, local_tmpdir=str, global_seed=nullable(int), backend=Backend - ) - def __init__(self, log, quiet, append, tmpdir, local_tmpdir, global_seed, backend): + @typecheck_method(log=str, quiet=bool, append=bool, global_seed=nullable(int), backend=Backend) + def __init__(self, log, quiet, append, global_seed, backend: Backend): assert not Env._hc self._log = log - - self._tmpdir = tmpdir - self._local_tmpdir = local_tmpdir - self._backend = backend self._warn_cols_order = True @@ -138,6 +128,14 @@ def initialize_references(self, default_reference): else: self._default_ref = ReferenceGenome.read(default_reference) + @property + def _tmpdir(self) -> str: + return self._backend.remote_tmpdir + + @property + def _local_tmpdir(self) -> str: + return self._backend.local_tmpdir + @property def default_reference(self) -> ReferenceGenome: assert self._default_ref is not None, '_default_ref should have been initialized in HailContext.create' @@ -500,7 +498,7 @@ def init_spark( if not backend.fs.exists(tmpdir): backend.fs.mkdir(tmpdir) - HailContext.create(log, quiet, append, tmpdir, local_tmpdir, default_reference, global_seed, backend) + HailContext.create(log, quiet, append, default_reference, global_seed, backend) if not quiet: connect_logger(backend._utils_package_object, 'localhost', 12888) @@ -571,7 +569,7 @@ async def init_batch( tmpdir = os.path.join(backend.remote_tmpdir, 'tmp/hail', secret_alnum_string()) local_tmpdir = _get_local_tmpdir(local_tmpdir) - HailContext.create(log, quiet, append, tmpdir, local_tmpdir, default_reference, global_seed, backend) + HailContext.create(log, quiet, append, default_reference, global_seed, backend) @typecheck( @@ -623,7 +621,7 @@ def init_local( if not backend.fs.exists(tmpdir): backend.fs.mkdir(tmpdir) - HailContext.create(log, quiet, append, tmpdir, tmpdir, default_reference, global_seed, backend) + HailContext.create(log, quiet, append, default_reference, global_seed, backend) if not quiet: connect_logger(backend._utils_package_object, 'localhost', 12888)