diff --git a/src/fmu/ensemble/common.py b/src/fmu/ensemble/common.py index ba3267a4..f562ac0c 100644 --- a/src/fmu/ensemble/common.py +++ b/src/fmu/ensemble/common.py @@ -1,14 +1,9 @@ """Common functions for fmu.ensemble""" import os -import sys +import logging -import six - -from .etc import Interaction - -fmux = Interaction() -logger = fmux.basiclogger(__name__) +logger = logging.getLogger(__name__) ENV_NAME = "FMU_CONCURRENCY" @@ -18,27 +13,20 @@ def use_concurrent(): This is based on both an environment variable and presence of concurrent.futures, and on Python version - (Py2 deliberately not attempted to support) Returns: bool: True if concurrency mode should be used """ - if six.PY2: - # Py2-support not attempted - return False - if "concurrent.futures" in sys.modules: - if ENV_NAME not in os.environ: - return True - env_var = os.environ[ENV_NAME] - if ( - str(env_var) == "0" - or str(env_var).lower() == "false" - or str(env_var).lower() == "no" - ): - return False + if ENV_NAME not in os.environ: return True - # If concurrent.futures is not available to import, we end here. - return False + env_var = os.environ[ENV_NAME] + if ( + str(env_var) == "0" + or str(env_var).lower() == "false" + or str(env_var).lower() == "no" + ): + return False + return True def set_concurrent(concurrent): diff --git a/src/fmu/ensemble/ensemble.py b/src/fmu/ensemble/ensemble.py index c12691f3..907f0510 100644 --- a/src/fmu/ensemble/ensemble.py +++ b/src/fmu/ensemble/ensemble.py @@ -749,7 +749,7 @@ def load_smry( time_index="raw", column_keys=None, stacked=True, - cache_eclsum=True, + cache_eclsum=False, start_date=None, end_date=None, include_restart=True, @@ -828,7 +828,7 @@ def load_smry( "load_smry": { "column_keys": column_keys, "time_index": time_index, - "cache_eclsum": cache_eclsum, + "cache_eclsum": False, "start_date": start_date, "end_date": end_date, "include_restart": include_restart, @@ -1074,7 +1074,7 @@ def get_smry_dates( normalize=True, start_date=None, end_date=None, - cache_eclsum=True, + cache_eclsum=False, include_restart=True, ): """Return list of datetimes for an ensemble according to frequency @@ -1111,11 +1111,11 @@ def get_smry_dates( eclsumsdates = [] for _, realization in self.realizations.items(): if realization.get_eclsum( - cache=cache_eclsum, include_restart=include_restart + cache=False, include_restart=include_restart ): eclsumsdates.append( realization.get_eclsum( - cache=cache_eclsum, include_restart=include_restart + cache=False, include_restart=include_restart ).dates ) return unionize_smry_dates(eclsumsdates, freq, normalize, start_date, end_date) @@ -1125,7 +1125,7 @@ def get_smry_stats( column_keys=None, time_index="monthly", quantiles=None, - cache_eclsum=True, + cache_eclsum=False, start_date=None, end_date=None, ): @@ -1181,7 +1181,7 @@ def get_smry_stats( dframe = self.get_smry( time_index=time_index, column_keys=column_keys, - cache_eclsum=cache_eclsum, + cache_eclsum=False, start_date=start_date, end_date=end_date, ) @@ -1434,7 +1434,7 @@ def get_smry( self, time_index=None, column_keys=None, - cache_eclsum=True, + cache_eclsum=False, start_date=None, end_date=None, include_restart=True, @@ -1497,7 +1497,7 @@ def get_smry( realization.get_smry, time_index=time_index, column_keys=column_keys, - cache_eclsum=cache_eclsum, + cache_eclsum=False, include_restart=include_restart, ) for realization in self.realizations.values() @@ -1515,7 +1515,7 @@ def get_smry( realization.get_smry( time_index=time_index, column_keys=column_keys, - cache_eclsum=cache_eclsum, + cache_eclsum=False, include_restart=include_restart, ) .assign(REAL=realidx) diff --git a/src/fmu/ensemble/realization.py b/src/fmu/ensemble/realization.py index 07d0bba8..371c7ef0 100644 --- a/src/fmu/ensemble/realization.py +++ b/src/fmu/ensemble/realization.py @@ -1025,7 +1025,7 @@ def get_smry( self, time_index=None, column_keys=None, - cache_eclsum=True, + cache_eclsum=False, start_date=None, end_date=None, include_restart=True, diff --git a/tests/test_batch.py b/tests/test_batch.py index 5b44b569..36af3562 100644 --- a/tests/test_batch.py +++ b/tests/test_batch.py @@ -38,6 +38,7 @@ def test_batch(): ens = ScratchEnsemble( "reektest", testdir + "/data/testensemble-reek001/" + "realization-*/iter-0" ) + ens.process_batch( batch=[ {"load_scalar": {"localpath": "npv.txt"}}, @@ -45,6 +46,7 @@ def test_batch(): {"load_smry": {"column_keys": "*", "time_index": "daily"}}, ] ) + assert len(ens.get_df("npv.txt")) == 5 assert len(ens.get_df("unsmry--daily")["FOPR"]) == 5490 assert len(ens.get_df("unsmry--yearly")["FOPT"]) == 25 diff --git a/tests/test_ensemble.py b/tests/test_ensemble.py index 0d7a3f5d..d4caa72b 100644 --- a/tests/test_ensemble.py +++ b/tests/test_ensemble.py @@ -13,7 +13,6 @@ from .test_ensembleset import symlink_iter from fmu.ensemble import ScratchEnsemble, ScratchRealization -from fmu.ensemble.common import use_concurrent try: @@ -842,59 +841,6 @@ def test_nonexisting(): assert not nopermission -def test_eclsumcaching(): - """Test caching of eclsum, but only if we don't use concurrency""" - - if use_concurrent(): - pytest.skip("Not testing caching when we use concurrency") - - if "__file__" in globals(): - # Easen up copying test code into interactive sessions - testdir = os.path.dirname(os.path.abspath(__file__)) - else: - testdir = os.path.abspath(".") - - dirs = testdir + "/data/testensemble-reek001/" + "realization-*/iter-0" - ens = ScratchEnsemble("reektest", dirs) - - # The problem here is if you load in a lot of UNSMRY files - # and the Python process keeps them in memory. Not sure - # how to check in code that an object has been garbage collected - # but for garbage collection to work, at least the realization - # _eclsum variable must be None. - - ens.load_smry() - # Default is to do caching, so these will not be None: - assert all([x._eclsum for (idx, x) in ens.realizations.items()]) - - # If we redo this operation, the same objects should all - # be None afterwards: - ens.load_smry(cache_eclsum=None) - assert not any([x._eclsum for (idx, x) in ens.realizations.items()]) - - ens.get_smry() - assert all([x._eclsum for (idx, x) in ens.realizations.items()]) - - ens.get_smry(cache_eclsum=False) - assert not any([x._eclsum for (idx, x) in ens.realizations.items()]) - - ens.get_smry_stats() - assert all([x._eclsum for (idx, x) in ens.realizations.items()]) - - ens.get_smry_stats(cache_eclsum=False) - assert not any([x._eclsum for (idx, x) in ens.realizations.items()]) - - ens.get_smry_dates() - assert all([x._eclsum for (idx, x) in ens.realizations.items()]) - - # Clear the cached objects because the statement above has cached it.. - for _, realization in ens.realizations.items(): - realization._eclsum = None - - ens.get_smry_dates(cache_eclsum=False) - assert not any([x._eclsum for (idx, x) in ens.realizations.items()]) - - def test_filedescriptors(): """Test how filedescriptors are used. diff --git a/tests/test_etc.py b/tests/test_etc.py deleted file mode 100644 index 2db039b3..00000000 --- a/tests/test_etc.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Test a deprecated submodule""" - -import pytest - -from fmu import ensemble - - -def test_deprecated_etc(): - """The following lines was in documentation up to 1.4.0 inclusive. - - v1.4.1 and v1.4.2 miss support for fmu.ensemble.etc.Interaction() - and then it is reinstated in v1.4.3. To be removed for fmu.ensemble v2.0.0 - """ - - with pytest.warns(DeprecationWarning): - fmux = ensemble.etc.Interaction() - logger = fmux.basiclogger(__name__, level="WARNING") - logger.info("testing deprecated code") - - # pylint: disable=import-outside-toplevel - # Test different import: - from fmu.ensemble import etc - - with pytest.warns(DeprecationWarning): - fmux = etc.Interaction() - logger = fmux.basiclogger(__name__, level="WARNING") - logger.info("testing deprecated code")