From b5952d1479478d243270b75c30c7db886203447e Mon Sep 17 00:00:00 2001 From: "Jeremy M. G. Leung" Date: Fri, 14 Feb 2025 14:58:29 -0500 Subject: [PATCH 1/9] remove extra export and classes --- src/westpa/core/_rc.py | 4 +- src/westpa/core/binning/_assign.pyx | 1 - src/westpa/core/propagators/executable.py | 2 +- src/westpa/core/reweight/_reweight.pyx | 32 ++++---- src/westpa/core/sim_manager.py | 2 +- src/westpa/core/yamlcfg.py | 93 ----------------------- 6 files changed, 19 insertions(+), 115 deletions(-) diff --git a/src/westpa/core/_rc.py b/src/westpa/core/_rc.py index 50abbc30a..07efed852 100644 --- a/src/westpa/core/_rc.py +++ b/src/westpa/core/_rc.py @@ -16,7 +16,7 @@ from westpa.core.binning.assign import BinMapper from westpa.core.binning import RectilinearBinMapper, RecursiveBinMapper, MABBinMapper, BinlessMapper from .yamlcfg import YAMLConfig -from .yamlcfg import YAMLSystem +from .systems import WESTSystem from . import extloader from ..work_managers import SerialWorkManager @@ -548,7 +548,7 @@ def system_from_yaml(self, system_dict): the parsed settings from the config file. """ - yamlSystem = YAMLSystem() + yamlSystem = WESTSystem() print("System building only off of the configuration file") # Now for the building of the system from YAML we need to use # require for these settings since they are musts. diff --git a/src/westpa/core/binning/_assign.pyx b/src/westpa/core/binning/_assign.pyx index 75a976c05..efcb71e23 100644 --- a/src/westpa/core/binning/_assign.pyx +++ b/src/westpa/core/binning/_assign.pyx @@ -71,7 +71,6 @@ cpdef rectilinear_assign(coord_t[:,:] coords, # backwards iteration needs signed values, so that the final != -1 works for idim in range(ndim-1,-1,-1): - found = 0 cval = coords[icoord,idim] boundlen = boundlens[idim] bvec = boundvecs[idim] diff --git a/src/westpa/core/propagators/executable.py b/src/westpa/core/propagators/executable.py index e28734a90..16dd404a0 100644 --- a/src/westpa/core/propagators/executable.py +++ b/src/westpa/core/propagators/executable.py @@ -52,7 +52,7 @@ def pcoord_loader(fieldname, pcoord_return_filename, destobj, single_point): pcoord.shape = expected_shape if pcoord.shape != expected_shape: raise ValueError( - 'progress coordinate data has incorrect shape {!r} [expected {!r}] Check pcoord.err or seg_logs for more information.'.format( + 'progress coordinate data has incorrect shape {!r} [expected {!r}] Check get_pcoord stderr or seg_logs for more information.'.format( pcoord.shape, expected_shape ) ) diff --git a/src/westpa/core/reweight/_reweight.pyx b/src/westpa/core/reweight/_reweight.pyx index a944a3cf9..3570a8719 100644 --- a/src/westpa/core/reweight/_reweight.pyx +++ b/src/westpa/core/reweight/_reweight.pyx @@ -4,40 +4,38 @@ from __future__ import print_function,division import cython -import numpy import numpy as np import h5py from scipy.sparse import csgraph import warnings from collections import Counter -cimport numpy cimport numpy as np cimport scipy.linalg cimport scipy.linalg.cython_lapack as cl import scipy.linalg from libc.math cimport isnan -ctypedef numpy.uint16_t index_t -ctypedef numpy.float64_t weight_t -ctypedef numpy.uint8_t bool_t -ctypedef numpy.int64_t trans_t -ctypedef numpy.uintp_t uint_t # 32 bits on 32-bit systems, 64 bits on 64-bit systems +ctypedef np.uint16_t index_t +ctypedef np.float64_t weight_t +ctypedef np.uint8_t bool_t +ctypedef np.int64_t trans_t +ctypedef np.uintp_t uint_t # 32 bits on 32-bit systems, 64 bits on 64-bit systems ctypedef unsigned short Ushort ctypedef double complex Cdouble -weight_dtype = numpy.float64 -index_dtype = numpy.uint16 -bool_dtype = numpy.bool_ -intc_dtype = numpy.intc +weight_dtype = np.float64 +index_dtype = np.uint16 +bool_dtype = np.bool_ +intc_dtype = np.intc @cython.boundscheck(False) @cython.wraparound(False) -cpdef stats_process(numpy.ndarray[index_t, ndim=2] bin_assignments, - numpy.ndarray[weight_t, ndim=1] weights, - numpy.ndarray[weight_t, ndim=2] fluxes, - numpy.ndarray[weight_t, ndim=1] populations, - numpy.ndarray[trans_t, ndim=2] trans, - numpy.ndarray[index_t, ndim=2] mask, +cpdef stats_process(np.ndarray[index_t, ndim=2] bin_assignments, + np.ndarray[weight_t, ndim=1] weights, + np.ndarray[weight_t, ndim=2] fluxes, + np.ndarray[weight_t, ndim=1] populations, + np.ndarray[trans_t, ndim=2] trans, + np.ndarray[index_t, ndim=2] mask, str interval='timepoint' ): cdef: Py_ssize_t i,k diff --git a/src/westpa/core/sim_manager.py b/src/westpa/core/sim_manager.py index 163581ccc..e5acf0b04 100644 --- a/src/westpa/core/sim_manager.py +++ b/src/westpa/core/sim_manager.py @@ -306,7 +306,7 @@ def initialize_simulation(self, basis_states, target_states, start_states, segs_ # Process start states # Unlike the above, does not create an ibstate group. - # TODO: Should it? I don't think so, if needed it can be traced back through basis_auxref + # Should it? I don't think so, if needed it can be traced back through basis_auxref # Here, we are trying to assign a state_id to the start state to be initialized, without actually # saving it to the ibstates records in any of the h5 files. It might actually be a problem diff --git a/src/westpa/core/yamlcfg.py b/src/westpa/core/yamlcfg.py index 812c9f183..6efa9b057 100644 --- a/src/westpa/core/yamlcfg.py +++ b/src/westpa/core/yamlcfg.py @@ -4,9 +4,7 @@ import os import warnings - import yaml -import numpy as np try: from yaml import CLoader as YLoader @@ -15,10 +13,6 @@ from yaml import Loader as YLoader from . import extloader -from .binning import NopMapper - -# Only needed for temporary class -import westpa NotProvided = object() @@ -302,90 +296,3 @@ def get_choice(self, key, choices, default=NotProvided, value_transform=None): ), ) return value - - -# Temporary class here -class YAMLSystem: - '''A description of the system being simulated, including the dimensionality and - data type of the progress coordinate, the number of progress coordinate entries - expected from each segment, and binning. To construct a simulation, the user must - subclass WESTSystem and set several instance variables. - - At a minimum, the user must subclass ``WESTSystem`` and override - :method:`initialize` to set the data type and dimensionality of progress - coordinate data and define a bin mapper. - - :ivar pcoord_ndim: The number of dimensions in the progress coordinate. - Defaults to 1 (i.e. a one-dimensional progress - coordinate). - :ivar pcoord_dtype: The data type of the progress coordinate, which must be - callable (e.g. ``np.float32`` and ``long`` will work, - but ``' Date: Fri, 14 Feb 2025 15:01:40 -0500 Subject: [PATCH 2/9] math.ceil and math.floor already returns int --- src/westpa/core/we_driver.py | 5 ++--- src/westpa/mclib/_mclib.pyx | 8 ++++---- src/westpa/oldtools/aframe/mcbs.py | 8 ++++---- src/westpa/oldtools/stats/mcbs.py | 4 ++-- src/westpa/tools/binning.py | 2 +- 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/westpa/core/we_driver.py b/src/westpa/core/we_driver.py index fa99a7610..67a1a0c60 100644 --- a/src/westpa/core/we_driver.py +++ b/src/westpa/core/we_driver.py @@ -281,7 +281,6 @@ def new_iteration(self, initial_states=None, target_states=None, new_weights=Non self.initial_binning = self.bin_mapper.construct_bins() self.final_binning = self.bin_mapper.construct_bins() - self.next_iter_binning = None flux_matrix = self.flux_matrix = np.zeros((nbins, nbins), dtype=np.float64) transition_matrix = self.transition_matrix = np.zeros((nbins, nbins), np.uint) @@ -532,7 +531,7 @@ def _split_by_weight(self, bin, target_count, ideal_weight): to_split = segments[weights > self.weight_split_threshold * ideal_weight] for segment in to_split: - m = int(math.ceil(segment.weight / ideal_weight)) + m = math.ceil(segment.weight / ideal_weight) bin.remove(segment) new_segments_list = self._split_walker(segment, m, bin) bin.update(new_segments_list) @@ -625,7 +624,7 @@ def _split_by_threshold(self, bin, subgroup): to_split = segments[weights > self.largest_allowed_weight] for segment in to_split: - m = int(math.ceil(segment.weight / self.largest_allowed_weight)) + m = math.ceil(segment.weight / self.largest_allowed_weight) bin.remove(segment) subgroup.remove(segment) new_segments_list = self._split_walker(segment, m, bin) diff --git a/src/westpa/mclib/_mclib.pyx b/src/westpa/mclib/_mclib.pyx index b57bf1777..94773eaae 100644 --- a/src/westpa/mclib/_mclib.pyx +++ b/src/westpa/mclib/_mclib.pyx @@ -151,8 +151,8 @@ cpdef mcbs_ci(dataset, estimator, alpha, dlen, n_sets=None, args=None, kwargs=No del indices f_synth_sorted = sort(f_synth) - lbi = int(math.floor(n_sets*alpha/2.0)) - ubi = int(math.ceil(n_sets*(1-alpha/2.0))) + lbi = math.floor(n_sets*alpha/2.0) + ubi = math.ceil(n_sets*(1-alpha/2.0)) lb = f_synth_sorted[lbi] ub = f_synth_sorted[ubi] sterr = numpy.std(f_synth_sorted) @@ -200,8 +200,8 @@ cpdef mcbs_correltime_small(dataset, alpha, n_sets): raise ValueError('alpha ({}) > 0.5'.format(alpha)) dlen = len(dataset) - lbi = int(math.floor(n_sets*alpha/2.0)) - ubi = int(math.ceil(n_sets*(1-alpha/2.0))) + lbi = math.floor(n_sets*alpha/2.0) + ubi = math.ceil(n_sets*(1-alpha/2.0)) synth_sets = numpy.empty((n_sets,)+dataset.shape, dtype=dataset.dtype) synth_acf_elems = numpy.empty((n_sets,), numpy.float64) diff --git a/src/westpa/oldtools/aframe/mcbs.py b/src/westpa/oldtools/aframe/mcbs.py index b7f3e1b04..f0d25abe4 100644 --- a/src/westpa/oldtools/aframe/mcbs.py +++ b/src/westpa/oldtools/aframe/mcbs.py @@ -55,7 +55,7 @@ def process_args(self, args, upcall=True): self.mcbs_alpha = 1 - args.mcbs_confidence self.mcbs_nsets = args.mcbs_size if args.mcbs_nsets else min(1000, calc_mcbs_nsets(self.mcbs_alpha)) self.mcbs_display_confidence = '{:.{cp}f}'.format( - 100 * args.mcbs_confidence, cp=-int(math.floor(math.log10(self.mcbs_alpha))) - 2 + 100 * args.mcbs_confidence, cp=-math.floor(math.log10(self.mcbs_alpha)) - 2 ) westpa.rc.pstatus( 'Using bootstrap of {:d} sets to calculate {:s}% confidence interval (alpha={:g}).'.format( @@ -90,7 +90,7 @@ def calc_mcbs_nsets(alpha): def calc_ci_bound_indices(n_sets, alpha): - return (int(math.floor(n_sets * alpha / 2)), int(math.ceil(n_sets * (1 - alpha / 2)))) + return (math.floor(n_sets * alpha / 2), math.ceil(n_sets * (1 - alpha / 2))) def bootstrap_ci_ll(estimator, data, alpha, n_sets, storage, sort, eargs=(), ekwargs={}, fhat=None): @@ -112,8 +112,8 @@ def bootstrap_ci_ll(estimator, data, alpha, n_sets, storage, sort, eargs=(), ekw storage[iset] = estimator(data[indices], *eargs, **ekwargs) synth_sorted = sort(storage) - lbi = int(math.floor(n_sets * alpha / 2)) - ubi = int(math.ceil(n_sets * (1 - alpha / 2))) + lbi = math.floor(n_sets * alpha / 2) + ubi = math.ceil(n_sets * (1 - alpha / 2)) lb = synth_sorted[lbi] ub = synth_sorted[ubi] diff --git a/src/westpa/oldtools/stats/mcbs.py b/src/westpa/oldtools/stats/mcbs.py index 89f3e61aa..f0b0ab9d8 100644 --- a/src/westpa/oldtools/stats/mcbs.py +++ b/src/westpa/oldtools/stats/mcbs.py @@ -81,8 +81,8 @@ def bootstrap_ci(estimator, data, alpha, n_sets=None, args=(), kwargs={}, sort=m f_synth[i] = estimator(data[indices], *args, **kwargs) f_synth_sorted = sort(f_synth) - lbi = int(math.floor(n_sets * alpha / 2)) - ubi = int(math.ceil(n_sets * (1 - alpha / 2))) + lbi = math.floor(n_sets * alpha / 2) + ubi = math.ceil(n_sets * (1 - alpha / 2)) lb = f_synth_sorted[lbi] ub = f_synth_sorted[ubi] diff --git a/src/westpa/tools/binning.py b/src/westpa/tools/binning.py index 668d18ef6..867a87446 100644 --- a/src/westpa/tools/binning.py +++ b/src/westpa/tools/binning.py @@ -144,7 +144,7 @@ def write_bin_info(mapper, assignments, weights, n_target_states, outfile=sys.st min_seg_weight = weights.min() max_seg_weight = weights.max() - ndec = int(math.ceil(-math.log10(1 / n_active))) + ndec = math.ceil(-math.log10(1 / n_active)) outfile.write('{:d} segments\n'.format(len(weights))) outfile.write( From fdbb70ae58c234eb112dd5fa9952f5fad54da89c Mon Sep 17 00:00:00 2001 From: "Jeremy M. G. Leung" Date: Fri, 14 Feb 2025 15:04:20 -0500 Subject: [PATCH 3/9] change tests import --- tests/refs/odld/odld_system.py | 4 +--- tests/test_yamlfe.py | 5 +++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/refs/odld/odld_system.py b/tests/refs/odld/odld_system.py index e769b70b4..3c8ee0df2 100644 --- a/tests/refs/odld/odld_system.py +++ b/tests/refs/odld/odld_system.py @@ -1,11 +1,9 @@ import numpy as np from numpy.random import Generator, MT19937 -from westpa.core.binning import RectilinearBinMapper +from westpa.core.binning import RectilinearBinMapper, MABBinMapper, BinlessMapper from westpa.core.propagators import WESTPropagator from westpa.core.systems import WESTSystem -from westpa.core.binning.mab import MABBinMapper -from westpa.core.binning.binless import BinlessMapper PI = np.pi diff --git a/tests/test_yamlfe.py b/tests/test_yamlfe.py index c6eae389d..5ceb92fd5 100644 --- a/tests/test_yamlfe.py +++ b/tests/test_yamlfe.py @@ -1,11 +1,12 @@ import numpy as np import westpa -import westpa.core.yamlcfg as ycf +from westpa.core.yamlcfg import ycf +from westpa.core.systems import WESTSystem from westpa.core.binning import RectilinearBinMapper -class TESTSystem(ycf.YAMLSystem): +class TESTSystem(WESTSystem): def initialize(self): self.pcoord_ndim = 1 self.pcoord_dtype = np.float32 From c6817c23e7d93c647960fefc52efe942405f8309 Mon Sep 17 00:00:00 2001 From: "Jeremy M. G. Leung" Date: Fri, 14 Feb 2025 15:18:41 -0500 Subject: [PATCH 4/9] fix faulty import yamlcfg --- tests/test_yamlfe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_yamlfe.py b/tests/test_yamlfe.py index 5ceb92fd5..81d178ece 100644 --- a/tests/test_yamlfe.py +++ b/tests/test_yamlfe.py @@ -1,7 +1,7 @@ import numpy as np import westpa -from westpa.core.yamlcfg import ycf +import westpa.core.yamlcfg as ycf from westpa.core.systems import WESTSystem from westpa.core.binning import RectilinearBinMapper From ece683d7d47e986ff0db7750adae1cccce729dc8 Mon Sep 17 00:00:00 2001 From: "Jeremy M. G. Leung" Date: Wed, 19 Feb 2025 15:16:55 -0500 Subject: [PATCH 5/9] first tests for westpa.tools.binning --- tests/test_binningfe.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/test_binningfe.py diff --git a/tests/test_binningfe.py b/tests/test_binningfe.py new file mode 100644 index 000000000..92b0fe3de --- /dev/null +++ b/tests/test_binningfe.py @@ -0,0 +1,13 @@ +import numpy as np + +from westpa.tools.binning import mapper_from_expr + + +class TestBinParsing: + def testMapperFromExpr(self): + mapper = mapper_from_expr(r'[[-inf, 0, 15, np.inf]]') + + assert mapper.ndim == 1 + assert mapper.nbins == 3 + assert np.array_equal(mapper._boundaries, [np.array([-float('inf'), 0, 15, float('inf')], dtype=np.float32)]) + assert mapper.labels == ['[(-inf, 0.0)]', '[(0.0, 15.0)]', '[(15.0, inf)]'] From 99dbc8204b5eea7cf8b0bcad19878840e093788b Mon Sep 17 00:00:00 2001 From: "Jeremy M. G. Leung" Date: Mon, 5 May 2025 12:04:14 -0400 Subject: [PATCH 6/9] add new YAMLSystem tests --- tests/test_yamlfe.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_yamlfe.py b/tests/test_yamlfe.py index 81d178ece..0dad31c25 100644 --- a/tests/test_yamlfe.py +++ b/tests/test_yamlfe.py @@ -1,4 +1,6 @@ +import pytest import numpy as np +from numpy.testing import assert_array_equal import westpa import westpa.core.yamlcfg as ycf @@ -112,3 +114,20 @@ def testYAMLFEConfig(self): rc.config['west', 'propagation', 'max_total_iteration'] = 1000 assert rc.config['west', 'propagation', 'max_total_iteration'] == 1000 + + def testSystemDefaults(self): + # First the objects that will be used for testing + testSystem = WESTSystem() + + with pytest.raises(NotImplementedError): + testSystem.new_region_set() + + # Test that the new pcoord array is of the correct shape + test_zero = np.zeros((2, 1), np.float32) + pcoord_array = testSystem.new_pcoord_array() + assert testSystem.pcoord_len == 2 + assert_array_equal(test_zero, pcoord_array) + + testSystem.initialize() + testSystem.prepare_run() + testSystem.finalize_run() From 641f4925e45c04d4c17aecada9a5b10f8676195f Mon Sep 17 00:00:00 2001 From: "Jeremy M. G. Leung" Date: Mon, 5 May 2025 15:55:35 -0400 Subject: [PATCH 7/9] tests for split/merge by threshold --- tests/test_we_driver.py | 46 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/test_we_driver.py b/tests/test_we_driver.py index 7ba4aeaca..7d523c35f 100644 --- a/tests/test_we_driver.py +++ b/tests/test_we_driver.py @@ -251,5 +251,51 @@ def test_populate_initial(self): for tcount in range(30, 60): yield self.check_populate_initial, prob, tcount + def test_merge_by_threshold(self): + # Create the segments + segments = [self.segment(0.0, 0.5, weight=0.25) for i in range(4)] + self.we_driver.smallest_allowed_weight = 0.5 + + # Create the bins + self.we_driver.new_iteration() + self.we_driver.assign(segments) + self.we_driver.construct_next() + + for ibin, bin in enumerate(self.we_driver.next_iter_binning): + subgroup = self.we_driver.subgroup_function(self.we_driver, ibin, **self.we_driver.subgroup_function_kwargs) + + if len(bin) > 0: + # Occupied bin + self.we_driver._merge_by_threshold(bin, subgroup[ibin]) + + assert len(self.we_driver.next_iter_binning[ibin]) == 1 + assert np.allclose([seg.weight for seg in self.we_driver.next_iter_binning[0]], [1]) + else: + # Unoccupied bins + assert len(self.we_driver.next_iter_binning[ibin]) == 0 + + def test_split_by_threshold(self): + # Create the segments + segments = [self.segment(0.0, 0.5, weight=1)] + self.we_driver.largest_allowed_weight = 0.125 + + # Create the bins + self.we_driver.new_iteration() + self.we_driver.assign(segments) + self.we_driver.construct_next() + + for ibin, bin in enumerate(self.we_driver.next_iter_binning): + subgroup = self.we_driver.subgroup_function(self.we_driver, ibin, **self.we_driver.subgroup_function_kwargs) + + if len(bin) > 0: + # Occupied bin + self.we_driver._split_by_threshold(bin, subgroup[ibin]) + + assert len(self.we_driver.next_iter_binning[ibin]) == 8 + assert np.allclose([seg.weight for seg in self.we_driver.next_iter_binning[0]], [0.125 for _i in range(8)]) + else: + # Unoccupied bins + assert len(self.we_driver.next_iter_binning[ibin]) == 0 + # TODO: add test for seeding the flux matrix based on recycling # TODO: add test for split after merge in adjust count From 83dff7111989635d3ca16f145fe028a4ff700d26 Mon Sep 17 00:00:00 2001 From: "Jeremy M. G. Leung" Date: Mon, 5 May 2025 17:07:12 -0400 Subject: [PATCH 8/9] some mundane value/type testing for yamlcfg --- tests/test_yamlfe.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_yamlfe.py b/tests/test_yamlfe.py index 0dad31c25..040c0dd8d 100644 --- a/tests/test_yamlfe.py +++ b/tests/test_yamlfe.py @@ -131,3 +131,16 @@ def testSystemDefaults(self): testSystem.initialize() testSystem.prepare_run() testSystem.finalize_run() + + +class TestYAMLConfig: + def test_dubious_config_entry(self): + with pytest.warns(ycf.ConfigValueWarning): + ycf.warn_dubious_config_entry('1', 1, expected_type=str) + ycf.warn_dubious_config_entry('1', 1) + + def test_check_bool(self): + with pytest.warns(ycf.ConfigValueWarning): + ycf.check_bool(100, action='warn') + with pytest.raises(ValueError): + ycf.check_bool(100, action='raise') From d72a92878630f7a3f4396c0042b53d3879c0a2ab Mon Sep 17 00:00:00 2001 From: "Jeremy M. G. Leung" Date: Tue, 6 May 2025 12:40:34 -0400 Subject: [PATCH 9/9] set next_iter_binning to None when creating new iteration --- src/westpa/core/we_driver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/westpa/core/we_driver.py b/src/westpa/core/we_driver.py index 67a1a0c60..d843c4d3b 100644 --- a/src/westpa/core/we_driver.py +++ b/src/westpa/core/we_driver.py @@ -281,6 +281,7 @@ def new_iteration(self, initial_states=None, target_states=None, new_weights=Non self.initial_binning = self.bin_mapper.construct_bins() self.final_binning = self.bin_mapper.construct_bins() + self.next_iter_binning = None flux_matrix = self.flux_matrix = np.zeros((nbins, nbins), dtype=np.float64) transition_matrix = self.transition_matrix = np.zeros((nbins, nbins), np.uint)