From 9d9a980d68840c91617cce86012aeffc073945d2 Mon Sep 17 00:00:00 2001 From: jrzaurin Date: Sun, 27 Sep 2020 19:28:28 +0200 Subject: [PATCH 01/10] Added a condition to account for 2D images (images with no color channels). This way we replicate exactly what ToTensor does within the WideDeepDataset class --- pytorch_widedeep/models/_wd_dataset.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pytorch_widedeep/models/_wd_dataset.py b/pytorch_widedeep/models/_wd_dataset.py index aa5dc6e4..a3dcd2f0 100644 --- a/pytorch_widedeep/models/_wd_dataset.py +++ b/pytorch_widedeep/models/_wd_dataset.py @@ -68,6 +68,8 @@ def __getitem__(self, idx: int): # then we need to replicate what Tensor() does -> transpose axis # and normalize if necessary if not self.transforms or "ToTensor" not in self.transforms_names: + if xdi.ndim == 2: + xdi = xdi[:, :, None] xdi = xdi.transpose(2, 0, 1) if "int" in str(xdi.dtype): xdi = (xdi / xdi.max()).astype("float32") From 5ee5fbe84288f29aa4a5739c2ea326116c43dc22 Mon Sep 17 00:00:00 2001 From: jrzaurin Date: Sun, 27 Sep 2020 19:40:41 +0200 Subject: [PATCH 02/10] check travis inconsistency by simply removing a commented line --- examples/adult_script.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/adult_script.py b/examples/adult_script.py index 36e7733b..00930020 100644 --- a/examples/adult_script.py +++ b/examples/adult_script.py @@ -113,4 +113,3 @@ # torch.save(model.state_dict(), "model_weights/model_dict.t") # model = WideDeep(wide=wide, deepdense=deepdense) # model.load_state_dict(torch.load("model_weights/model_dict.t")) - # # From 9a6a24d3e2693cb91a5ef3c2b8a9a71236f0757e Mon Sep 17 00:00:00 2001 From: jrzaurin Date: Fri, 27 Nov 2020 23:56:13 +0100 Subject: [PATCH 03/10] all started by trying to add a line so that the builder accepted 2D images (or in general images of dim different than 3. But it ended up by adding functionalities so that each individual component (wide, deepdense, deeptext and deepimage) can be used individually --- examples/adult_script.py | 1 + pytorch_widedeep/models/_wd_dataset.py | 25 +- pytorch_widedeep/models/wide_deep.py | 281 ++++++++---------- setup.py | 2 +- tests/test_model_components/test_wide_deep.py | 2 +- .../test_data_inputs.py | 46 +++ 6 files changed, 194 insertions(+), 163 deletions(-) diff --git a/examples/adult_script.py b/examples/adult_script.py index 00930020..36e7733b 100644 --- a/examples/adult_script.py +++ b/examples/adult_script.py @@ -113,3 +113,4 @@ # torch.save(model.state_dict(), "model_weights/model_dict.t") # model = WideDeep(wide=wide, deepdense=deepdense) # model.load_state_dict(torch.load("model_weights/model_dict.t")) + # # diff --git a/pytorch_widedeep/models/_wd_dataset.py b/pytorch_widedeep/models/_wd_dataset.py index a3dcd2f0..447877bb 100644 --- a/pytorch_widedeep/models/_wd_dataset.py +++ b/pytorch_widedeep/models/_wd_dataset.py @@ -27,11 +27,11 @@ class WideDeepDataset(Dataset): def __init__( self, - X_wide: np.ndarray, - X_deep: np.ndarray, - target: Optional[np.ndarray] = None, + X_wide: Optional[np.ndarray] = None, + X_deep: Optional[np.ndarray] = None, X_text: Optional[np.ndarray] = None, X_img: Optional[np.ndarray] = None, + target: Optional[np.ndarray] = None, transforms: Optional[Any] = None, ): @@ -48,10 +48,12 @@ def __init__( self.transforms_names = [] self.Y = target - def __getitem__(self, idx: int): - # X_wide and X_deep are assumed to be *always* present - X = Bunch(wide=self.X_wide[idx]) - X.deepdense = self.X_deep[idx] + def __getitem__(self, idx: int): # noqa: C901 + X = Bunch() + if self.X_wide is not None: + X.wide = self.X_wide[idx] + if self.X_deep is not None: + X.deepdense = self.X_deep[idx] if self.X_text is not None: X.deeptext = self.X_text[idx] if self.X_img is not None: @@ -89,4 +91,11 @@ def __getitem__(self, idx: int): return X def __len__(self): - return len(self.X_deep) + if self.X_wide is not None: + return len(self.X_wide) + if self.X_deep is not None: + return len(self.X_deep) + if self.X_text is not None: + return len(self.X_text) + if self.X_img is not None: + return len(self.X_img) diff --git a/pytorch_widedeep/models/wide_deep.py b/pytorch_widedeep/models/wide_deep.py index 70ce529b..84757d6f 100644 --- a/pytorch_widedeep/models/wide_deep.py +++ b/pytorch_widedeep/models/wide_deep.py @@ -1,5 +1,4 @@ import os -import warnings import numpy as np import torch @@ -21,6 +20,9 @@ from ._multiple_transforms import MultipleTransforms from ._multiple_lr_scheduler import MultipleLRScheduler +# import warnings + + n_cpus = os.cpu_count() use_cuda = torch.cuda.is_available() @@ -104,37 +106,24 @@ class WideDeep(nn.Module): """ - def __init__( + def __init__( # noqa: C901 self, - wide: nn.Module, - deepdense: nn.Module, - pred_dim: int = 1, + wide: Optional[nn.Module] = None, + deepdense: Optional[nn.Module] = None, deeptext: Optional[nn.Module] = None, deepimage: Optional[nn.Module] = None, deephead: Optional[nn.Module] = None, head_layers: Optional[List[int]] = None, head_dropout: Optional[List] = None, head_batchnorm: Optional[bool] = None, + pred_dim: int = 1, ): super(WideDeep, self).__init__() - # check that model components have the required output_dim attribute - if not hasattr(deepdense, "output_dim"): - raise AttributeError( - "deepdense model must have an 'output_dim' attribute. " - "See pytorch-widedeep.models.deep_dense.DeepDense" - ) - if deeptext is not None and not hasattr(deeptext, "output_dim"): - raise AttributeError( - "deeptext model must have an 'output_dim' attribute. " - "See pytorch-widedeep.models.deep_dense.DeepText" - ) - if deepimage is not None and not hasattr(deepimage, "output_dim"): - raise AttributeError( - "deepimage model must have an 'output_dim' attribute. " - "See pytorch-widedeep.models.deep_dense.DeepText" - ) + self._check_params( + deepdense, deeptext, deepimage, deephead, head_layers, head_dropout + ) # required as attribute just in case we pass a deephead self.pred_dim = pred_dim @@ -146,17 +135,11 @@ def __init__( self.deepimage = deepimage self.deephead = deephead - if deephead is not None and head_layers is not None: - warnings.simplefilter("module") - warnings.warn( - "both 'deephead' and 'head_layers' are not None." - "'deephead' takes priority and will be used", - UserWarning, - ) - if self.deephead is None: if head_layers is not None: - input_dim: int = self.deepdense.output_dim # type:ignore + input_dim = 0 + if self.deepdense is not None: + input_dim += self.deepdense.output_dim # type:ignore if self.deeptext is not None: input_dim += self.deeptext.output_dim # type:ignore if self.deepimage is not None: @@ -179,9 +162,10 @@ def __init__( "head_out", nn.Linear(head_layers[-1], pred_dim) ) else: - self.deepdense = nn.Sequential( - self.deepdense, nn.Linear(self.deepdense.output_dim, pred_dim) # type: ignore - ) + if self.deepdense is not None: + self.deepdense = nn.Sequential( + self.deepdense, nn.Linear(self.deepdense.output_dim, pred_dim) # type: ignore + ) if self.deeptext is not None: self.deeptext = nn.Sequential( self.deeptext, nn.Linear(self.deeptext.output_dim, pred_dim) # type: ignore @@ -190,34 +174,42 @@ def __init__( self.deepimage = nn.Sequential( self.deepimage, nn.Linear(self.deepimage.output_dim, pred_dim) # type: ignore ) - else: - self.deephead + # else: + # self.deephead - def forward(self, X: Dict[str, Tensor]) -> Tensor: # type: ignore + def forward(self, X: Dict[str, Tensor]) -> Tensor: # type: ignore # noqa: C901 # Wide output: direct connection to the output neuron(s) - out = self.wide(X["wide"]) + if self.wide is not None: + out = self.wide(X["wide"]) + else: + batch_size = X[list(X.keys())[0]].size(0) + out = torch.zeros(batch_size, self.pred_dim) # Deep output: either connected directly to the output neuron(s) or # passed through a head first if self.deephead: - deepside = self.deepdense(X["deepdense"]) + if self.deepdense is not None: + deepside = self.deepdense(X["deepdense"]) + else: + deepside = torch.FloatTensor() if self.deeptext is not None: deepside = torch.cat([deepside, self.deeptext(X["deeptext"])], axis=1) # type: ignore if self.deepimage is not None: deepside = torch.cat([deepside, self.deepimage(X["deepimage"])], axis=1) # type: ignore deephead_out = self.deephead(deepside) deepside_out = nn.Linear(deephead_out.size(1), self.pred_dim)(deephead_out) - return out.add(deepside_out) + return out.add_(deepside_out) else: - out.add(self.deepdense(X["deepdense"])) + if self.deepdense is not None: + out.add_(self.deepdense(X["deepdense"])) if self.deeptext is not None: - out.add(self.deeptext(X["deeptext"])) + out.add_(self.deeptext(X["deeptext"])) if self.deepimage is not None: - out.add(self.deepimage(X["deepimage"])) + out.add_(self.deepimage(X["deepimage"])) return out - def compile( + def compile( # noqa: C901 self, method: str, optimizers: Optional[Union[Optimizer, Dict[str, Optimizer]]] = None, @@ -372,14 +364,7 @@ def compile( if optimizers is not None: if isinstance(optimizers, Optimizer): self.optimizer: Union[Optimizer, MultipleOptimizer] = optimizers - elif isinstance(optimizers, Dict) and len(optimizers) == 1: - raise ValueError( - "The dictionary of optimizers must contain one item per model component, " - "i.e. at least two for the 'wide' and 'deepdense' components. Otherwise " - "pass one Optimizer object that will be used for all components" - "i.e. optimizers = torch.optim.Adam(model.parameters())" - ) - elif len(optimizers) > 1: + elif isinstance(optimizers, Dict): opt_names = list(optimizers.keys()) mod_names = [n for n, c in self.named_children()] for mn in mod_names: @@ -430,7 +415,7 @@ def compile( if use_cuda: self.cuda() - def fit( + def fit( # noqa: C901 self, X_wide: Optional[np.ndarray] = None, X_deep: Optional[np.ndarray] = None, @@ -590,13 +575,6 @@ def fit( """ - if X_train is None and (X_wide is None or X_deep is None or target is None): - raise ValueError( - "Training data is missing. Either a dictionary (X_train) with " - "the training dataset or at least 3 arrays (X_wide, X_deep, " - "target) must be passed to the fit method" - ) - self.batch_size = batch_size train_set, eval_set = self._train_val_split( X_wide, X_deep, X_text, X_img, X_train, X_val, val_split, target @@ -807,7 +785,7 @@ def _loss_fn(self, y_pred: Tensor, y_true: Tensor) -> Tensor: # type: ignore if self.method == "multiclass": return F.cross_entropy(y_pred, y_true, weight=self.class_weight) - def _train_val_split( + def _train_val_split( # noqa: C901 self, X_wide: Optional[np.ndarray] = None, X_deep: Optional[np.ndarray] = None, @@ -835,100 +813,51 @@ def _train_val_split( :obj:`torch.utils.data.DataLoader`. See :class:`pytorch_widedeep.models._wd_dataset` """ - #  Without validation - if X_val is None and val_split is None: - # if a train dictionary is passed, check if text and image datasets - # are present and instantiate the WideDeepDataset class - if X_train is not None: - X_wide, X_deep, target = ( - X_train["X_wide"], - X_train["X_deep"], - X_train["target"], - ) - if "X_text" in X_train.keys(): - X_text = X_train["X_text"] - if "X_img" in X_train.keys(): - X_img = X_train["X_img"] - X_train = {"X_wide": X_wide, "X_deep": X_deep, "target": target} - try: - X_train.update({"X_text": X_text}) - except: - pass - try: - X_train.update({"X_img": X_img}) - except: - pass + + if X_val is not None: + assert ( + X_train is not None + ), "if the validation set is passed as a dictionary, the training set must also be a dictionary" train_set = WideDeepDataset(**X_train, transforms=self.transforms) # type: ignore - eval_set = None - #  With validation - else: - if X_val is not None: - # if a validation dictionary is passed, then if not train - # dictionary is passed we build it with the input arrays - # (either the dictionary or the arrays must be passed) - if X_train is None: - X_train = {"X_wide": X_wide, "X_deep": X_deep, "target": target} - if X_text is not None: - X_train.update({"X_text": X_text}) - if X_img is not None: - X_train.update({"X_img": X_img}) - else: - # if a train dictionary is passed, check if text and image - # datasets are present. The train/val split using val_split - if X_train is not None: - X_wide, X_deep, target = ( - X_train["X_wide"], - X_train["X_deep"], - X_train["target"], - ) - if "X_text" in X_train.keys(): - X_text = X_train["X_text"] - if "X_img" in X_train.keys(): - X_img = X_train["X_img"] - ( - X_tr_wide, - X_val_wide, - X_tr_deep, - X_val_deep, - y_tr, - y_val, - ) = train_test_split( - X_wide, - X_deep, - target, - test_size=val_split, - random_state=self.seed, - stratify=target if self.method != "regression" else None, + eval_set = WideDeepDataset(**X_val, transforms=self.transforms) # type: ignore + elif val_split is not None: + if not X_train: + X_train = self._build_train_dict(X_wide, X_deep, X_text, X_img, target) + y_tr, y_val, idx_tr, idx_val = train_test_split( + X_train["target"], + np.arange(len(X_train["target"])), + test_size=val_split, + stratify=X_train["target"] if self.method != "regression" else None, + ) + X_tr, X_val = {"target": y_tr}, {"target": y_val} + if "X_wide" in X_train.keys(): + X_tr["X_wide"], X_val["X_wide"] = ( + X_train["X_wide"][idx_tr], + X_train["X_wide"][idx_val], ) - X_train = {"X_wide": X_tr_wide, "X_deep": X_tr_deep, "target": y_tr} - X_val = {"X_wide": X_val_wide, "X_deep": X_val_deep, "target": y_val} - try: - X_tr_text, X_val_text = train_test_split( - X_text, - test_size=val_split, - random_state=self.seed, - stratify=target if self.method != "regression" else None, - ) - X_train.update({"X_text": X_tr_text}), X_val.update( - {"X_text": X_val_text} - ) - except: - pass - try: - X_tr_img, X_val_img = train_test_split( - X_img, - test_size=val_split, - random_state=self.seed, - stratify=target if self.method != "regression" else None, - ) - X_train.update({"X_img": X_tr_img}), X_val.update( - {"X_img": X_val_img} - ) - except: - pass - # At this point the X_train and X_val dictionaries have been built - train_set = WideDeepDataset(**X_train, transforms=self.transforms) # type: ignore + if "X_deep" in X_train.keys(): + X_tr["X_deep"], X_val["X_deep"] = ( + X_train["X_deep"][idx_tr], + X_train["X_deep"][idx_val], + ) + if "X_text" in X_train.keys(): + X_tr["X_text"], X_val["X_text"] = ( + X_train["X_text"][idx_tr], + X_train["X_text"][idx_val], + ) + if "X_img" in X_train.keys(): + X_tr["X_img"], X_val["X_img"] = ( + X_train["X_img"][idx_tr], + X_train["X_img"][idx_val], + ) + train_set = WideDeepDataset(**X_tr, transforms=self.transforms) # type: ignore eval_set = WideDeepDataset(**X_val, transforms=self.transforms) # type: ignore + else: + if not X_train: + X_train = self._build_train_dict(X_wide, X_deep, X_text, X_img, target) + train_set = WideDeepDataset(**X_train, transforms=self.transforms) # type: ignore + eval_set = None + return train_set, eval_set def _warm_up( @@ -981,7 +910,7 @@ def _warm_up( else: warmer.warm_all(self.deepimage, "deepimage", loader, n_epochs, max_lr) - def _lr_scheduler_step(self, step_location: str): + def _lr_scheduler_step(self, step_location: str): # noqa: C901 r""" Function to execute the learning rate schedulers steps. If the lr_scheduler is Cyclic (i.e. CyclicLR or OneCycleLR), the step @@ -1095,7 +1024,7 @@ def _predict( num_workers=n_cpus, shuffle=False, ) - test_steps = (len(test_loader.dataset) // test_loader.batch_size) + 1 + test_steps = (len(test_loader.dataset) // test_loader.batch_size) + 1 # type: ignore[arg-type] self.eval() preds_l = [] @@ -1113,3 +1042,49 @@ def _predict( preds_l.append(preds) self.train() return preds_l + + @staticmethod + def _build_train_dict(X_wide, X_deep, X_text, X_img, target): + X_train = {"target": target} + if X_wide is not None: + X_train["X_wide"] = X_wide + if X_deep is not None: + X_train["X_deep"] = X_deep + if X_text is not None: + X_train["X_text"] = X_text + if X_img is not None: + X_train["X_img"] = X_img + return X_train + + @staticmethod + def _check_params( + deepdense, deeptext, deepimage, deephead, head_layers, head_dropout + ): + + if deepdense is not None and not hasattr(deepdense, "output_dim"): + raise AttributeError( + "deepdense model must have an 'output_dim' attribute. " + "See pytorch-widedeep.models.deep_dense.DeepText" + ) + if deeptext is not None and not hasattr(deeptext, "output_dim"): + raise AttributeError( + "deeptext model must have an 'output_dim' attribute. " + "See pytorch-widedeep.models.deep_dense.DeepText" + ) + if deepimage is not None and not hasattr(deepimage, "output_dim"): + raise AttributeError( + "deepimage model must have an 'output_dim' attribute. " + "See pytorch-widedeep.models.deep_dense.DeepText" + ) + if deephead is not None and head_layers is not None: + raise ValueError( + "both 'deephead' and 'head_layers' are not None. Use one of the other, but not both" + ) + if head_layers is not None and not deepdense and not deeptext and not deepimage: + raise ValueError( + "if 'head_layers' is not None, at least one deep component must be used" + ) + if head_layers is not None and head_dropout is not None: + assert len(head_layers) == len( + head_dropout + ), "'head_layers' and 'head_dropout' must have the same length" diff --git a/setup.py b/setup.py index 9f9d8702..33619d0e 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ ] extras["quality"] = [ "black", - "isort @ git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort", + "isort", "flake8", ] diff --git a/tests/test_model_components/test_wide_deep.py b/tests/test_model_components/test_wide_deep.py index 5a6fc249..1e822862 100644 --- a/tests/test_model_components/test_wide_deep.py +++ b/tests/test_model_components/test_wide_deep.py @@ -55,7 +55,7 @@ def test_history_callback(deepcomponent, component_name): def test_deephead_and_head_layers(): deephead = nn.Sequential(nn.Linear(32, 16), nn.Linear(16, 8)) - with pytest.warns(UserWarning): + with pytest.raises(ValueError): model = WideDeep( # noqa: F841 wide=wide, deepdense=deepdense, head_layers=[16, 8], deephead=deephead ) diff --git a/tests/test_model_functioning/test_data_inputs.py b/tests/test_model_functioning/test_data_inputs.py index da484fff..89f7021e 100644 --- a/tests/test_model_functioning/test_data_inputs.py +++ b/tests/test_model_functioning/test_data_inputs.py @@ -266,3 +266,49 @@ def test_widedeep_inputs( model.history.epoch[0] == nepoch and model.history._history["train_loss"] is not null ) + + +@pytest.mark.parametrize( + "X_wide, X_deep, X_text, X_img, X_train, X_val, target", + [ + ( + X_wide, + X_deep, + X_text, + X_img, + None, + { + "X_wide": X_wide_val, + "X_deep": X_deep_val, + "X_text": X_text_val, + "X_img": X_img_val, + "target": y_val, + }, + target, + ), + ], +) +def test_xtrain_xval_assertion( + X_wide, + X_deep, + X_text, + X_img, + X_train, + X_val, + target, +): + model = WideDeep( + wide=wide, deepdense=deepdense, deeptext=deeptext, deepimage=deepimage + ) + model.compile(method="binary", verbose=0) + with pytest.raises(AssertionError): + model.fit( + X_wide=X_wide, + X_deep=X_deep, + X_text=X_text, + X_img=X_img, + X_train=X_train, + X_val=X_val, + target=target, + batch_size=16, + ) From 72e961b45a0e01d63cd53bda0121ef349bd985ff Mon Sep 17 00:00:00 2001 From: jrzaurin Date: Tue, 1 Dec 2020 00:42:47 +0100 Subject: [PATCH 04/10] Added more tests in test_data_inputs --- pytorch_widedeep/models/wide_deep.py | 29 +++-- .../test_data_inputs.py | 102 +++++++++++++++++- tests/test_warm_up/test_warm_up_routines.py | 4 +- 3 files changed, 124 insertions(+), 11 deletions(-) diff --git a/pytorch_widedeep/models/wide_deep.py b/pytorch_widedeep/models/wide_deep.py index 84757d6f..873008fc 100644 --- a/pytorch_widedeep/models/wide_deep.py +++ b/pytorch_widedeep/models/wide_deep.py @@ -667,8 +667,8 @@ def fit( # noqa: C901 def predict( self, - X_wide: np.ndarray, - X_deep: np.ndarray, + X_wide: Optional[np.ndarray] = None, + X_deep: Optional[np.ndarray] = None, X_text: Optional[np.ndarray] = None, X_img: Optional[np.ndarray] = None, X_test: Optional[Dict[str, np.ndarray]] = None, @@ -711,8 +711,8 @@ def predict( def predict_proba( self, - X_wide: np.ndarray, - X_deep: np.ndarray, + X_wide: Optional[np.ndarray] = None, + X_deep: Optional[np.ndarray] = None, X_text: Optional[np.ndarray] = None, X_img: Optional[np.ndarray] = None, X_test: Optional[Dict[str, np.ndarray]] = None, @@ -998,8 +998,8 @@ def _validation_step(self, data: Dict[str, Tensor], target: Tensor, batch_idx: i def _predict( self, - X_wide: np.ndarray, - X_deep: np.ndarray, + X_wide: Optional[np.ndarray] = None, + X_deep: Optional[np.ndarray] = None, X_text: Optional[np.ndarray] = None, X_img: Optional[np.ndarray] = None, X_test: Optional[Dict[str, np.ndarray]] = None, @@ -1056,7 +1056,7 @@ def _build_train_dict(X_wide, X_deep, X_text, X_img, target): X_train["X_img"] = X_img return X_train - @staticmethod + @staticmethod # noqa: C901 def _check_params( deepdense, deeptext, deepimage, deephead, head_layers, head_dropout ): @@ -1088,3 +1088,18 @@ def _check_params( assert len(head_layers) == len( head_dropout ), "'head_layers' and 'head_dropout' must have the same length" + if deephead is not None: + deephead_inp_feat = next(deephead.parameters()).size(1) + output_dim = 0 + if deepdense is not None: + output_dim += deepdense.output_dim + if deeptext is not None: + output_dim += deeptext.output_dim + if deepimage is not None: + output_dim += deepimage.output_dim + assert deephead_inp_feat == output_dim, ( + "if a custom 'deephead' is used its input features ({}) must be equal to " + "the output features of the deep component ({})".format( + deephead_inp_feat, output_dim + ) + ) diff --git a/tests/test_model_functioning/test_data_inputs.py b/tests/test_model_functioning/test_data_inputs.py index 89f7021e..37f2f8b2 100644 --- a/tests/test_model_functioning/test_data_inputs.py +++ b/tests/test_model_functioning/test_data_inputs.py @@ -3,6 +3,7 @@ import numpy as np import pytest from torchvision.transforms import ToTensor, Normalize +from torch import nn from sklearn.model_selection import train_test_split from pytorch_widedeep.models import ( @@ -67,11 +68,16 @@ transforms1 = [ToTensor, Normalize(mean=mean, std=std)] transforms2 = [Normalize(mean=mean, std=std)] +deephead_ds = nn.Sequential(nn.Linear(16, 8), nn.Linear(8, 4)) +deephead_dt = nn.Sequential(nn.Linear(64, 8), nn.Linear(8, 4)) +deephead_di = nn.Sequential(nn.Linear(512, 8), nn.Linear(8, 4)) -############################################################################## +# ############################################################################# # Test many possible scenarios of data inputs I can think off. Surely users # will input something unexpected -############################################################################## +# ############################################################################# + + @pytest.mark.parametrize( "X_wide, X_deep, X_text, X_img, X_train, X_val, target, val_split, transforms, nepoch, null", [ @@ -312,3 +318,95 @@ def test_xtrain_xval_assertion( target=target, batch_size=16, ) + + +@pytest.mark.parametrize( + "wide, deepdense, deeptext, deepimage, X_wide, X_deep, X_text, X_img, target", + [ + (wide, None, None, None, X_wide, None, None, None, target), + (None, deepdense, None, None, None, X_deep, None, None, target), + (None, None, deeptext, None, None, None, X_text, None, target), + (None, None, None, deepimage, None, None, None, X_img, target), + ], +) +def test_individual_inputs( + wide, deepdense, deeptext, deepimage, X_wide, X_deep, X_text, X_img, target +): + model = WideDeep( + wide=wide, deepdense=deepdense, deeptext=deeptext, deepimage=deepimage + ) + model.compile(method="binary", verbose=0) + model.fit( + X_wide=X_wide, + X_deep=X_deep, + X_text=X_text, + X_img=X_img, + target=target, + batch_size=16, + ) + # check it has run succesfully + assert len(model.history._history) == 1 + + +############################################################################### +#  test deephead is not None and individual components +############################################################################### + + +@pytest.mark.parametrize( + "deepdense, deeptext, deepimage, X_deep, X_text, X_img, deephead, target", + [ + (deepdense, None, None, X_deep, None, None, deephead_ds, target), + (None, deeptext, None, None, X_text, None, deephead_dt, target), + (None, None, deepimage, None, None, X_img, deephead_di, target), + ], +) +def test_deephead_individual_components( + deepdense, deeptext, deepimage, X_deep, X_text, X_img, deephead, target +): + model = WideDeep( + deepdense=deepdense, deeptext=deeptext, deepimage=deepimage, deephead=deephead + ) # noqa: F841 + model.compile(method="binary", verbose=0) + model.fit( + X_wide=X_wide, + X_deep=X_deep, + X_text=X_text, + X_img=X_img, + target=target, + batch_size=16, + ) + # check it has run succesfully + assert len(model.history._history) == 1 + + +############################################################################### +#  test deephead is None and head_layers is not None and individual components +############################################################################### + + +@pytest.mark.parametrize( + "deepdense, deeptext, deepimage, X_deep, X_text, X_img, target", + [ + (deepdense, None, None, X_deep, None, None, target), + (None, deeptext, None, None, X_text, None, target), + (None, None, deepimage, None, None, X_img, target), + ], +) +def test_head_layers_individual_components( + deepdense, deeptext, deepimage, X_deep, X_text, X_img, target +): + model = WideDeep( + deepdense=deepdense, deeptext=deeptext, deepimage=deepimage, head_layers=[8, 4] + ) # noqa: F841 + model.compile(method="binary", verbose=0) + model.fit( + X_wide=X_wide, + X_deep=X_deep, + X_text=X_text, + X_img=X_img, + target=target, + batch_size=16, + ) + # check it has run succesfully + assert len(model.history._history) == 1 diff --git a/tests/test_warm_up/test_warm_up_routines.py b/tests/test_warm_up/test_warm_up_routines.py index c5611d77..2fd1c951 100644 --- a/tests/test_warm_up/test_warm_up_routines.py +++ b/tests/test_warm_up/test_warm_up_routines.py @@ -161,7 +161,7 @@ def test_warm_all(model, modelname, loader, n_epochs, max_lr): has_run = True try: warmer.warm_all(model, modelname, loader, n_epochs, max_lr) - except: + except Exception: has_run = False assert has_run @@ -182,6 +182,6 @@ def test_warm_gradual(model, modelname, loader, max_lr, layers, routine): has_run = True try: warmer.warm_gradual(model, modelname, loader, max_lr, layers, routine) - except: + except Exception: has_run = False assert has_run From e75ae5a5edff61237d4e462afe5363f986b78f21 Mon Sep 17 00:00:00 2001 From: jrzaurin Date: Thu, 3 Dec 2020 12:02:52 +0100 Subject: [PATCH 05/10] Added a few tests to increase coverage --- pytorch_widedeep/models/wide_deep.py | 41 +++- .../test_data_inputs.py | 2 +- .../test_miscellaneous.py | 194 ++++++++++++++++++ 3 files changed, 228 insertions(+), 9 deletions(-) create mode 100644 tests/test_model_functioning/test_miscellaneous.py diff --git a/pytorch_widedeep/models/wide_deep.py b/pytorch_widedeep/models/wide_deep.py index 873008fc..b4d5d3a0 100644 --- a/pytorch_widedeep/models/wide_deep.py +++ b/pytorch_widedeep/models/wide_deep.py @@ -121,8 +121,15 @@ def __init__( # noqa: C901 super(WideDeep, self).__init__() - self._check_params( - deepdense, deeptext, deepimage, deephead, head_layers, head_dropout + self._check_model_components( + wide, + deepdense, + deeptext, + deepimage, + deephead, + head_layers, + head_dropout, + pred_dim, ) # required as attribute just in case we pass a deephead @@ -337,9 +344,9 @@ def compile( # noqa: C901 if isinstance(optimizers, Dict) and not isinstance(lr_schedulers, Dict): raise ValueError( - "'parameters 'optimizers' and 'lr_schedulers' must have consistent type. " - "(Optimizer, LRScheduler) or (Dict[str, Optimizer], Dict[str, LRScheduler]) " - "Please, read the Documentation for more details" + "''optimizers' and 'lr_schedulers' must have consistent type: " + "(Optimizer and LRScheduler) or (Dict[str, Optimizer] and Dict[str, LRScheduler]) " + "Please, read the documentation or see the examples for more details" ) self.verbose = verbose @@ -1011,7 +1018,11 @@ def _predict( if X_test is not None: test_set = WideDeepDataset(**X_test) else: - load_dict = {"X_wide": X_wide, "X_deep": X_deep} + load_dict = {} + if X_wide is not None: + load_dict = {"X_wide": X_wide} + if X_deep is not None: + load_dict.update({"X_deep": X_deep}) if X_text is not None: load_dict.update({"X_text": X_text}) if X_img is not None: @@ -1057,10 +1068,24 @@ def _build_train_dict(X_wide, X_deep, X_text, X_img, target): return X_train @staticmethod # noqa: C901 - def _check_params( - deepdense, deeptext, deepimage, deephead, head_layers, head_dropout + def _check_model_components( + wide, + deepdense, + deeptext, + deepimage, + deephead, + head_layers, + head_dropout, + pred_dim, ): + if wide is not None: + assert wide.wide_linear.weight.size(1) == pred_dim, ( + "the 'pred_dim' of the wide component ({}) must be equal to the 'pred_dim' " + "of the deep component and the overall model itself ({})".format( + wide.wide_linear.weight.size(1), pred_dim + ) + ) if deepdense is not None and not hasattr(deepdense, "output_dim"): raise AttributeError( "deepdense model must have an 'output_dim' attribute. " diff --git a/tests/test_model_functioning/test_data_inputs.py b/tests/test_model_functioning/test_data_inputs.py index 37f2f8b2..483a8670 100644 --- a/tests/test_model_functioning/test_data_inputs.py +++ b/tests/test_model_functioning/test_data_inputs.py @@ -2,8 +2,8 @@ import numpy as np import pytest -from torchvision.transforms import ToTensor, Normalize from torch import nn +from torchvision.transforms import ToTensor, Normalize from sklearn.model_selection import train_test_split from pytorch_widedeep.models import ( diff --git a/tests/test_model_functioning/test_miscellaneous.py b/tests/test_model_functioning/test_miscellaneous.py new file mode 100644 index 00000000..5c518fd7 --- /dev/null +++ b/tests/test_model_functioning/test_miscellaneous.py @@ -0,0 +1,194 @@ +import string + +import numpy as np +import pytest +import torch + +from sklearn.model_selection import train_test_split + +from pytorch_widedeep.models import ( + Wide, + DeepText, + WideDeep, + DeepDense, + DeepImage, +) +from pytorch_widedeep.callbacks import EarlyStopping +from pytorch_widedeep.metrics import Accuracy, Precision + +# Wide array +X_wide = np.random.choice(50, (32, 10)) + +# Deep Array +colnames = list(string.ascii_lowercase)[:10] +embed_cols = [np.random.choice(np.arange(5), 32) for _ in range(5)] +embed_input = [(u, i, j) for u, i, j in zip(colnames[:5], [5] * 5, [16] * 5)] +cont_cols = [np.random.rand(32) for _ in range(5)] +X_deep = np.vstack(embed_cols + cont_cols).transpose() + +#  Text Array +padded_sequences = np.random.choice(np.arange(1, 100), (32, 48)) +X_text = np.hstack((np.repeat(np.array([[0, 0]]), 32, axis=0), padded_sequences)) +vocab_size = 100 + +#  Image Array +X_img = np.random.choice(256, (32, 224, 224, 3)) +X_img_norm = X_img / 255.0 + +# Target +target = np.random.choice(2, 32) +target_multi = np.random.choice(3, 32) + +# train/validation split +( + X_wide_tr, + X_wide_val, + X_deep_tr, + X_deep_val, + X_text_tr, + X_text_val, + X_img_tr, + X_img_val, + y_train, + y_val, +) = train_test_split(X_wide, X_deep, X_text, X_img, target) + +# build model components +wide = Wide(np.unique(X_wide).shape[0], 1) +deepdense = DeepDense( + hidden_layers=[32, 16], + dropout=[0.5, 0.5], + deep_column_idx={k: v for v, k in enumerate(colnames)}, + embed_input=embed_input, + continuous_cols=colnames[-5:], +) +deeptext = DeepText(vocab_size=vocab_size, embed_dim=32, padding_idx=0) +deepimage = DeepImage(pretrained=True) + +############################################################################### +#  test consistecy between optimizers and lr_schedulers format +############################################################################### + + +def test_optimizer_scheduler_format(): + model = WideDeep(deepdense=deepdense) + optimizers = {"deepdense": torch.optim.Adam(model.deepdense.parameters(), lr=0.01)} + schedulers = torch.optim.lr_scheduler.StepLR(optimizers["deepdense"], step_size=3) + with pytest.raises(ValueError): + model.compile( + method="binary", + optimizers=optimizers, + lr_schedulers=schedulers, + ) + + +############################################################################### +#  test that callbacks are properly initialised internally +############################################################################### + + +def test_non_instantiated_callbacks(): + model = WideDeep(wide=wide, deepdense=deepdense) + callbacks = [EarlyStopping] + model.compile(method="binary", callbacks=callbacks) + assert model.callbacks[1].__class__.__name__ == "EarlyStopping" + + +############################################################################### +#  test that multiple metrics are properly constructed internally +############################################################################### + + +def test_multiple_metrics(): + model = WideDeep(wide=wide, deepdense=deepdense) + metrics = [Accuracy, Precision] + model.compile(method="binary", metrics=metrics) + assert ( + model.metric._metrics[0].__class__.__name__ == "Accuracy" + and model.metric._metrics[1].__class__.__name__ == "Precision" + ) + + +############################################################################### +#  test the train step with metrics runs well for a binary prediction +############################################################################### + +def test_basic_run_with_metrics_binary(): + model = WideDeep(wide=wide, deepdense=deepdense) + model.compile(method="binary", metrics=[Accuracy], verbose=False) + model.fit( + X_wide=X_wide, + X_deep=X_deep, + target=target, + n_epochs=1, + batch_size=16, + val_split=0.2, + ) + assert ( + "train_loss" in model.history._history.keys() + and "train_acc" in model.history._history.keys() + ) + + +############################################################################### +#  test the train step with metrics runs well for a muticlass prediction +############################################################################### + +def test_basic_run_with_metrics_multiclass(): + wide = Wide(np.unique(X_wide).shape[0], 3) + deepdense = DeepDense( + hidden_layers=[32, 16], + dropout=[0.5, 0.5], + deep_column_idx={k: v for v, k in enumerate(colnames)}, + embed_input=embed_input, + continuous_cols=colnames[-5:], + ) + model = WideDeep(wide=wide, deepdense=deepdense, pred_dim=3) + model.compile(method="multiclass", metrics=[Accuracy], verbose=False) + model.fit( + X_wide=X_wide, + X_deep=X_deep, + target=target_multi, + n_epochs=1, + batch_size=16, + val_split=0.2, + ) + assert ( + "train_loss" in model.history._history.keys() + and "train_acc" in model.history._history.keys() + ) + + +############################################################################### +#  test predict method for individual components +############################################################################### + +@pytest.mark.parametrize( + "wide, deepdense, deeptext, deepimage, X_wide, X_deep, X_text, X_img, target", + [ + (wide, None, None, None, X_wide, None, None, None, target), + (None, deepdense, None, None, None, X_deep, None, None, target), + (None, None, deeptext, None, None, None, X_text, None, target), + (None, None, None, deepimage, None, None, None, X_img, target), + ], +) +def test_predict_with_individual_component( + wide, deepdense, deeptext, deepimage, X_wide, X_deep, X_text, X_img, target +): + + model = WideDeep( + wide=wide, deepdense=deepdense, deeptext=deeptext, deepimage=deepimage + ) + model.compile(method="binary", verbose=0) + model.fit( + X_wide=X_wide, + X_deep=X_deep, + X_text=X_text, + X_img=X_img, + target=target, + batch_size=16, + ) + # simply checking that runs and produces outputs + preds = model.predict(X_wide=X_wide, X_deep=X_deep, X_text=X_text, X_img=X_img) + + assert preds.shape[0] == 32 and "train_loss" in model.history._history From 3b0f9b7eb2c609082eb75e2a85a30d680118ffc6 Mon Sep 17 00:00:00 2001 From: jrzaurin Date: Thu, 3 Dec 2020 12:15:17 +0100 Subject: [PATCH 06/10] Fix minor style conflicts --- tests/test_model_functioning/test_miscellaneous.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_model_functioning/test_miscellaneous.py b/tests/test_model_functioning/test_miscellaneous.py index 5c518fd7..140d5a76 100644 --- a/tests/test_model_functioning/test_miscellaneous.py +++ b/tests/test_model_functioning/test_miscellaneous.py @@ -1,9 +1,8 @@ import string import numpy as np -import pytest import torch - +import pytest from sklearn.model_selection import train_test_split from pytorch_widedeep.models import ( @@ -13,8 +12,8 @@ DeepDense, DeepImage, ) -from pytorch_widedeep.callbacks import EarlyStopping from pytorch_widedeep.metrics import Accuracy, Precision +from pytorch_widedeep.callbacks import EarlyStopping # Wide array X_wide = np.random.choice(50, (32, 10)) @@ -113,6 +112,7 @@ def test_multiple_metrics(): #  test the train step with metrics runs well for a binary prediction ############################################################################### + def test_basic_run_with_metrics_binary(): model = WideDeep(wide=wide, deepdense=deepdense) model.compile(method="binary", metrics=[Accuracy], verbose=False) @@ -134,6 +134,7 @@ def test_basic_run_with_metrics_binary(): #  test the train step with metrics runs well for a muticlass prediction ############################################################################### + def test_basic_run_with_metrics_multiclass(): wide = Wide(np.unique(X_wide).shape[0], 3) deepdense = DeepDense( @@ -163,6 +164,7 @@ def test_basic_run_with_metrics_multiclass(): #  test predict method for individual components ############################################################################### + @pytest.mark.parametrize( "wide, deepdense, deeptext, deepimage, X_wide, X_deep, X_text, X_img, target", [ From 5a71fb7d03d3ec4a9459b842a2dacba60f0fa37e Mon Sep 17 00:00:00 2001 From: jrzaurin Date: Thu, 3 Dec 2020 18:21:48 +0100 Subject: [PATCH 07/10] updated docs so they are consistent with new functionalities. Updated logo. Updated README and fix a typo in setup.py --- README.md | 32 +++++++++++++++------------ docs/figures/widedeep_logo.png | Bin 74544 -> 38911 bytes docs/figures/widedeep_logo_old.png | Bin 0 -> 74544 bytes pypi_README.md | 6 +---- pytorch_widedeep/models/wide_deep.py | 10 --------- setup.py | 3 ++- 6 files changed, 21 insertions(+), 30 deletions(-) create mode 100644 docs/figures/widedeep_logo_old.png diff --git a/README.md b/README.md index dad4f128..23ba9947 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@

- +

[![Build Status](https://travis-ci.org/jrzaurin/pytorch-widedeep.svg?branch=master)](https://travis-ci.org/jrzaurin/pytorch-widedeep) @@ -9,11 +9,7 @@ [![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://github.com/jrzaurin/pytorch-widedeep/graphs/commit-activity) [![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/jrzaurin/pytorch-widedeep/issues) [![codecov](https://codecov.io/gh/jrzaurin/pytorch-widedeep/branch/master/graph/badge.svg)](https://codecov.io/gh/jrzaurin/pytorch-widedeep) - -Platform | Version Support ----------|:--------------- -OSX | [![Python 3.6 3.7](https://img.shields.io/badge/python-3.6%20%7C%203.7-blue.svg)](https://www.python.org/) -Linux | [![Python 3.6 3.7 3.8](https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8-blue.svg)](https://www.python.org/) +[![Python 3.6 3.7 3.8](https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8-blue.svg)](https://www.python.org/) # pytorch-widedeep @@ -88,15 +84,23 @@ as:

-When using `pytorch-widedeep`, the assumption is that the so called `Wide` and -`deep dense` (this can be either `DeepDense` or `DeepDenseResnet`. See the -documentation and examples folder for more details) components in the figures -are **always** present, while `DeepText text` and `DeepImage` are optional. +Note that each individual component, `wide`, `deepdense` (either `DeepDense` +or `DeepDenseResnet`), `deeptext` and `deepimage`, can be used independently +and in isolation. For example, one could use only `wide`, which is in simply a +linear model. + +On the other hand, while I recommend using the `Wide` and `DeepDense` (or +`DeepDenseResnet`) classes in `pytorch-widedeep` to build the `wide` and +`deepdense` component, it is very likely that users will want to use their own +models in the case of the `deeptext` and `deepimage` components. That is +perfectly possible as long as the the custom models have an attribute called +`output_dim` with the size of the last layer of activations, so that +`WideDeep` can be constructed + `pytorch-widedeep` includes standard text (stack of LSTMs) and image -(pre-trained ResNets or stack of CNNs) models. However, the user can use any -custom model as long as it has an attribute called `output_dim` with the size -of the last layer of activations, so that `WideDeep` can be constructed. See -the examples folder or the docs for more information. +(pre-trained ResNets or stack of CNNs) models. + +See the examples folder or the docs for more information. ### Installation diff --git a/docs/figures/widedeep_logo.png b/docs/figures/widedeep_logo.png index a444feff993b3ba8cad46abc3a23d1b881f12df4..2c703fc66450d1857f76151f2a7476160b9df1cb 100644 GIT binary patch literal 38911 zcmeGD1zTLr(gq6Smf#RHFt|f-cN^T@-QC^YJ-EBOI|L`ef@^Shmoty-{qFBOf8oq^ z^>k0KuCDG{YjsuKRTCjED~^bOivR`&hA06PQ3L~nNCX1|cZ7oh^@x#jd4Pc->{tj3 z%S#9g1LPg;Of9TUz`%eJDQU3E=*zhN7n4OvC;%b|MIpODX9yrXFUFCIvJe`Ol877% zy0f7zure?lPSsvTq<;Z3z8UbPijwX0Qv)UCiD7@}IMO9t=gYL$MVH@n_Di7$uQ*1Sz^II6x|kBGY!LBJF|C^Pf#(ZDF3lQpRPuF~(>Kd2Bsv~4p2DUNZm0&9WfW4rrrxmq{9WHd^A zID=4Mjq_Q)p=IM7zOPS=BB|DKn7A&HMm_$y|3jM$`$L*ZZa;}*%)7>}&d_H6G+sg$ z^T`Rr*O6l;i8lU8RHlK2TM?B;<#;g1xW<{fhKEEf9}9s4;p1uGuuWnbb9g$(SVK(t zGC3wrh#^Is`Wh=7va59Gna1*G;#;vUs(3w{IQG}FvTW{@UV+J+YpN)<-5*$w&oawH zl&55z8R3LyyB)~kLya*OYI5qz3P#_P0pAsrE7%k735)n<<(Hy=BmpW83 zU(%di`O_QJ*RW}Uv^?2EzIjMQ0lU1P-yo6IkZMRuL}4j&)Z{ zET=-&L@oH8SYDsK2)tld!+hcfx3M{tvY6>n()^`H=zdfW+KjMnX*y)L>8^bvh+7=e z+F8FcaHr>k)eGN@5aG;^dqH2y@hjy@)IH!lvWp|q`(Jc5wDT*P(-7sg#Qpx zGsI4inQ)l#rup(7;}2?dJwj!c;~*R3gv&7@1l%I#vk{Tq}!w2gWoG;l3CVF zFPv1^Jx@qF$inWzI_%(Ml zd@Fp5^c>%j(GlcX^-O$6eV25+{yErSVCx$b4j#5Rwi`}v>5;UN9J;g?ceU@zlm#J6 ziIuwr97`k%F3Y*O_itm%XA4vFRC6{9zTX4p8s?(rbn`3uF1l9sTs|GE&KtJ{N8%;? zze1bonn;?ltQO7n76^XNEWrOtIJ92itYj{m_;oicvq(3`Tg;s&K6mpA{V3%q`?puW zvROlV`Xs_W=YD)oi*H_!wqL7n*n46gN54Top(ESo?dak- z<-oi19G01)6}k%E`mRyZ2-SE$dF;Y&f6#H^dU3DsN$PcaeRx0ru7l)AI37zh?8C`gga=w#K(giM|lc#;3}2uuwWH9TqPCXdG_L z91y^!uvS|7V?TEZz1=IaMLa-KL7XGHB1Ru!9akVa6~~2g$G5`mDSSJK-H7cWtwqbq zM%BW0FPvqkZK3VF65YAo57mpxjh1fiPkvE&#=~zhs29vhCj$~YtY4k2j%DG z#pQ1n=N6f>Fu^$kj@x;o;>uK>*#zHvnBRoO)PM9u&rG&WhSCbDI{tR3&)^XDrq^Qk zQgAIDQdFco(%tINzX>F=sB^CUoH&)jdhlyT?5JW*`aQdgVv(ek`r*&OQiwQKIXxOf zM}}EKE8B%fkKcP-6l0V})N1M>gGPIi8I{-3xyi4FWD}bS#56403oX<-!w<>X@#vIA z)#WN`ojY5Xwc}KX>_AND?oDWKHMKHrO1e|cKP{#Q58p3DH{Gh5^*UY7T8VvU_6#$3 zu6Gzt+)wB~^k0OBkqb|h7Tv7vSL%#NYzy80tfT#i7==}f=y?pW6j+KN8vC5OXI4Xo^!AJPU1=NpEQlM%o+|Ii>_n$yCv2Us}Zg& zF5FsfJ-eQ5$HHPK)r0cP9QL1U*WE7LYzOx#+>IB`8!_!(wKV0GF9aXDA8(T{wS>ET z*dA>=sP_spMkynEa!t9)AC~V{!mUaVV+5Nym2!nSe5ZSNJ+CtNt7$!fX31uIKksrD z?^o|~u`dbRt_-K6deRcpNP8W7ul&*Sy?IaTvQ@lLqp%RICYgH!0dVNA zqy+BXhO1y*(h0q|r5BU%Jo(u`tX;q8%i=bWDupPi!Mz@h6h>PuEg#^=s=vz#jnMuf)nW<$pZ$Jh&pUONu z84=k7fOW#y2xkkO^Jj8Hy%VAHe&|`g9(?}6WDoX{^#O&h_jQ&AO8OaO^hTPfOPI>Y zfKh|ma9~j2xM0wr7C5Nzf#dzREe1{r2KjeC1Q=MD1sK%7V`M?~U!QnT`HSYC8Zt2y z3>NeY4OHCoApSENA~6s0KW%VF&^0hYC1D8(P_1O-XkudPWNzoIw~uuV>VUTgYB+&` zeIfg+fJ-QnUV`TTZK160tS%$XWn^bVZ(wX^XhQF9WB=DYU_91>@y{+Z-I^N5%@ z897?mJ6qV<0{)uUz|hXcnU|ROFGm0U{IgCIcZ>hyWb5?rwm=(X`0EM-BmFmq|IQ5} z<@u|ZOWwlW#9Cd%!UiNW5Dz{!Rt}!O$N#^s{2#}EC{>(H9EI&{K!nbG|0nc+iU0TF z|3Bg1EH(a*B{TcC|7H0L1lOF(`@MPm