diff --git a/bo_output/.DS_Store b/bo_output/.DS_Store index 2407e75..3c962ac 100644 Binary files a/bo_output/.DS_Store and b/bo_output/.DS_Store differ diff --git a/bo_output/test/scalerX_0.joblib b/bo_output/test/scalerX_0.joblib deleted file mode 100644 index 036f366..0000000 Binary files a/bo_output/test/scalerX_0.joblib and /dev/null differ diff --git a/bo_output/test/scalerX_1.joblib b/bo_output/test/scalerX_1.joblib deleted file mode 100644 index 78fa4a7..0000000 Binary files a/bo_output/test/scalerX_1.joblib and /dev/null differ diff --git a/bo_output/test/scalerX_2.joblib b/bo_output/test/scalerX_2.joblib deleted file mode 100644 index 62f448f..0000000 Binary files a/bo_output/test/scalerX_2.joblib and /dev/null differ diff --git a/bo_output/test/scalerY_0.joblib b/bo_output/test/scalerY_0.joblib deleted file mode 100644 index c119a49..0000000 Binary files a/bo_output/test/scalerY_0.joblib and /dev/null differ diff --git a/bo_output/test/scalerY_1.joblib b/bo_output/test/scalerY_1.joblib deleted file mode 100644 index 5fd952c..0000000 Binary files a/bo_output/test/scalerY_1.joblib and /dev/null differ diff --git a/bo_output/test/scalerY_2.joblib b/bo_output/test/scalerY_2.joblib deleted file mode 100644 index 0944eb4..0000000 Binary files a/bo_output/test/scalerY_2.joblib and /dev/null differ diff --git a/bo_output/test/surrogate_models.py b/bo_output/test/surrogate_models.py deleted file mode 100644 index a96e173..0000000 --- a/bo_output/test/surrogate_models.py +++ /dev/null @@ -1,95 +0,0 @@ -# This file costructs surrogate models for the input datasets -import numpy as np -import os -import json -from sklearn.model_selection import train_test_split -from sklearn.metrics import mean_squared_error -import math - -# Torch specific module imports -import torch -import gpytorch - -# botorch specific modules -from botorch.fit import fit_gpytorch_model -from botorch.models.gpytorch import GPyTorchModel - -# Plotting libraries -import matplotlib as mpl -import matplotlib.pyplot as plt - -# User defined python classes and files -import utils_dataset as utilsd -import input_class -import code_inputs as model_input - -np.random.seed(0) -torch.manual_seed(0) - -device = "cuda" if torch.cuda.is_available() else "cpu" -print(f"Using {device} device") - -# We will use the simplest form of GP model, exact inference -class ExactGPModel(gpytorch.models.ExactGP,GPyTorchModel): - _num_outputs = 1 # to inform GPyTorchModel API - MIN_INFERRED_NOISE_LEVEL = 1e-5 - def __init__(self, train_x, train_y, likelihood): - super(ExactGPModel, self).__init__(train_x, train_y, likelihood) - self.mean_module = gpytorch.means.ConstantMean() - if model_input.kernel=='RBF': - self.covar_module = gpytorch.kernels.ScaleKernel(gpytorch.kernels.RBFKernel()) - elif model_input.kernel=='Matern': - self.covar_module = gpytorch.kernels.ScaleKernel(gpytorch.kernels.MaternKernel(nu=0.5)) - - def forward(self, x): - mean_x = self.mean_module(x) - covar_x = self.covar_module(x) - return gpytorch.distributions.MultivariateNormal(mean_x, covar_x) - -#--------------------------- GP-0 ---------------------------# -def train_surrogate_gp0(X_train,Y_train): - - mse_gp0 = 0.0 - training_iter = model_input.epochs_GP0 - - # initialize likelihood and model - likelihood_gp0 = gpytorch.likelihoods.GaussianLikelihood() - model_gp0 = ExactGPModel(X_train, Y_train, likelihood_gp0) - - # Find optimal model hyperparameters - model_gp0.train() - likelihood_gp0.train() - - # Use the adam optimizer - optimizer = torch.optim.Adam(model_gp0.parameters(), lr=model_input.learning_rate_gp0) # Includes GaussianLikelihood parameters - - # "Loss" for GPs - the marginal log likelihood - mll = gpytorch.mlls.ExactMarginalLogLikelihood(likelihood_gp0, model_gp0) - - for i in range(training_iter): - optimizer.zero_grad() # Zero gradients from previous iteration - output = model_gp0(X_train) # Output from model - loss = -mll(output, Y_train) # Calc loss and backprop gradients - loss.backward() - optimizer.step() - - return model_gp0, likelihood_gp0 - -def predict_surrogates(model, likelihood, X): - - # Get into evaluation (predictive posterior) mode - model.eval() - likelihood.eval() - - # Make predictions by feeding model through likelihood - with torch.no_grad(), gpytorch.settings.fast_pred_var(): - prediction = model(X) - prediction = likelihood(model(X)) - - observed_mean = prediction.mean - observed_var = prediction.variance - observed_covar = prediction.covariance_matrix - - return observed_mean, observed_var - - \ No newline at end of file diff --git a/bo_output/test_no_cluster/scalerX_0.joblib b/bo_output/test_no_cluster/scalerX_0.joblib deleted file mode 100644 index 3545351..0000000 Binary files a/bo_output/test_no_cluster/scalerX_0.joblib and /dev/null differ diff --git a/bo_output/test_no_cluster/scalerY_0.joblib b/bo_output/test_no_cluster/scalerY_0.joblib deleted file mode 100644 index 9714360..0000000 Binary files a/bo_output/test_no_cluster/scalerY_0.joblib and /dev/null differ diff --git a/bo_output/test_no_cluster/surrogate_models.py b/bo_output/test_no_cluster/surrogate_models.py deleted file mode 100644 index a96e173..0000000 --- a/bo_output/test_no_cluster/surrogate_models.py +++ /dev/null @@ -1,95 +0,0 @@ -# This file costructs surrogate models for the input datasets -import numpy as np -import os -import json -from sklearn.model_selection import train_test_split -from sklearn.metrics import mean_squared_error -import math - -# Torch specific module imports -import torch -import gpytorch - -# botorch specific modules -from botorch.fit import fit_gpytorch_model -from botorch.models.gpytorch import GPyTorchModel - -# Plotting libraries -import matplotlib as mpl -import matplotlib.pyplot as plt - -# User defined python classes and files -import utils_dataset as utilsd -import input_class -import code_inputs as model_input - -np.random.seed(0) -torch.manual_seed(0) - -device = "cuda" if torch.cuda.is_available() else "cpu" -print(f"Using {device} device") - -# We will use the simplest form of GP model, exact inference -class ExactGPModel(gpytorch.models.ExactGP,GPyTorchModel): - _num_outputs = 1 # to inform GPyTorchModel API - MIN_INFERRED_NOISE_LEVEL = 1e-5 - def __init__(self, train_x, train_y, likelihood): - super(ExactGPModel, self).__init__(train_x, train_y, likelihood) - self.mean_module = gpytorch.means.ConstantMean() - if model_input.kernel=='RBF': - self.covar_module = gpytorch.kernels.ScaleKernel(gpytorch.kernels.RBFKernel()) - elif model_input.kernel=='Matern': - self.covar_module = gpytorch.kernels.ScaleKernel(gpytorch.kernels.MaternKernel(nu=0.5)) - - def forward(self, x): - mean_x = self.mean_module(x) - covar_x = self.covar_module(x) - return gpytorch.distributions.MultivariateNormal(mean_x, covar_x) - -#--------------------------- GP-0 ---------------------------# -def train_surrogate_gp0(X_train,Y_train): - - mse_gp0 = 0.0 - training_iter = model_input.epochs_GP0 - - # initialize likelihood and model - likelihood_gp0 = gpytorch.likelihoods.GaussianLikelihood() - model_gp0 = ExactGPModel(X_train, Y_train, likelihood_gp0) - - # Find optimal model hyperparameters - model_gp0.train() - likelihood_gp0.train() - - # Use the adam optimizer - optimizer = torch.optim.Adam(model_gp0.parameters(), lr=model_input.learning_rate_gp0) # Includes GaussianLikelihood parameters - - # "Loss" for GPs - the marginal log likelihood - mll = gpytorch.mlls.ExactMarginalLogLikelihood(likelihood_gp0, model_gp0) - - for i in range(training_iter): - optimizer.zero_grad() # Zero gradients from previous iteration - output = model_gp0(X_train) # Output from model - loss = -mll(output, Y_train) # Calc loss and backprop gradients - loss.backward() - optimizer.step() - - return model_gp0, likelihood_gp0 - -def predict_surrogates(model, likelihood, X): - - # Get into evaluation (predictive posterior) mode - model.eval() - likelihood.eval() - - # Make predictions by feeding model through likelihood - with torch.no_grad(), gpytorch.settings.fast_pred_var(): - prediction = model(X) - prediction = likelihood(model(X)) - - observed_mean = prediction.mean - observed_var = prediction.variance - observed_covar = prediction.covariance_matrix - - return observed_mean, observed_var - - \ No newline at end of file diff --git a/bo_output/test_with_cluster/scalerX_0.joblib b/bo_output/test_with_cluster/scalerX_0.joblib deleted file mode 100644 index c829044..0000000 Binary files a/bo_output/test_with_cluster/scalerX_0.joblib and /dev/null differ diff --git a/bo_output/test_with_cluster/scalerX_1.joblib b/bo_output/test_with_cluster/scalerX_1.joblib deleted file mode 100644 index a39bbb0..0000000 Binary files a/bo_output/test_with_cluster/scalerX_1.joblib and /dev/null differ diff --git a/bo_output/test_with_cluster/scalerX_2.joblib b/bo_output/test_with_cluster/scalerX_2.joblib deleted file mode 100644 index 3afd09f..0000000 Binary files a/bo_output/test_with_cluster/scalerX_2.joblib and /dev/null differ diff --git a/bo_output/test_with_cluster/scalerY_0.joblib b/bo_output/test_with_cluster/scalerY_0.joblib deleted file mode 100644 index 1384151..0000000 Binary files a/bo_output/test_with_cluster/scalerY_0.joblib and /dev/null differ diff --git a/bo_output/test_with_cluster/scalerY_1.joblib b/bo_output/test_with_cluster/scalerY_1.joblib deleted file mode 100644 index 3998c4d..0000000 Binary files a/bo_output/test_with_cluster/scalerY_1.joblib and /dev/null differ diff --git a/bo_output/test_with_cluster/scalerY_2.joblib b/bo_output/test_with_cluster/scalerY_2.joblib deleted file mode 100644 index d0a4511..0000000 Binary files a/bo_output/test_with_cluster/scalerY_2.joblib and /dev/null differ diff --git a/bo_output/test_with_cluster/surrogate_models.py b/bo_output/test_with_cluster/surrogate_models.py deleted file mode 100644 index 31aa376..0000000 --- a/bo_output/test_with_cluster/surrogate_models.py +++ /dev/null @@ -1,95 +0,0 @@ -# This file costructs surrogate models for the input datasets -import numpy as np -import os -import json -from sklearn.model_selection import train_test_split -from sklearn.metrics import mean_squared_error -import math - -# Torch specific module imports -import torch -import gpytorch - -# botorch specific modules -from botorch.fit import fit_gpytorch_model -from botorch.models.gpytorch import GPyTorchModel - -# Plotting libraries -import matplotlib as mpl -import matplotlib.pyplot as plt - -# User defined python classes and files -import utils_dataset as utilsd -import input_class -import code_inputs as model_input - -np.random.seed(0) -torch.manual_seed(0) - -device = "cuda" if torch.cuda.is_available() else "cpu" -print(f"Using {device} device") - -# We will use the simplest form of GP model, exact inference -class ExactGPModel(gpytorch.models.ExactGP,GPyTorchModel): - _num_outputs = 1 # to inform GPyTorchModel API - MIN_INFERRED_NOISE_LEVEL = 1e-5 - def __init__(self, train_x, train_y, likelihood): - super(ExactGPModel, self).__init__(train_x, train_y, likelihood) - self.mean_module = gpytorch.means.ConstantMean() - if model_input.KERNEL=='RBF': - self.covar_module = gpytorch.kernels.ScaleKernel(gpytorch.kernels.RBFKernel()) - elif model_input.KERNEL=='Matern': - self.covar_module = gpytorch.kernels.ScaleKernel(gpytorch.kernels.MaternKernel(nu=0.5)) - - def forward(self, x): - mean_x = self.mean_module(x) - covar_x = self.covar_module(x) - return gpytorch.distributions.MultivariateNormal(mean_x, covar_x) - -#--------------------------- GP-0 ---------------------------# -def train_surrogate_gp0(X_train,Y_train): - - mse_gp0 = 0.0 - training_iter = model_input.EPOCHS_GP0 - - # initialize likelihood and model - likelihood_gp0 = gpytorch.likelihoods.GaussianLikelihood() - model_gp0 = ExactGPModel(X_train, Y_train, likelihood_gp0) - - # Find optimal model hyperparameters - model_gp0.train() - likelihood_gp0.train() - - # Use the adam optimizer - optimizer = torch.optim.Adam(model_gp0.parameters(), lr=model_input.LR_GP0) # Includes GaussianLikelihood parameters - - # "Loss" for GPs - the marginal log likelihood - mll = gpytorch.mlls.ExactMarginalLogLikelihood(likelihood_gp0, model_gp0) - - for i in range(training_iter): - optimizer.zero_grad() # Zero gradients from previous iteration - output = model_gp0(X_train) # Output from model - loss = -mll(output, Y_train) # Calc loss and backprop gradients - loss.backward() - optimizer.step() - - return model_gp0, likelihood_gp0 - -def predict_surrogates(model, likelihood, X): - - # Get into evaluation (predictive posterior) mode - model.eval() - likelihood.eval() - - # Make predictions by feeding model through likelihood - with torch.no_grad(), gpytorch.settings.fast_pred_var(): - prediction = model(X) - prediction = likelihood(model(X)) - - observed_mean = prediction.mean - observed_var = prediction.variance - observed_covar = prediction.covariance_matrix - - return observed_mean, observed_var - - \ No newline at end of file diff --git a/src/BO.ipynb b/src/BO.ipynb index 6c2037e..9b07571 100644 --- a/src/BO.ipynb +++ b/src/BO.ipynb @@ -12,16 +12,6 @@ "text": [ "Using cpu device\n" ] - }, - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" } ], "source": [ @@ -36,7 +26,8 @@ "import warnings\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.metrics import mean_squared_error\n", - "from joblib import Parallel, delayed, dump\n", + "from joblib import Parallel, delayed, dump, load\n", + "import sys\n", "\n", "# Torch specific module imports\n", "import torch\n", @@ -87,7 +78,16 @@ "\n", "# Set the random seeds\n", "np.random.seed(0)\n", - "torch.manual_seed(0)" + "torch.manual_seed(0)\n", + "\n", + "# Perform input checks\n", + "if model_input.N_SEARCH == 0:\n", + " raise ValueError('Number of searches cannot be 0')\n", + "\n", + "if model_input.NUM_CLUSTER == 1:\n", + " n_batch_per_search = int(model_input.N_BATCH_PER_TRIAL)\n", + "else:\n", + " n_batch_per_search = int(model_input.N_BATCH_PER_TRIAL/model_input.N_SEARCH)" ] }, { @@ -103,68 +103,6 @@ "execution_count": 2, "id": "a8de62ac", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['dimensions',\n", - " ' bond type',\n", - " ' void fraction [widom]',\n", - " ' supercell volume [A^3]',\n", - " ' density [kg/m^3]',\n", - " ' heat desorption high P [kJ/mol]',\n", - " ' absolute methane uptake high P [molec/unit cell]',\n", - " ' absolute methane uptake high P [mol/kg]',\n", - " ' excess methane uptake high P [molec/unit cell]',\n", - " ' excess methane uptake high P [mol/kg]',\n", - " ' heat desorption low P [kJ/mol]',\n", - " ' absolute methane uptake low P [molec/unit cell]',\n", - " ' absolute methane uptake low P [mol/kg]',\n", - " ' excess methane uptake low P [molec/unit cell]',\n", - " ' excess methane uptake low P [mol/kg]',\n", - " ' surface area [m^2/g]',\n", - " ' linkerA',\n", - " ' linkerB',\n", - " ' net',\n", - " ' cell_a [A]',\n", - " ' cell_b [A]',\n", - " ' cell_c [A]',\n", - " ' alpha [deg]',\n", - " ' beta [deg]',\n", - " ' gamma [deg]',\n", - " ' num carbon',\n", - " ' num fluorine',\n", - " ' num hydrogen',\n", - " ' num nitrogen',\n", - " ' num oxygen',\n", - " ' num sulfur',\n", - " ' num silicon',\n", - " ' vertices',\n", - " ' edges',\n", - " ' genus',\n", - " ' largest included sphere diameter [A]',\n", - " ' largest free sphere diameter [A]',\n", - " ' largest included sphere along free sphere path diameter [A]',\n", - " ' absolute methane uptake high P [v STP/v]',\n", - " ' absolute methane uptake low P [v STP/v]']" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Input = input_class.inputs(input_path='../')\n", - "XX_prop, YY, descriptors = Input.read_inputs()\n", - "descriptors" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "147caa07", - "metadata": {}, "outputs": [ { "data": { @@ -342,19 +280,21 @@ "[69840 rows x 7 columns]" ] }, - "execution_count": 3, + "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ + "Input = input_class.inputs(input_path='../datasets/')\n", + "XX_prop, YY, descriptors = Input.read_inputs()\n", "XX_comp_df, YY_df = Input.get_comp()\n", "XX_comp_df" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 3, "id": "fe95ea1f", "metadata": {}, "outputs": [ @@ -392,58 +332,58 @@ " \n", " \n", " 0\n", - " 246\n", + " 832\n", + " 0\n", + " 448\n", + " 384\n", " 0\n", - " 156\n", - " 36\n", - " 24\n", " 0\n", " 0\n", - " 118.707728\n", + " 165.565439\n", " \n", " \n", " 1\n", - " 576\n", + " 1152\n", " 0\n", - " 512\n", + " 832\n", " 128\n", + " 64\n", " 0\n", " 0\n", - " 0\n", - " 170.133088\n", + " 152.524690\n", " \n", " \n", " 2\n", - " 656\n", + " 1376\n", " 0\n", - " 320\n", - " 64\n", + " 896\n", + " 256\n", " 64\n", " 0\n", " 0\n", - " 137.350112\n", + " 115.996501\n", " \n", " \n", " 3\n", - " 576\n", - " 0\n", - " 360\n", + " 864\n", " 0\n", + " 720\n", + " 192\n", " 0\n", " 0\n", " 0\n", - " 92.034339\n", + " 143.024802\n", " \n", " \n", " 4\n", - " 936\n", + " 1088\n", " 0\n", - " 648\n", - " 144\n", + " 768\n", + " 128\n", " 0\n", " 0\n", " 0\n", - " 147.296600\n", + " 153.528996\n", " \n", " \n", " ...\n", @@ -458,58 +398,58 @@ " \n", " \n", " 69835\n", - " 1248\n", + " 1536\n", " 0\n", - " 576\n", - " 320\n", + " 960\n", + " 160\n", " 0\n", " 0\n", " 0\n", - " 143.719775\n", + " 110.196985\n", " \n", " \n", " 69836\n", - " 1504\n", + " 1440\n", " 0\n", - " 1280\n", - " 256\n", - " 192\n", + " 1368\n", + " 216\n", " 0\n", " 0\n", - " 101.241397\n", + " 36\n", + " 137.095297\n", " \n", " \n", " 69837\n", - " 752\n", - " 0\n", - " 544\n", - " 160\n", + " 2560\n", " 0\n", + " 1536\n", + " 384\n", + " 384\n", " 0\n", " 0\n", - " 105.661907\n", + " 169.809763\n", " \n", " \n", " 69838\n", - " 440\n", + " 2784\n", " 0\n", - " 320\n", - " 64\n", - " 64\n", + " 1824\n", + " 576\n", + " 96\n", " 0\n", " 0\n", - " 132.487553\n", + " 110.963253\n", " \n", " \n", " 69839\n", - " 504\n", + " 1920\n", " 0\n", - " 288\n", - " 36\n", - " 72\n", + " 1200\n", + " 96\n", + " 48\n", " 0\n", " 0\n", - " 113.547143\n", + " 166.323232\n", " \n", " \n", "\n", @@ -518,80 +458,46 @@ ], "text/plain": [ " num carbon num fluorine num hydrogen num nitrogen num oxygen \\\n", - "0 246 0 156 36 24 \n", - "1 576 0 512 128 0 \n", - "2 656 0 320 64 64 \n", - "3 576 0 360 0 0 \n", - "4 936 0 648 144 0 \n", + "0 832 0 448 384 0 \n", + "1 1152 0 832 128 64 \n", + "2 1376 0 896 256 64 \n", + "3 864 0 720 192 0 \n", + "4 1088 0 768 128 0 \n", "... ... ... ... ... ... \n", - "69835 1248 0 576 320 0 \n", - "69836 1504 0 1280 256 192 \n", - "69837 752 0 544 160 0 \n", - "69838 440 0 320 64 64 \n", - "69839 504 0 288 36 72 \n", + "69835 1536 0 960 160 0 \n", + "69836 1440 0 1368 216 0 \n", + "69837 2560 0 1536 384 384 \n", + "69838 2784 0 1824 576 96 \n", + "69839 1920 0 1200 96 48 \n", "\n", " num sulfur num silicon deliverable capacity [v STP/v] \n", - "0 0 0 118.707728 \n", - "1 0 0 170.133088 \n", - "2 0 0 137.350112 \n", - "3 0 0 92.034339 \n", - "4 0 0 147.296600 \n", + "0 0 0 165.565439 \n", + "1 0 0 152.524690 \n", + "2 0 0 115.996501 \n", + "3 0 0 143.024802 \n", + "4 0 0 153.528996 \n", "... ... ... ... \n", - "69835 0 0 143.719775 \n", - "69836 0 0 101.241397 \n", - "69837 0 0 105.661907 \n", - "69838 0 0 132.487553 \n", - "69839 0 0 113.547143 \n", + "69835 0 0 110.196985 \n", + "69836 0 36 137.095297 \n", + "69837 0 0 169.809763 \n", + "69838 0 0 110.963253 \n", + "69839 0 0 166.323232 \n", "\n", "[69840 rows x 8 columns]" ] }, - "execution_count": 6, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "num_cluster = 1\n", - "clustered_dfs = km.k_means(XX_comp_df, YY_df, num_cluster)\n", + "clustered_dfs = km.k_means(XX_comp_df, YY_df, model_input.NUM_CLUSTER)\n", "sample_dfs = km.draw_samples(clustered_dfs, sample_fraction = 1.00)\n", "samples = km.concat(sample_dfs)\n", "samples" ] }, - { - "cell_type": "code", - "execution_count": 7, - "id": "3d105025", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[[246 0 156 ... 24 0 0]\n", - " [576 0 512 ... 0 0 0]\n", - " [656 0 320 ... 64 0 0]\n", - " ...\n", - " [752 0 544 ... 0 0 0]\n", - " [440 0 320 ... 64 0 0]\n", - " [504 0 288 ... 72 0 0]]\n", - "\n", - "\n", - "[118.70772784 170.13308797 137.35011199 ... 105.66190747 132.48755321\n", - " 113.54714269]\n" - ] - } - ], - "source": [ - "cluster_idx = 0\n", - "XX_desc = list(sample_dfs[cluster_idx].columns[:-1])\n", - "YY_desc = sample_dfs[cluster_idx].columns[-1]\n", - "print(sample_dfs[cluster_idx][XX_desc].to_numpy())\n", - "print('\\n')\n", - "print(sample_dfs[cluster_idx][YY_desc].to_numpy())" - ] - }, { "cell_type": "markdown", "id": "35ed4601", @@ -602,7 +508,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 4, "id": "955bc734-96c5-4d3a-9325-920c041e256b", "metadata": {}, "outputs": [], @@ -614,11 +520,9 @@ "num_restarts= 10 \n", "raw_samples = 512\n", "\n", - "def optimize_acqf_and_get_observation(acq_func, X_test, Y_test):\n", + "def optimize_acqf_and_get_observation(acq_func, X_test, Y_test, iteration):\n", " \"\"\"Optimizes the acquisition function, and returns a new candidate\"\"\"\n", - " # print(X_test)\n", - " # print(Y_test)\n", - " # optimize\n", + "\n", " candidates, _ = optimize_acqf_discrete(\n", " acq_function=acq_func,\n", " choices=X_test,\n", @@ -630,7 +534,7 @@ " unique=True\n", " )\n", " \n", - " print(candidates)\n", + " print(f'\\nBatch {iteration:>2}: Selected Candidate = {candidates}')\n", " # observe new values\n", " new_x = candidates.detach()\n", " b = [1 if torch.all(X_test[i].eq(new_x)) else 0 for i in range(0,X_test.shape[0]) ]\n", @@ -640,10 +544,6 @@ " \n", " X_test_new = X_test[torch.arange(0, X_test.shape[0]) != index, ...]\n", " Y_test_new = Y_test[..., torch.arange(0, Y_test.shape[1]) != index]\n", - " # X_test_new = X_test\n", - " # Y_test_new = Y_test\n", - " print(X_test_new)\n", - " print(Y_test_new)\n", " \n", " return new_x, new_y, index, X_test_new, Y_test_new" ] @@ -658,13 +558,13 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 5, "id": "72bb9112-1749-44fd-bc9d-7c0edb2e59a6", "metadata": {}, "outputs": [], "source": [ "def create_train_test_data(cluster_dataXX, cluster_dataYY, random_seed):\n", - " if model_input.standardize_data:\n", + " if model_input.STANDARDIZE:\n", " cluster_dataXX, scalerX_transform = utilsd.standardize_data(cluster_dataXX)\n", " cluster_dataYY, scalerY_transform = utilsd.standardize_data(cluster_dataYY.reshape(-1,1))\n", " else:\n", @@ -677,7 +577,7 @@ " # ....\n", "\n", " # Create train and test sets\n", - " X_train, X_test, Y_train, Y_test = train_test_split(cluster_dataXX, cluster_dataYY, test_size=model_input.test_size, random_state=random_seed)\n", + " X_train, X_test, Y_train, Y_test = train_test_split(cluster_dataXX, cluster_dataYY, test_size=model_input.TEST_SIZE, random_state=random_seed)\n", "\n", " # Convert to tensors\n", " X_train = torch.tensor(X_train, dtype=torch.float32)\n", @@ -692,7 +592,7 @@ "def train_gp(X_train, X_test, Y_train, Y_test, model=None):\n", " best_observed = []\n", " # Finding best value in initial data\n", - " if model_input.maximization:\n", + " if model_input.MAXIMIZATION:\n", " best_observed_value = Y_train.max()\n", " optimal_solution = torch.cat([Y_train[0],Y_test[0]]).max()\n", " else:\n", @@ -700,7 +600,7 @@ " optimal_solution = torch.cat([Y_train[0],Y_test[0]]).min()\n", " \n", " # If optimal value is present in the initial dataset sample remove it \n", - " if (best_observed_value.eq(optimal_solution)) and model_input.maximization:\n", + " if (best_observed_value.eq(optimal_solution)) and model_input.MAXIMIZATION:\n", " print('Max in training set, removing it before training models.')\n", " optimal_position = torch.argmax(Y_train)\n", " \n", @@ -717,7 +617,7 @@ " # Update best observed value\n", " best_observed_value = Y_train.max()\n", " \n", - " elif (best_observed_value.eq(optimal_solution)) and not model_input.maximization:\n", + " elif (best_observed_value.eq(optimal_solution)) and not model_input.MAXIMIZATION:\n", " print('Min in training set, removing it before training models.')\n", " optimal_position = torch.argmin(Y_train)\n", " \n", @@ -736,8 +636,6 @@ " \n", " # Initialize data for training gp-0 and gp-l models\n", " X_train0, Y_train0, X_test0, Y_test0 = X_train, Y_train, X_test, Y_test\n", - " \n", - " n_batch = model_input.n_batch_perTrial\n", " \n", " # Initialize likelihood, GP model and acquisition function for the models\n", " #--------------------------- GP-0 ---------------------------#\n", @@ -746,19 +644,20 @@ " model_gp0 = surrogate_models.ExactGPModel(X_train0, Y_train0, likelihood_gp0) \n", " else:\n", " model_gp0 = model\n", - " AcqFunc_0 = ExpectedImprovement(model=model_gp0, best_f=best_observed_value, maximize=model_input.maximization)\n", - " best_observed.append(best_observed_value) # Appending to best_observed list for the given trial\n", + " AcqFunc_0 = ExpectedImprovement(model=model_gp0, best_f=best_observed_value, maximize=model_input.MAXIMIZATION)\n", + " best_observed.append(best_observed_value.item()) # Appending to best_observed list for the given trial\n", " \n", " # run N_BATCH rounds of BayesOpt after the initial random batch\n", - " for iteration in range(1, n_batch + 1):\n", + " for iteration in range(1, n_batch_per_search + 1):\n", " # Time start of iteration and end\n", " t0 = time.monotonic()\n", - " if ((iteration-1)%model_input.n_update==0):\n", + " \n", + " if (model_input.N_UPDATE !=0) and ((iteration-1)%model_input.N_UPDATE==0):\n", " # fit the models every 10 iterations\n", " model_gp0, likelihood_gp0 = surrogate_models.train_surrogate_gp0(X_train0, Y_train0)\n", " \n", " # optimize and get new observation using acquisition function\n", - " new_x0, new_y0, index, X_test_new0, Y_test_new0 = optimize_acqf_and_get_observation(AcqFunc_0, X_test0, Y_test0)\n", + " new_x0, new_y0, index, X_test_new0, Y_test_new0 = optimize_acqf_and_get_observation(AcqFunc_0, X_test0, Y_test0, iteration)\n", " \n", " # Update remaining choices tensor\n", " X_test0 = X_test_new0\n", @@ -770,29 +669,26 @@ " Y_train0 = torch.reshape(Y_train0,(1,Y_train0.shape[0]))\n", "\n", " # update progress\n", - " if model_input.maximization:\n", + " if model_input.MAXIMIZATION:\n", " best_value_ei0 = Y_train0.max()\n", - " elif not model_input.maximization:\n", + " elif not model_input.MAXIMIZATION:\n", " best_value_ei0 = Y_train0.min()\n", - " best_observed.append(best_value_ei0)\n", + " best_observed.append(best_value_ei0.item())\n", "\n", " # AcqFunc_0 = UpperConfidenceBound(model_gp0, beta=0.1) \n", - " AcqFunc_0 = ExpectedImprovement(model=model_gp0, best_f=best_value_ei0, maximize=model_input.maximization)\n", + " AcqFunc_0 = ExpectedImprovement(model=model_gp0, best_f=best_value_ei0, maximize=model_input.MAXIMIZATION)\n", "\n", " # Time end of iteration\n", " t1 = time.monotonic()\n", " \n", - " if model_input.verbose:\n", + " if model_input.VERBOSE:\n", " print(\n", - " f\"\\nBatch {iteration:>2}: best_value (GP-0) = \",\n", - " f\"({best_value_ei0:>4.2f}\",\n", + " f\"Batch {iteration:>2}: best_value (GP-0) = \",\n", + " f\"{best_value_ei0.item():>4.2f}\",\n", " end=\"\",)\n", - " print(f'Iteration time = {t1-t0:>4.2f}.')\n", - "\n", - " # t1 = time.monotonic()\n", - " # print(f\"time = {t1-t0:>4.2f}.\")\n", + " print(f'\\nBatch {iteration:>2}: Iteration time = {t1-t0:>4.2f}.')\n", "\n", - " return best_observed, X_train0, X_test0, Y_train0, Y_test0, model_gp0 " + " return [best_observed, X_train0, X_test0, Y_train0, Y_test0, model_gp0] " ] }, { @@ -805,7 +701,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 6, "id": "bfb03e8a", "metadata": {}, "outputs": [ @@ -813,675 +709,32 @@ "name": "stdout", "output_type": "stream", "text": [ - "\n", - " -------------------- Trial 1 of 1 --------------------\n", - "\n", - " -------------------- Cluster 0 of 1 --------------------\n", - "tensor([[-0.4551, -0.0350, -1.0010, 1.2717, 0.2370, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 1: best_value (GP-0) = (2.59Iteration time = 174.28.\n", - "tensor([[-0.4551, -0.0350, -0.7803, 1.2717, 0.2370, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 2: best_value (GP-0) = (2.59Iteration time = 6.89.\n", - "tensor([[-0.8127, -0.0350, -1.0746, 1.2717, -0.0506, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 3: best_value (GP-0) = (2.59Iteration time = 6.38.\n", - "tensor([[-0.5743, -0.0350, -0.7557, 1.0076, 0.1411, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 4: best_value (GP-0) = (2.59Iteration time = 6.33.\n", - "tensor([[ 0.5988, -0.0350, -0.1180, 0.6776, -0.6258, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 5: best_value (GP-0) = (2.59Iteration time = 6.28.\n", - "tensor([[ 0.5988, -0.0350, -0.1180, 0.6776, -0.6258, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 6: best_value (GP-0) = (2.59Iteration time = 6.38.\n", - "tensor([[-0.3018, -0.0350, -0.5595, 1.2717, 0.2370, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 7: best_value (GP-0) = (2.59Iteration time = 6.49.\n", - "tensor([[-0.3018, -0.0350, -0.5595, 1.2717, 0.2370, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 8: best_value (GP-0) = (2.59Iteration time = 6.32.\n", - "tensor([[-0.3018, -0.0350, -0.5595, 1.2717, 0.2370, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 9: best_value (GP-0) = (2.59Iteration time = 6.61.\n", - "tensor([[-0.3018, -0.0350, -0.5595, 1.2717, 0.2370, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 10: best_value (GP-0) = (2.59Iteration time = 6.41.\n", - "tensor([[-0.3018, -0.0350, -0.5595, 1.2717, 0.2370, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 11: best_value (GP-0) = (2.59Iteration time = 6.28.\n", - "tensor([[-0.3018, -0.0350, -0.5595, 1.2717, 0.2370, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 12: best_value (GP-0) = (2.59Iteration time = 6.21.\n", - "tensor([[-0.3018, -0.0350, -0.5595, 1.2717, 0.2370, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 13: best_value (GP-0) = (2.59Iteration time = 6.39.\n", - "tensor([[-0.4551, -0.0350, -1.0010, 1.2717, 0.2370, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 14: best_value (GP-0) = (2.59Iteration time = 6.55.\n", - "tensor([[-0.4551, -0.0350, -1.0010, 1.2717, 0.2370, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 15: best_value (GP-0) = (2.59Iteration time = 6.53.\n", - "tensor([[-0.4551, -0.0350, -1.0010, 1.2717, 0.2370, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 16: best_value (GP-0) = (2.59Iteration time = 6.40.\n", - "tensor([[-0.0974, -0.0350, -0.7067, 1.2717, -0.0506, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 17: best_value (GP-0) = (2.59Iteration time = 6.39.\n", - "tensor([[-0.0974, -0.0350, -0.7067, 1.2717, -0.0506, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 18: best_value (GP-0) = (2.59Iteration time = 6.33.\n", - "tensor([[-0.0974, -0.0350, -0.7067, 1.2717, -0.0506, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 19: best_value (GP-0) = (2.59Iteration time = 6.33.\n", - "tensor([[-0.0974, -0.0350, -0.7067, 1.2717, -0.0506, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 20: best_value (GP-0) = (2.59Iteration time = 6.36.\n", - "tensor([[-0.0974, -0.0350, -0.7067, 1.2717, -0.0506, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 21: best_value (GP-0) = (2.59Iteration time = 170.42.\n", - "tensor([[-0.7616, -0.0350, -1.2218, 1.2717, 0.2370, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 22: best_value (GP-0) = (2.59Iteration time = 6.56.\n", - "tensor([[-0.7233, -0.0350, -0.9458, 1.1232, 0.0213, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 23: best_value (GP-0) = (2.59Iteration time = 6.50.\n", - "tensor([[-0.7233, -0.0350, -0.9458, 1.1232, 0.0213, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 24: best_value (GP-0) = (2.59Iteration time = 6.49.\n", - "tensor([[-0.8383, -0.0350, -1.1114, 0.9746, 0.2370, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 25: best_value (GP-0) = (2.59Iteration time = 6.64.\n", - "tensor([[-0.9149, -0.0350, -0.8906, 0.9746, 0.2370, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 26: best_value (GP-0) = (2.59Iteration time = 6.56.\n", - "tensor([[-0.9149, -0.0350, -0.8906, 0.9746, 0.2370, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 27: best_value (GP-0) = (2.59Iteration time = 6.62.\n", - "tensor([[-0.0974, -0.0350, -0.7067, 1.2717, 0.5246, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 28: best_value (GP-0) = (2.59Iteration time = 6.43.\n", - "tensor([[-0.0974, -0.0350, -0.7067, 1.2717, 0.5246, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 29: best_value (GP-0) = (2.59Iteration time = 6.32.\n", - "tensor([[-0.0974, -0.0350, -0.7067, 1.2717, 0.5246, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 30: best_value (GP-0) = (2.59Iteration time = 6.38.\n", - "tensor([[-0.0974, -0.0350, -0.7067, 1.2717, 0.5246, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 31: best_value (GP-0) = (2.59Iteration time = 6.38.\n", - "tensor([[-0.0974, -0.0350, -0.7067, 1.2717, 0.5246, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 32: best_value (GP-0) = (2.59Iteration time = 6.09.\n", - "tensor([[-0.0974, -0.0350, -0.7067, 1.2717, 0.5246, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 33: best_value (GP-0) = (2.59Iteration time = 5.88.\n", - "tensor([[-0.4551, -0.0350, -0.5595, 1.2717, 0.2370, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 34: best_value (GP-0) = (2.59Iteration time = 6.21.\n", - "tensor([[-0.4551, -0.0350, -0.5595, 1.2717, 0.2370, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 35: best_value (GP-0) = (2.59Iteration time = 6.16.\n", - "tensor([[-0.4551, -0.0350, -0.5595, 1.2717, 0.2370, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 36: best_value (GP-0) = (2.59Iteration time = 6.16.\n", - "tensor([[-0.4551, -0.0350, -0.5595, 1.2717, 0.2370, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 37: best_value (GP-0) = (2.59Iteration time = 6.15.\n", - "tensor([[-0.4551, -0.0350, -0.5595, 1.2717, 0.2370, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 38: best_value (GP-0) = (2.59Iteration time = 5.96.\n", - "tensor([[-0.4551, -0.0350, -0.5595, 1.2717, 0.2370, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 39: best_value (GP-0) = (2.59Iteration time = 5.99.\n", - "tensor([[-0.4551, -0.0350, -0.5595, 1.2717, 0.2370, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 40: best_value (GP-0) = (2.59Iteration time = 6.08.\n", - "tensor([[-0.4551, -0.0350, -0.5595, 1.2717, 0.2370, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 41: best_value (GP-0) = (2.59Iteration time = 137.45.\n", - "tensor([[-0.6084, -0.0350, -0.7803, 1.5688, 0.6684, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 42: best_value (GP-0) = (2.59Iteration time = 7.03.\n", - "tensor([[-0.8127, -0.0350, -1.0746, 1.2717, -0.3382, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 43: best_value (GP-0) = (2.59Iteration time = 7.27.\n", - "tensor([[-0.4551, -0.0350, -0.7803, 1.8658, 0.2370, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 44: best_value (GP-0) = (2.59Iteration time = 7.36.\n", - "tensor([[-0.4551, -0.0350, -0.7803, 1.8658, 0.2370, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 45: best_value (GP-0) = (2.59Iteration time = 7.34.\n", - "tensor([[-0.4551, -0.0350, -0.7803, 1.8658, 0.2370, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 46: best_value (GP-0) = (2.59Iteration time = 7.45.\n", - "tensor([[-0.4551, -0.0350, -0.7803, 1.8658, 0.2370, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 47: best_value (GP-0) = (2.59Iteration time = 7.51.\n", - "tensor([[-0.2635, -0.0350, -0.6147, 1.5688, 0.0213, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 48: best_value (GP-0) = (2.74Iteration time = 7.53.\n", - "tensor([[-0.8383, -0.0350, -0.8906, 0.9746, 0.6684, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 49: best_value (GP-0) = (2.74Iteration time = 7.45.\n", - "tensor([[-0.3784, -0.0350, -1.1114, 0.6776, 0.6684, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 50: best_value (GP-0) = (2.74Iteration time = 7.68.\n", - "tensor([[-0.3784, -0.0350, -1.1114, 0.6776, 0.6684, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 51: best_value (GP-0) = (2.74Iteration time = 7.49.\n", - "tensor([[-1.1193, -0.0350, -1.1482, 0.8756, -0.0506, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 52: best_value (GP-0) = (2.74Iteration time = 7.51.\n", - "tensor([[-1.1193, -0.0350, -1.1482, 0.8756, -0.0506, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 53: best_value (GP-0) = (2.74Iteration time = 7.36.\n", - "tensor([[-0.8383, -0.0350, -1.2770, 1.1232, 0.6684, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 54: best_value (GP-0) = (2.74Iteration time = 7.58.\n", - "tensor([[-0.8383, -0.0350, -1.2770, 1.1232, 0.6684, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 55: best_value (GP-0) = (2.74Iteration time = 7.30.\n", - "tensor([[-0.8383, -0.0350, -1.2770, 1.1232, 0.6684, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 56: best_value (GP-0) = (2.74Iteration time = 7.38.\n", - "tensor([[-0.5062, -0.0350, -0.4123, 1.2717, 0.5246, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 57: best_value (GP-0) = (2.74Iteration time = 7.21.\n", - "tensor([[-0.5062, -0.0350, -0.4123, 1.2717, 0.5246, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 58: best_value (GP-0) = (2.74Iteration time = 7.47.\n", - "tensor([[-0.7233, -0.0350, -1.1114, 0.6776, 0.6684, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 59: best_value (GP-0) = (2.74Iteration time = 7.58.\n", - "tensor([[-0.7233, -0.0350, -1.1114, 0.6776, 0.6684, -0.0614, -0.1589]])\n", - "tensor([[-0.1315, -0.0350, 0.3236, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.4380, -0.0350, -0.1670, ..., -0.2423, -0.0614, -0.1589],\n", - " [-0.1485, -0.0350, -0.1180, ..., 0.2370, -0.0614, -0.1589],\n", - " ...,\n", - " [-0.4551, -0.0350, -0.4859, ..., -0.6258, -0.0614, -0.1589],\n", - " [-0.7106, -0.0350, -0.7067, ..., -0.6258, -0.0614, -0.1589],\n", - " [-1.0171, -0.0350, -0.9274, ..., -0.6258, -0.0614, -0.1589]])\n", - "tensor([[ 0.7023, -0.4456, 1.6468, ..., -0.4606, -0.2699, -2.1043]])\n", - "\n", - "Batch 60: best_value (GP-0) = (2.74Iteration time = 7.36.\n", - "\n", - "\n", - "Starting the epsilon greedy search\n", - "\n", - "\n" + "The new directory is created! /Users/nikhilthota/Desktop/lab/projects/SPIRAL/codes_and_datasets/T-NIKHIL/project-localGPs_for_COF/bo_output/results_with_three_clusters/\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[6], line 102\u001b[0m\n\u001b[1;32m 100\u001b[0m search_text \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mexploiting\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m 101\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124mIteration \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mi\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m : Cluster \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mcluster_idx\u001b[38;5;250m \u001b[39m\u001b[38;5;241m+\u001b[39m\u001b[38;5;250m \u001b[39m\u001b[38;5;241m1\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m is selected for training (\u001b[39m\u001b[38;5;132;01m{\u001b[39;00msearch_text\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m)\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m--> 102\u001b[0m results \u001b[38;5;241m=\u001b[39m \u001b[43mtrain_gp\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX_train_all_clusters\u001b[49m\u001b[43m[\u001b[49m\u001b[43mcluster_idx\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m\\\u001b[49m\n\u001b[1;32m 103\u001b[0m \u001b[43m \u001b[49m\u001b[43mX_test_all_clusters\u001b[49m\u001b[43m[\u001b[49m\u001b[43mcluster_idx\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m\\\u001b[49m\n\u001b[1;32m 104\u001b[0m \u001b[43m \u001b[49m\u001b[43mY_train_all_clusters\u001b[49m\u001b[43m[\u001b[49m\u001b[43mcluster_idx\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m\\\u001b[49m\n\u001b[1;32m 105\u001b[0m \u001b[43m \u001b[49m\u001b[43mY_test_all_clusters\u001b[49m\u001b[43m[\u001b[49m\u001b[43mcluster_idx\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m\\\u001b[49m\n\u001b[1;32m 106\u001b[0m \u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmodel_gps_all_clusters\u001b[49m\u001b[43m[\u001b[49m\u001b[43mcluster_idx\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 108\u001b[0m \u001b[38;5;66;03m# Add the best values\u001b[39;00m\n\u001b[1;32m 109\u001b[0m best_observed_all_clusters[cluster_idx]\u001b[38;5;241m.\u001b[39mextend(results[\u001b[38;5;241m0\u001b[39m])\n", + "Cell \u001b[0;32mIn[5], line 95\u001b[0m, in \u001b[0;36mtrain_gp\u001b[0;34m(X_train, X_test, Y_train, Y_test, model)\u001b[0m\n\u001b[1;32m 92\u001b[0m model_gp0, likelihood_gp0 \u001b[38;5;241m=\u001b[39m surrogate_models\u001b[38;5;241m.\u001b[39mtrain_surrogate_gp0(X_train0, Y_train0)\n\u001b[1;32m 94\u001b[0m \u001b[38;5;66;03m# optimize and get new observation using acquisition function\u001b[39;00m\n\u001b[0;32m---> 95\u001b[0m new_x0, new_y0, index, X_test_new0, Y_test_new0 \u001b[38;5;241m=\u001b[39m \u001b[43moptimize_acqf_and_get_observation\u001b[49m\u001b[43m(\u001b[49m\u001b[43mAcqFunc_0\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mX_test0\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mY_test0\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43miteration\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 97\u001b[0m \u001b[38;5;66;03m# Update remaining choices tensor\u001b[39;00m\n\u001b[1;32m 98\u001b[0m X_test0 \u001b[38;5;241m=\u001b[39m X_test_new0\n", + "Cell \u001b[0;32mIn[4], line 11\u001b[0m, in \u001b[0;36moptimize_acqf_and_get_observation\u001b[0;34m(acq_func, X_test, Y_test, iteration)\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21moptimize_acqf_and_get_observation\u001b[39m(acq_func, X_test, Y_test, iteration):\n\u001b[1;32m 9\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Optimizes the acquisition function, and returns a new candidate\"\"\"\u001b[39;00m\n\u001b[0;32m---> 11\u001b[0m candidates, _ \u001b[38;5;241m=\u001b[39m \u001b[43moptimize_acqf_discrete\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 12\u001b[0m \u001b[43m \u001b[49m\u001b[43macq_function\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43macq_func\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 13\u001b[0m \u001b[43m \u001b[49m\u001b[43mchoices\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mX_test\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 14\u001b[0m \u001b[43m \u001b[49m\u001b[43mq\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbatch_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 15\u001b[0m \u001b[43m \u001b[49m\u001b[43mmax_batch_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m2048\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 16\u001b[0m \u001b[43m \u001b[49m\u001b[43mnum_restarts\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnum_restarts\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 17\u001b[0m \u001b[43m \u001b[49m\u001b[43mraw_samples\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mraw_samples\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# used for intialization heuristic\u001b[39;49;00m\n\u001b[1;32m 18\u001b[0m \u001b[43m \u001b[49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mbatch_limit\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m5\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mmaxiter\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m200\u001b[39;49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 19\u001b[0m \u001b[43m \u001b[49m\u001b[43munique\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\n\u001b[1;32m 20\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 22\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124mBatch \u001b[39m\u001b[38;5;132;01m{\u001b[39;00miteration\u001b[38;5;132;01m:\u001b[39;00m\u001b[38;5;124m>2\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m: Selected Candidate = \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mcandidates\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 23\u001b[0m \u001b[38;5;66;03m# observe new values\u001b[39;00m\n", + "File \u001b[0;32m~/miniconda3/envs/bo-hackathon/lib/python3.12/site-packages/botorch/optim/optimize.py:1087\u001b[0m, in \u001b[0;36moptimize_acqf_discrete\u001b[0;34m(acq_function, q, choices, max_batch_size, unique, **kwargs)\u001b[0m\n\u001b[1;32m 1084\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m candidates, torch\u001b[38;5;241m.\u001b[39mstack(acq_value_list)\n\u001b[1;32m 1086\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mno_grad():\n\u001b[0;32m-> 1087\u001b[0m acq_values \u001b[38;5;241m=\u001b[39m \u001b[43m_split_batch_eval_acqf\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1088\u001b[0m \u001b[43m \u001b[49m\u001b[43macq_function\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43macq_function\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mX\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchoices_batched\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmax_batch_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmax_batch_size\u001b[49m\n\u001b[1;32m 1089\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1090\u001b[0m best_idx \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39margmax(acq_values)\n\u001b[1;32m 1091\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m choices_batched[best_idx], acq_values[best_idx]\n", + "File \u001b[0;32m~/miniconda3/envs/bo-hackathon/lib/python3.12/site-packages/botorch/optim/optimize.py:1097\u001b[0m, in \u001b[0;36m_split_batch_eval_acqf\u001b[0;34m(acq_function, X, max_batch_size)\u001b[0m\n\u001b[1;32m 1094\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_split_batch_eval_acqf\u001b[39m(\n\u001b[1;32m 1095\u001b[0m acq_function: AcquisitionFunction, X: Tensor, max_batch_size: \u001b[38;5;28mint\u001b[39m\n\u001b[1;32m 1096\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[0;32m-> 1097\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mcat([\u001b[43macq_function\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX_\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m X_ \u001b[38;5;129;01min\u001b[39;00m X\u001b[38;5;241m.\u001b[39msplit(max_batch_size)])\n", + "File \u001b[0;32m~/miniconda3/envs/bo-hackathon/lib/python3.12/site-packages/torch/nn/modules/module.py:1511\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1509\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1510\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1511\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/miniconda3/envs/bo-hackathon/lib/python3.12/site-packages/torch/nn/modules/module.py:1520\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1515\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1518\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1519\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1520\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1523\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", + "File \u001b[0;32m~/miniconda3/envs/bo-hackathon/lib/python3.12/site-packages/botorch/utils/transforms.py:259\u001b[0m, in \u001b[0;36mt_batch_mode_transform..decorator..decorated\u001b[0;34m(acqf, X, *args, **kwargs)\u001b[0m\n\u001b[1;32m 257\u001b[0m \u001b[38;5;66;03m# add t-batch dim\u001b[39;00m\n\u001b[1;32m 258\u001b[0m X \u001b[38;5;241m=\u001b[39m X \u001b[38;5;28;01mif\u001b[39;00m X\u001b[38;5;241m.\u001b[39mdim() \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m2\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m X\u001b[38;5;241m.\u001b[39munsqueeze(\u001b[38;5;241m0\u001b[39m)\n\u001b[0;32m--> 259\u001b[0m output \u001b[38;5;241m=\u001b[39m \u001b[43mmethod\u001b[49m\u001b[43m(\u001b[49m\u001b[43macqf\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 260\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(acqf, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmodel\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;129;01mand\u001b[39;00m is_ensemble(acqf\u001b[38;5;241m.\u001b[39mmodel):\n\u001b[1;32m 261\u001b[0m \u001b[38;5;66;03m# IDEA: this could be wrapped into SampleReducingMCAcquisitionFunction\u001b[39;00m\n\u001b[1;32m 262\u001b[0m output \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m 263\u001b[0m output\u001b[38;5;241m.\u001b[39mmean(dim\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m) \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m acqf\u001b[38;5;241m.\u001b[39m_log \u001b[38;5;28;01melse\u001b[39;00m logmeanexp(output, dim\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m 264\u001b[0m )\n", + "File \u001b[0;32m~/miniconda3/envs/bo-hackathon/lib/python3.12/site-packages/botorch/acquisition/analytic.py:355\u001b[0m, in \u001b[0;36mExpectedImprovement.forward\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m 341\u001b[0m \u001b[38;5;129m@t_batch_mode_transform\u001b[39m(expected_q\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m 342\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, X: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[1;32m 343\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124;03m\"\"\"Evaluate Expected Improvement on the candidate set X.\u001b[39;00m\n\u001b[1;32m 344\u001b[0m \n\u001b[1;32m 345\u001b[0m \u001b[38;5;124;03m Args:\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 353\u001b[0m \u001b[38;5;124;03m given design points `X`.\u001b[39;00m\n\u001b[1;32m 354\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 355\u001b[0m mean, sigma \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_mean_and_sigma\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 356\u001b[0m u \u001b[38;5;241m=\u001b[39m _scaled_improvement(mean, sigma, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbest_f, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmaximize)\n\u001b[1;32m 357\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m sigma \u001b[38;5;241m*\u001b[39m _ei_helper(u)\n", + "File \u001b[0;32m~/miniconda3/envs/bo-hackathon/lib/python3.12/site-packages/botorch/acquisition/analytic.py:106\u001b[0m, in \u001b[0;36mAnalyticAcquisitionFunction._mean_and_sigma\u001b[0;34m(self, X, compute_sigma, min_var)\u001b[0m\n\u001b[1;32m 92\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Computes the first and second moments of the model posterior.\u001b[39;00m\n\u001b[1;32m 93\u001b[0m \n\u001b[1;32m 94\u001b[0m \u001b[38;5;124;03mArgs:\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 103\u001b[0m \u001b[38;5;124;03m returns a single tensor of means if compute_sigma is True.\u001b[39;00m\n\u001b[1;32m 104\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 105\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mto(device\u001b[38;5;241m=\u001b[39mX\u001b[38;5;241m.\u001b[39mdevice) \u001b[38;5;66;03m# ensures buffers / parameters are on the same device\u001b[39;00m\n\u001b[0;32m--> 106\u001b[0m posterior \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mposterior\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 107\u001b[0m \u001b[43m \u001b[49m\u001b[43mX\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mposterior_transform\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mposterior_transform\u001b[49m\n\u001b[1;32m 108\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 109\u001b[0m mean \u001b[38;5;241m=\u001b[39m posterior\u001b[38;5;241m.\u001b[39mmean\u001b[38;5;241m.\u001b[39msqueeze(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m2\u001b[39m)\u001b[38;5;241m.\u001b[39msqueeze(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m) \u001b[38;5;66;03m# removing redundant dimensions\u001b[39;00m\n\u001b[1;32m 110\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m compute_sigma:\n", + "File \u001b[0;32m~/miniconda3/envs/bo-hackathon/lib/python3.12/site-packages/botorch/models/gpytorch.py:183\u001b[0m, in \u001b[0;36mGPyTorchModel.posterior\u001b[0;34m(self, X, observation_noise, posterior_transform, **kwargs)\u001b[0m\n\u001b[1;32m 178\u001b[0m X \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtransform_inputs(X)\n\u001b[1;32m 179\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m gpt_posterior_settings():\n\u001b[1;32m 180\u001b[0m \u001b[38;5;66;03m# NOTE: BoTorch's GPyTorchModels also inherit from GPyTorch's ExactGP, thus\u001b[39;00m\n\u001b[1;32m 181\u001b[0m \u001b[38;5;66;03m# self(X) calls GPyTorch's ExactGP's __call__, which computes the posterior,\u001b[39;00m\n\u001b[1;32m 182\u001b[0m \u001b[38;5;66;03m# rather than e.g. SingleTaskGP's forward, which computes the prior.\u001b[39;00m\n\u001b[0;32m--> 183\u001b[0m mvn \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mX\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 184\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m observation_noise \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mFalse\u001b[39;00m:\n\u001b[1;32m 185\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(observation_noise, torch\u001b[38;5;241m.\u001b[39mTensor):\n\u001b[1;32m 186\u001b[0m \u001b[38;5;66;03m# TODO: Make sure observation noise is transformed correctly\u001b[39;00m\n", + "File \u001b[0;32m~/miniconda3/envs/bo-hackathon/lib/python3.12/site-packages/gpytorch/models/exact_gp.py:333\u001b[0m, in \u001b[0;36mExactGP.__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 328\u001b[0m \u001b[38;5;66;03m# Make the prediction\u001b[39;00m\n\u001b[1;32m 329\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m settings\u001b[38;5;241m.\u001b[39mcg_tolerance(settings\u001b[38;5;241m.\u001b[39meval_cg_tolerance\u001b[38;5;241m.\u001b[39mvalue()):\n\u001b[1;32m 330\u001b[0m (\n\u001b[1;32m 331\u001b[0m predictive_mean,\n\u001b[1;32m 332\u001b[0m predictive_covar,\n\u001b[0;32m--> 333\u001b[0m ) \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprediction_strategy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexact_prediction\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfull_mean\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfull_covar\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 335\u001b[0m \u001b[38;5;66;03m# Reshape predictive mean to match the appropriate event shape\u001b[39;00m\n\u001b[1;32m 336\u001b[0m predictive_mean \u001b[38;5;241m=\u001b[39m predictive_mean\u001b[38;5;241m.\u001b[39mview(\u001b[38;5;241m*\u001b[39mbatch_shape, \u001b[38;5;241m*\u001b[39mtest_shape)\u001b[38;5;241m.\u001b[39mcontiguous()\n", + "File \u001b[0;32m~/miniconda3/envs/bo-hackathon/lib/python3.12/site-packages/gpytorch/models/exact_prediction_strategies.py:290\u001b[0m, in \u001b[0;36mDefaultPredictionStrategy.exact_prediction\u001b[0;34m(self, joint_mean, joint_covar)\u001b[0m\n\u001b[1;32m 285\u001b[0m test_test_covar \u001b[38;5;241m=\u001b[39m joint_covar[\u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnum_train :, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnum_train :]\n\u001b[1;32m 286\u001b[0m test_train_covar \u001b[38;5;241m=\u001b[39m joint_covar[\u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnum_train :, : \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnum_train]\n\u001b[1;32m 288\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m (\n\u001b[1;32m 289\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexact_predictive_mean(test_mean, test_train_covar),\n\u001b[0;32m--> 290\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexact_predictive_covar\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtest_test_covar\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtest_train_covar\u001b[49m\u001b[43m)\u001b[49m,\n\u001b[1;32m 291\u001b[0m )\n", + "File \u001b[0;32m~/miniconda3/envs/bo-hackathon/lib/python3.12/site-packages/gpytorch/models/exact_prediction_strategies.py:357\u001b[0m, in \u001b[0;36mDefaultPredictionStrategy.exact_predictive_covar\u001b[0;34m(self, test_test_covar, test_train_covar)\u001b[0m\n\u001b[1;32m 354\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m test_test_covar \u001b[38;5;241m+\u001b[39m MatmulLinearOperator(test_train_covar, covar_correction_rhs\u001b[38;5;241m.\u001b[39mmul(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m))\n\u001b[1;32m 356\u001b[0m precomputed_cache \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcovar_cache\n\u001b[0;32m--> 357\u001b[0m covar_inv_quad_form_root \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_exact_predictive_covar_inv_quad_form_root\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprecomputed_cache\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtest_train_covar\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 358\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mis_tensor(test_test_covar):\n\u001b[1;32m 359\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m to_linear_operator(\n\u001b[1;32m 360\u001b[0m torch\u001b[38;5;241m.\u001b[39madd(\n\u001b[1;32m 361\u001b[0m test_test_covar, covar_inv_quad_form_root \u001b[38;5;241m@\u001b[39m covar_inv_quad_form_root\u001b[38;5;241m.\u001b[39mtranspose(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m2\u001b[39m), alpha\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m\n\u001b[1;32m 362\u001b[0m )\n\u001b[1;32m 363\u001b[0m )\n", + "File \u001b[0;32m~/miniconda3/envs/bo-hackathon/lib/python3.12/site-packages/gpytorch/models/exact_prediction_strategies.py:116\u001b[0m, in \u001b[0;36mDefaultPredictionStrategy._exact_predictive_covar_inv_quad_form_root\u001b[0;34m(self, precomputed_cache, test_train_covar)\u001b[0m\n\u001b[1;32m 103\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 104\u001b[0m \u001b[38;5;124;03mComputes :math:`K_{X^{*}X} S` given a precomputed cache\u001b[39;00m\n\u001b[1;32m 105\u001b[0m \u001b[38;5;124;03mWhere :math:`S` is a tensor such that :math:`SS^{\\top} = (K_{XX} + \\sigma^2 I)^{-1}`\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;124;03m :obj:`~linear_operator.operators.LinearOperator`: :math:`K_{X^{*}X} S`\u001b[39;00m\n\u001b[1;32m 113\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 114\u001b[0m \u001b[38;5;66;03m# Here the precomputed cache represents S,\u001b[39;00m\n\u001b[1;32m 115\u001b[0m \u001b[38;5;66;03m# where S S^T = (K_XX + sigma^2 I)^-1\u001b[39;00m\n\u001b[0;32m--> 116\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mtest_train_covar\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmatmul\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprecomputed_cache\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " ] } ], @@ -1490,35 +743,31 @@ "warnings.filterwarnings(\"ignore\", category=RuntimeWarning)\n", "\n", "# Create a new directory if it does not exist\n", - "isExist = os.path.exists(model_input.output_folder)\n", + "isExist = os.path.exists(model_input.OUT_FOLDER)\n", "if not isExist:\n", - " os.makedirs(model_input.output_folder)\n", - " print(\"The new directory is created!\", model_input.output_folder)\n", - " \n", - "# Commented out by NKT\n", - "# # Copy input parameters file to output folder\n", - "# shutil.copy2('surrogate_model_inputs.py',model_input.output_folder)\n", - "# Copy surrogate model file to output folder\n", - "shutil.copy2('surrogate_models.py',model_input.output_folder)\n", + " os.makedirs(model_input.OUT_FOLDER)\n", + " print(\"The new directory is created!\", model_input.OUT_FOLDER)\n", "\n", - "# Training a single GP for test\n", - "# a = train_gp(0)\n", + "shutil.copy2('surrogate_models.py',model_input.OUT_FOLDER)\n", "\n", - "# # Train the cluster of GP models in a parallel for loop\n", - "# best_observed_all_ei0 = Parallel(n_jobs=-1)(\n", - "# delayed(train_gp)(i) for i in range(num_cluster)\n", - "# )\n", + "# Send the print statements to a log file in the model output folder\n", + "print_file_path = model_input.OUT_FOLDER + 'log.txt'\n", + "sys.stdout = open(print_file_path, 'w')\n", "\n", "# Train each GP model sequentially first then apply the epsilon greed algorithm\n", - "for trial in range(1, model_input.n_trials + 1):\n", + "for trial in range(1, model_input.N_TRIALS + 1):\n", + " # Make a folder to save all trial data\n", + " os.makedirs(model_input.OUT_FOLDER + f'/trial_{trial}', exist_ok=True)\n", + "\n", " t0 = time.monotonic()\n", - " if model_input.random_seed == 'time':\n", + " if model_input.RANDOM_SEED == 'time':\n", " random_seed = int(t0)\n", - " elif model_input.random_seed == 'iteration':\n", + " elif model_input.RANDOM_SEED == 'iteration':\n", " random_seed = trial\n", "\n", - " print(f\"\\n -------------------- Trial {trial:>2} of {model_input.n_trials} --------------------\\n\", end=\"\")\n", + " print(f\"\\n -------------------- Trial {trial:>2} of {model_input.N_TRIALS} --------------------\\n\", end=\"\")\n", "\n", + " # Currently these are saved as np arrays at the end of the trial to the respective model output folder.\n", " best_observed_all_clusters = []\n", " X_train_all_clusters = []\n", " X_test_all_clusters = []\n", @@ -1526,111 +775,177 @@ " Y_test_all_clusters = []\n", " model_gps_all_clusters = []\n", "\n", - " # Now train each GP model sequentially first \n", - " ## TODO : This part can be parallizable using joblib\n", - " for cluster_idx in range(num_cluster):\n", - " print(f\"\\n -------------------- Cluster {cluster_idx:>2} of {num_cluster} --------------------\\n\", end=\"\")\n", + " # Creating the initial training and test sets for each cluster\n", + " for cluster_idx in range(model_input.NUM_CLUSTER):\n", " XX_desc = list(sample_dfs[cluster_idx].columns[:-1])\n", " YY_desc = sample_dfs[cluster_idx].columns[-1]\n", - " # (\n", - " # X_train,\n", - " # X_test,\n", - " # Y_train,\n", - " # Y_test,\n", - " # scalerX, \n", - " # scalerY\n", - " # ) = create_train_test_data(sample_dfs[cluster_idx][XX_desc].to_numpy(), sample_dfs[cluster_idx][YY_desc].to_numpy(), random_seed)\n", - " X_train, X_test, Y_train, Y_test, scalerX_transform, scalerY_transform = create_train_test_data(clustered_dfs[cluster_idx][XX_desc].to_numpy(), clustered_dfs[cluster_idx][YY_desc].to_numpy(), random_seed)\n", - " dump(scalerX_transform, os.path.join(model_input.output_folder, f'scalerX_{cluster_idx}.joblib'))\n", - " dump(scalerY_transform, os.path.join(model_input.output_folder, f'scalerY_{cluster_idx}.joblib'))\n", " (\n", - " best_observed_idx, \n", " X_train_idx,\n", - " X_test_idx, \n", - " Y_train_idx, \n", + " X_test_idx,\n", + " Y_train_idx,\n", " Y_test_idx,\n", - " model_gp_idx\n", - " ) = train_gp(X_train, X_test, Y_train, Y_test)\n", - " best_observed_all_clusters.append(best_observed_idx)\n", + " scalerX, \n", + " scalerY\n", + " ) = create_train_test_data(sample_dfs[cluster_idx][XX_desc].to_numpy(), sample_dfs[cluster_idx][YY_desc].to_numpy(), random_seed)\n", + " dump(scalerX, os.path.join(model_input.OUT_FOLDER + f'/trial_{trial}', f'scalerX_{cluster_idx + 1}.joblib'))\n", + " dump(scalerY, os.path.join(model_input.OUT_FOLDER + f'/trial_{trial}', f'scalerY_{cluster_idx + 1}.joblib'))\n", + " best_observed_all_clusters.append(None)\n", " X_train_all_clusters.append(X_train_idx)\n", " X_test_all_clusters.append(X_test_idx)\n", " Y_train_all_clusters.append(Y_train_idx)\n", " Y_test_all_clusters.append(Y_test_idx)\n", - " model_gps_all_clusters.append(model_gp_idx)\n", + " model_gps_all_clusters.append(None)\n", " \n", - " print(f'\\n')\n", - " print(f'Starting the epsilon greedy search')\n", - " print(f'\\n')\n", + " if model_input.NUM_CLUSTER > 1:\n", + " print(f'\\n')\n", + " print(f'Starting the initial search (training all local GPs.)')\n", + " print(f'\\n') \n", "\n", - " # # Now apply the epsilon greedy algorithm and choose which GP to train next\n", - " # for i in range(5):\n", - " # random_number = np.random.rand()\n", - " # epsilon = 0.1 # 10% exploration\n", - " # # Explore using the Epsilon Greedy Exploration Strategy\n", - " # if random_number <= epsilon:\n", - " # # Selecting a number between 1,2 and 3\n", - " # cluster_idx = np.random.choice(num_cluster)\n", - " # else:\n", - " # # Exploit best known action\n", - " # cluster_idx = np.argmax([best_observed_all_clusters[i][-1] for i in range(num_cluster)])\n", - " # print(f'Iteration {i} : Cluster {cluster_idx} is selected for training')\n", - " # (\n", - " # best_observed_idx,\n", - " # X_train_idx, \n", - " # X_test_idx,\n", - " # Y_train_idx,\n", - " # Y_test_idx,\n", - " # model_gp_idx\n", - " # ) = train_gp(X_train_all_clusters[cluster_idx], \\\n", - " # X_test_all_clusters[cluster_idx], \\\n", - " # Y_train_all_clusters[cluster_idx], \\\n", - " # Y_test_all_clusters[cluster_idx], \\\n", - " # model=model_gps_all_clusters[cluster_idx])\n", - " \n", - " # best_observed_all_clusters[cluster_idx].extend(best_observed_idx)\n", - " # X_train_all_clusters[cluster_idx] = X_train_idx\n", - " # X_test_all_clusters[cluster_idx] = X_test_idx\n", - " # Y_train_all_clusters[cluster_idx] = Y_train_idx\n", - " # Y_test_all_clusters[cluster_idx] = Y_test_idx\n", - " # model_gps_all_clusters[cluster_idx] = model_gp_idx\n" + " # Initially training of GPs done in parallel by using joblib\n", + " if model_input.TRAIN_PARALLEL:\n", + " results_cluster_0, results_cluster_1, results_cluster_2 = Parallel(n_jobs=-1)(delayed(train_gp)(X_train_all_clusters[i], X_test_all_clusters[i], Y_train_all_clusters[i], Y_test_all_clusters[i]) for i in range(model_input.NUM_CLUSTER))\n", + " results_all_clusters = [results_cluster_0, results_cluster_1, results_cluster_2]\n", + " else:\n", + " results_all_clusters = []\n", + " for i in range(model_input.NUM_CLUSTER):\n", + " print(f\"\\n -------------------- Cluster {i + 1:>2} of {model_input.NUM_CLUSTER} --------------------\\n\", end=\"\")\n", + " results_all_clusters.append(train_gp(X_train_all_clusters[i], X_test_all_clusters[i], Y_train_all_clusters[i], Y_test_all_clusters[i]))\n", + "\n", + " for i, results_i_cluster in enumerate(results_all_clusters):\n", + " best_observed_all_clusters[i] = results_i_cluster[0] \n", + " X_train_all_clusters[i] = results_i_cluster[1]\n", + " X_test_all_clusters[i] = results_i_cluster[2]\n", + " Y_train_all_clusters[i] = results_i_cluster[3]\n", + " Y_test_all_clusters[i] = results_i_cluster[4]\n", + " model_gps_all_clusters[i] = results_i_cluster[5] \n", + "\n", + " # Do the epsilon greedy search only if number of clusters is greater than 1\n", + " if (model_input.N_SEARCH > 1) and (model_input.NUM_CLUSTER > 1):\n", + " print(f'\\n')\n", + " print(f'Starting the epsilon greedy search (choosing which local GP to train next.)')\n", + " print(f'\\n') \n", + "\n", + " # Now apply the epsilon greedy algorithm and choose which GP to train next\n", + " # Subtracting 1 from the number of searches as the first search is already done\n", + " for i in range(model_input.N_SEARCH - 1):\n", + " random_number = np.random.rand()\n", + " epsilon = model_input.EPSILON\n", + " # Explore using the Epsilon Greedy Exploration Strategy\n", + " if random_number <= epsilon:\n", + " # Selecting a number between 1,2 and 3\n", + " cluster_idx = np.random.choice(range(model_input.NUM_CLUSTER))\n", + " search_text = 'exploring'\n", + " else:\n", + " # Exploit best known action\n", + " cluster_idx = np.argmax([best_observed_all_clusters[k][-1] for k in range(model_input.NUM_CLUSTER)])\n", + " search_text = 'exploiting'\n", + " print(f'\\nIteration {i} : Cluster {cluster_idx + 1} is selected for training ({search_text})')\n", + " results = train_gp(X_train_all_clusters[cluster_idx], \\\n", + " X_test_all_clusters[cluster_idx], \\\n", + " Y_train_all_clusters[cluster_idx], \\\n", + " Y_test_all_clusters[cluster_idx], \\\n", + " model=model_gps_all_clusters[cluster_idx])\n", + " \n", + " # Add the best values\n", + " best_observed_all_clusters[cluster_idx].extend(results[0])\n", + " # For the reamaining clusters, extend the last best value\n", + " for j in range(model_input.NUM_CLUSTER):\n", + " if j != cluster_idx:\n", + " best_observed_all_clusters[j].extend( list( np.ones((len(results[0]),))*best_observed_all_clusters[j][-1] ) )\n", + " X_train_all_clusters[cluster_idx] = results[1]\n", + " X_test_all_clusters[cluster_idx] = results[2]\n", + " Y_train_all_clusters[cluster_idx] = results[3]\n", + " Y_test_all_clusters[cluster_idx] = results[4]\n", + " model_gps_all_clusters[cluster_idx] = results[5]\n", + " \n", + " print(f'<----------- Search completed for trial {trial} ---------->\\n')\n", + "\n", + " # Save the numpy arrays\n", + "\n", + " if model_input.NUM_CLUSTER > 1:\n", + " filesave_name = 'all_clusters'\n", + " else:\n", + " filesave_name = 'one_cluster'\n", + " dump(best_observed_all_clusters, os.path.join(model_input.OUT_FOLDER + f'/trial_{trial}', f'best_observed_{filesave_name}.joblib'))\n", + " dump(X_train_all_clusters, os.path.join(model_input.OUT_FOLDER + f'/trial_{trial}', f'X_train_{filesave_name}.joblib'))\n", + " dump(X_test_all_clusters, os.path.join(model_input.OUT_FOLDER + f'/trial_{trial}', f'X_test_{filesave_name}.joblib'))\n", + " dump(Y_train_all_clusters, os.path.join(model_input.OUT_FOLDER + f'/trial_{trial}', f'Y_train_{filesave_name}.joblib'))\n", + " dump(Y_test_all_clusters, os.path.join(model_input.OUT_FOLDER + f'/trial_{trial}', f'Y_test_{filesave_name}.joblib'))\n", + " dump(model_gps_all_clusters, os.path.join(model_input.OUT_FOLDER + f'/trial_{trial}', f'model_gp_{filesave_name}.joblib'))\n" ] }, { "cell_type": "code", - "execution_count": 11, - "id": "c6e1bd01", + "execution_count": 16, + "id": "6623d178", "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3YAAAIjCAYAAABPmYl9AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAACA/ElEQVR4nO3deVhV5f7//9cGNjOiggM5m2OTQ2aKkWaJOKRJqV/LQrO0so5k5TmNkh5P/dI0Sy21lMyDpzLULMshh0ozMS1LkwaHTDFFcUCQYbN+f/DZW4lpw2baez8f18V1aK11r3Wv/b6XZ7+573XfJsMwDAEAAAAAnJZHdVcAAAAAAOAYEjsAAAAAcHIkdgAAAADg5EjsAAAAAMDJkdgBAAAAgJMjsQMAAAAAJ0diBwAAAABOjsQOAAAAAJycV3VXAIXl5eXp2LFjCgoKkslkqu7qAAAAAKgmhmHo/PnzuuKKK+ThUXy/HIldDXTs2DE1adKkuqsBAAAAoIY4cuSIGjduXOx+ErsaKCgoSFJ+8GrVqlWtdcnJydG6desUGRkps9lcrXVBxSGuroeYuh5i6pqIq+shpq6pJsX13LlzatKkiS1HKA6JXQ1kHX5Zq1atGpHY+fv7q1atWtXeqFFxiKvrIaauh5i6JuLqeoipa6qJcS3tFS0mTwEAAAAAJ+eUiV1GRoZWrlypMWPGqG3btvL19VVAQIA6dOigKVOmKD093e5zxcfHy2QylfqzZMmSEs+TnZ2tq666SiaTSV5edIQCAAAAqDpOmYEkJCTowQcflCS1b99egwYN0rlz57Rt2zZNnjxZy5Yt05YtW1S/fv1Sz9WqVSvFxMQUue/s2bNauXKlJOmmm24q8Tz/+c9/tH///rLdCAAAAABUAKdM7Mxms8aOHavY2Fi1b9/etj0lJUUDBgzQ7t27FRsbq4SEhFLPddNNNxWbtL355ptauXKlevTooZYtWxZ7jp9//lkvvfSSHnzwQS1YsKDsNwQAAAAADnDKoZgxMTGaP39+gaROksLCwjR37lxJUmJiorKzsx26ztKlSyVJ9957b7HHGIahsWPHqnbt2nr55Zcduh4AAAAAlIdTJnYl6dChgyQpKytLp06dKvd5Dh48qG3btsnb21vDhg0r9rj58+fr66+/1quvvqo6deqU+3oAAAAAUF5OORSzJAcOHJCUP1yzbt265T6PtbduwIABxSZsKSkp+te//qVbb71VI0eOLPe1KophGMrJyVFeXl6FnTMnJ0deXl66ePGiLBZLhZ0X1auy4+rp6VljpgYGAABwBy6X2M2ePVuSFBUVJR8fn3Kfx55hmI8++qguXryoefPmlesaKSkpSklJKbTdOqtnTk6OcnJySj1Pdna2Tp06pYyMjApN6qT8ZLFhw4b6448/Sl07A86jKuLq7e2tunXrlrqYJiqG9d8Ke/7NgHMgpq6JuLoeYuqaalJc7a2DyTAMo5LrUmXWrFmjgQMHysvLS0lJSbZhmWW1Y8cO3Xjjjapbt65SUlLk7e1d6JhVq1bpjjvu0OTJkxUXF2fbbjKZ5Onpqdzc3FKvExcXpxdffLHY/QkJCfL39y/xHGazWSEhIQoMDFRQUJC8vb3l4eFyI2zhZHJzc3XhwgWdPXtWqampysrKqu4qAQAAOKWMjAzdfffdOnv2rGrVqlXscS6T2O3fv1/h4eFKS0vTa6+9pgkTJpT7XP/4xz/0xhtv6KGHHtKbb75ZaP/58+d11VVXyd/fX3v27CnQM1iWxK6kHruePXsqNTW1xOBJ0tGjR5WXl6cmTZrI09PTjrsrG8MwdP78eQUFBdFj50KqIq6GYejo0aPKyclR06ZNK+UauCQnJ0fr169Xnz59GAbrIoipayKuroeYuqaaFNdz584pNDS01MTOJYZiHj16VFFRUUpLS9PEiRMdSupyc3P1/vvvSyp+GOYzzzyjP//8Uxs2bHBouGdYWJjCwsIKbT937pyk/N64khpSbm6uMjMzFRYWVmkNzjq002Qy0RPoQqoqrrVr19bRo0clqdr/UXQXpf27AedDTF0TcXU9xNQ11YS42nt9p0/sTp8+rcjISB0+fFijR4/WjBkzHDrfunXrdOLECbVs2VLh4eFFHrN69Wr5+vpq6tSpmjp1aqH9FotFvXr1kiS99tpr6tixo0N1Ko61V9CR5BKoTNZ/iCwWS7X/owgAAODKnDqxS09PV79+/bRv3z5FR0dr4cKFDg8rs06aUtoslxcvXtSWLVuK3W/dd+bMGYfqYw+GSKKmom0CAABUDadN7LKysjR48GDt2LFDffv21bJlyxx+xyw9PV2rVq2SVHJid+jQoWL3leUdOwAAAACoCE750pTFYtGIESO0ceNGRUREKDExsciZKy83Z84ctWvXTk8//XSxxyQmJiojI0PdunVT69atK7raAAAAAFApnLLHbs6cOVqxYoUkKTQ0VI888kiRx82YMUOhoaGSpNTUVCUnJxc5C6WVPWvXAQAAAEBN45SJXVpamu13a4JXlLi4OFtiV5qUlBRt3LhRZrNZw4cPd7iOqH4XLlzQ/PnztXr1au3bt09paWkKCAhQu3bt1KdPHz3wwAMFpuG3riu4ePFijRo1qvoqXkPs3btXcXFx2rx5s9LT09WqVSuNGTNG//jHP5ghFQDgVL75/ZQOpKZXyrktFot++suks0lHKmXpKVQPi8Wig6dM6l/dFSkDp0zs4uLiCiwKXhFlwsLCKuS9OBdZFtDpbdu2TXfeeaeOHz8uf39/devWTQ0aNNDZs2eVlJSk7du365VXXtEnn3yi2267rVrr2qtXL23ZskUHDx5U8+bNq7UuVt98841uvfVWZWZmqmvXrmrevLm+/PJLPf7449q2bZvef/99JkYBADiFP9MyNGLh9kq+iqc+OPBzJV8DVa1xgIeeqe5KlIFTJnZASb7//nvdeuutunjxov75z3/q+eefV0BAgG1/Xl6eVq5cqUmTJunPP/+sxprWTDk5ObrnnnuUmZmpmTNn6vHHH5eUP7lQZGSkPvzwQ/Xv359eTQCAUzhxPkuS5Gf21M1t7BvJVRZ5eYb++uu4GjRoKA8P/ujpKvLyDOWeOV7d1SgTEju4FMMwdO+99+rixYuKi4vT5MmTCx3j4eGh6Oho3XrrrTpy5Eg11LJmW7FihQ4ePKgOHTrYkjpJCgwM1Jw5c3T99dfr1VdfJbEDADiF7Nw8SdIVtX01/94uFX7+nJwcrVmzRv37d2TNVhdijasz4UUZuJTPP/9cP/30kxo3bqxnn322xGODg4N1zTXXlHrO5s2bFzvscPPmzTKZTIWSnOzsbM2bN0833HCDQkJC5O/vr+bNm2vgwIH63//+Jyl/2QyTyWRb87BFixYymUy2n8sZhqFly5apd+/eqlOnjnx9fdW+fXvFxcUpIyOjUL169eolk8mkQ4cOKSEhQd26dVNQUJBq165d6v1++umnkqS77rqr0L7OnTurZcuW+umnn0pc9gMAgJoi6/8SO28v3n+Da6PHDi7FmpQMHTpUXl7V17zvueceLV++XEFBQYqIiFCtWrV09OhRff3110pPT9f/+3//T4GBgYqJidHnn3+uv/76S3feeacCAwMLnSsvL08jR47UsmXLFBgYqC5duqhOnTrauXOnXnzxRX322WfavHmz/Pz8CpV96aWX9Pbbb6tHjx4aOHCgXT2UP/zwg6T8JK4onTt31oEDB7Rnz54a804gAADFybYldvRnwLWR2LkwwzCUmWNx6Bx5eXnKzLbIKzu30mdC9DN7Ojwhx/fffy+p+KSkKhw8eFDLly9Xs2bN9N133ykkJMS27+LFi9q9e7ek/KU64uPj1atXL/3111+aMWNGkYnSq6++qmXLlqlXr15atmyZGjZsKCm/V/CRRx7RO++8oxdffFEvv/xyobJLlizRxo0b1bNnT7vr/8cff0iSGjduXOR+6/bDhw/bfU4AAKqLNbHz8SSxg2sjsXNhmTkWXfXC2uquht32Tekrf2/HmuSpU6ckSfXq1auIKpXLyZMnJUmdOnUqkNRJkq+vr7p37273uXJzc/XKK68oICBA//vf/9SgQQPbPm9vb73xxhv69NNPtWDBAv3nP/8plHyPGTOmTEmdlD9JiiT5+/sXud86Ec358+fLdF4AAKpDtiX/j9w+ZhI7uDZaOFDB2rVrp4CAAH366aeaPn26jh07Vu5z7dq1S6mpqQoPDy+Q1Fn5+fnp+uuvV1pamn799ddC+wcNGlTuawMA4ApsQzHpsYOLo8fOhfmZPbVvSl+HzpGXl6fz584rqFZQlQzFdJS1h8zaa1YdatWqpYULF2rs2LGaNGmSJk2apDZt2uiWW27Rvffeqx49eth9LusEJevXry91mGpqaqratm1bYNvlC7DbKzAwUGlpaUVOyiLlL/wuSUFBQWU+NwAAVY137OAuSOxcmMlkcnhoY15ennK9PeXv7VXpiV1F6Nixo7Zu3apdu3Zp5MiRlX69vLy8IrePGDFCt912m1atWqV169Zpy5Ytmj9/vubPn6+JEyfq1VdfLdP5W7VqVWpC+Pdhn1L+0M+yatq0qdLS0vTnn3/quuuuK7TfuvZfs2bNynxuAACqWhaJHdwEiR1cyoABAzR37lx9+OGHeuWVVypkZkxvb29J+e+e/X3WypJmmaxXr54eeOABPfDAAzIMQ2vXrtXw4cM1c+ZM3X///br66qtLvbZ1opJ27dopPj6+/DdRBh06dNAPP/ygXbt2qX///oX279q1S5KKTPoAAKhpshiKCTdBC4dLiYqK0tVXX60///xT06ZNK/HYc+fOae/evaWeMywsTJL0yy+/FNq3fv16u+plMpkUFRWlAQMGSFKB61oTx9zc3ELlbrjhBgUHB2vLli06ffq0XddylLWOy5cvL7Rv9+7dOnDggK655hqWOgAAOAWGYsJd0MLhUkwmk5YuXSpfX1/FxcXp6aeftr0TZmUYhj7++GN16dJFSUlJpZ7TOqvkSy+9JIvl0vIRy5Yt07Jlywodv3v3biUmJio7O7vA9tOnT+vbb7+VJDVp0sS2/YorrpAkJScnFzqXj4+PJk2apPPnzys6OloHDhwodMzRo0f13nvvlXof9hoyZIhatGihH374QbNmzbJtv3DhgsaPHy9JeuKJJyrsegAAVKZsy/8td8AC5XBxDMWEy+nYsaM2bNigO++8Uy+//LJef/11de/eXQ0aNNDZs2e1c+dO/fXXX/L19S2QYBVn/Pjxeuutt7R8+XJdddVVuu666/Trr7/qp59+0oQJEwokP1L++m533nmngoOD1aVLFzVs2FBnzpzRl19+qfPnz+v2228vsOTBoEGD9O677+ruu+9WZGSkgoODJUlvv/22JOlf//qX9u/fr/fee0/t27dXp06d1KJFC2VnZys5OVn79u3Tddddp3vvvbdCPj+z2aylS5fqtttu08SJE/X++++rWbNm+uqrr5SSkqK77rpLMTExFXItAAAqGz12cBe0cLikHj166LffftOMGTN0ww03aM+ePfrggw+0detWNW/eXJMnT9avv/6qW2+9tdRzNWjQQF9++aUGDhyolJQUffbZZwoODtb69euLXE6gW7du+ve//63rr79eycnJ+vDDD7Vz505dd911WrRokT766KMCx0dHR2vWrFlq3LixVq9erXfeeUfvvPOObb+Hh4eWLFmiVatWqU+fPjp48KA++ugjff311/L19dVTTz2lRYsWOf6hXSY8PFxJSUm688479dtvv+njjz9W3bp1NXPmTL3//vsOLyQPAEBVIbGDuzAZhmFUdyVQ0Llz5xQcHKyzZ8+qVq1axR538eJFHTx4UC1atCjX7If2yMvL07lz51SrVi2nmBUT9qmquFZFG0W+nJwcrVmzRv3795fZbK7u6qACEFPXRFyr3qTlP+iDnX/qqb5tNf6WVhV+fmLqmmpSXO3NDfimDgAAAJfFAuVwF7RwAAAAuCzr5CkMxYSro4UDAADAZfGOHdwFLRwAAAAuy7pAuQ+JHVwcLRwAAAAuix47uAtaOAAAAFyW7R07Jk+Bi6OFAwAAwGVl5dBjB/dAC3cBLEWImoq2CQCobsyKCXdBC3diXl5ekqSsrKxqrglQtJycHEmSp6dnNdcEAOCuspk8BW6CFu7EvLy8FBAQoNOnT8tisVR3dYACDMPQ2bNn5ePjI7PZXN3VAQC4qUuJHX9khGvzqu4KwDGhoaE6cuSIDh48qODgYPn5+cnT01Mmk6lCzp+Xl6fs7GxdvHhRHh78HcBVVGZcDcNQTk6Ozp49q/T0dDVq1KhCzw8AQFkwFBPugsTOyfn7+6tFixY6ceKE0tLSlJqaWqHnNwxDmZmZ8vPzq7BkEdWvKuLq4+OjRo0aqVatWpVyfgAA7GFb7oBZMeHiSOxcgLe3txo3bmzrKcnLy6uwc+fk5OjLL7/UzTffzHA6F1LZcfX09KS9AABqhKzc/NdV6LGDqyOxcyEmk0ne3t4Vek5PT0/l5ubK19eXL+ouhLgCANxBXp6hHEv+DM0kdnB1tHAAAAC4JOv7dRKJHVwfLRwAAAAu6fLEjuUO4Opo4QAAAHBJ1olTJCZPgeujhQMAAMAlXT4jJrN7w9WR2AEAAMAlZeWyhh3cB60cAAAALimbxA5uhFYOAAAAl8Ti5HAntHIAAAC4pGwLi5PDfdDKAQAA4JKs79ix1AHcAa0cAAAALol37OBOaOUAAABwSSR2cCe0cgAAALikLCZPgRtxylaekZGhlStXasyYMWrbtq18fX0VEBCgDh06aMqUKUpPT7f7XPHx8TKZTKX+LFmyxFYmJydH69at06OPPqprrrlG/v7+8vPzU/v27fXkk0/q5MmTlXHbAAAAKAN67OBOvKq7AuWRkJCgBx98UJLUvn17DRo0SOfOndO2bds0efJkLVu2TFu2bFH9+vVLPVerVq0UExNT5L6zZ89q5cqVkqSbbrrJtn3Lli3q27evJKl58+bq16+fcnJy9M033+jVV1/Vf//7X23evFlt27Z18E4BAABQXtkWJk+B+3DKxM5sNmvs2LGKjY1V+/btbdtTUlI0YMAA7d69W7GxsUpISCj1XDfddFOBpO1yb775plauXKkePXqoZcuWtu0eHh4aNmyYnnjiCXXt2tW2/ezZsxo+fLjWrl2r0aNHa9u2bQ7cJQAAAByRbZsV07OaawJUPqdM7GJiYorsZQsLC9PcuXMVHh6uxMREZWdny9vbu9zXWbp0qSTp3nvvLbC9d+/e6t27d6Hjg4ODtWjRIjVq1EjffPONDh8+rGbNmpX7+gAAACg/hmLCnbhcK+/QoYMkKSsrS6dOnSr3eQ4ePKht27bJ29tbw4YNs7vcFVdcoXr16kmSjh07Vu7rAwAAwDHWoZhMngJ34HKt/MCBA5Lyh2vWrVu33Oex9tYNGDBAderUsbvcmTNnlJaWJklq2LBhua8PAAAAx2TlWCTRYwf34JRDMUsye/ZsSVJUVJR8fHzKfZ7ihmGWZu7cucrNzdW1116rFi1alHhsSkqKUlJSCm23zuqZk5OjnJycMl2/olmvX931QMUirq6HmLoeYuqaiGvVyszOlSR5eVTeZ05MXVNNiqu9dTAZhmFUcl2qzJo1azRw4EB5eXkpKSnJNiyzrHbs2KEbb7xRdevWVUpKit3v6e3evVvh4eG6ePGi1qxZo379+pV4fFxcnF588cVi9yckJMjf379MdQcAAEC+xIMe2nLcQ7ddkafbm+VVd3WAcsnIyNDdd9+ts2fPqlatWsUe5zI9dvv379fIkSNlGIamT59e7qROutRbN2zYMLuTur/++kvR0dG6ePGiYmNjS03qJGncuHEaNGhQoe3p6enq2bOnIiMjSwxeVcjJydH69evVp08fmc3maq0LKg5xdT3E1PUQU9dEXKvW9o/3Scf/1FVtW6t/7ysr5RrE1DXVpLieO3fOruNcIrE7evSooqKilJaWpokTJ2rChAnlPldubq7ef/99SfYPwzx//rz69++vQ4cOaejQoXr11VftKhcWFqawsLBC263BM5vN1d6QrGpSXVBxiKvrIaauh5i6JuJaNf5vUkz5+nhV+udNTF1TTYirvdd3+jdJT58+rcjISB0+fFijR4/WjBkzHDrfunXrdOLECbVs2VLh4eGlHn/x4kUNGjRIu3btUmRkpJYuXSoPD6f/WAEAAJwes2LCnTh1K09PT1e/fv20b98+RUdHa+HChTKZTA6d0zoMc+TIkaUem5ubq+HDh2vz5s22tfMcWTcPAAAAFefSAuVO/ZUXsIvTtvKsrCwNHjxYO3bsUN++fbVs2TJ5eno6dM709HStWrVKUumJnWEYGj16tD7++GN17NhRn376qQICAhy6PgAAACpOFguUw404ZSu3WCwaMWKENm7cqIiICLt6yubMmaN27drp6aefLvaYxMREZWRkqFu3bmrdunWJ54uNjdXSpUvVrl07rVu3TrVr1y7PrQAAAKCSZJPYwY045eQpc+bM0YoVKyRJoaGheuSRR4o8bsaMGQoNDZUkpaamKjk5uch146zsXbtu1apVev311yVJTZo00VNPPVXkcf/617/Url27km8GAAAAlcKW2Dk4qgtwBk6Z2KWlpdl+tyZ4RYmLi7MldqVJSUnRxo0bZTabNXz4cLuvv379+mKPGzVqFIkdAABANcmy8I4d3IdTtvK4uDgZhlHqT/PmzQuViY+PL/KcYWFhys3NVXZ2tkJCQkq8/qhRo+y6fq9evSrupgEAAFAmDMWEO6GVAwAAwCVl51okkdjBPdDKAQAA4JKYFRPuhFYOAAAAl3Rp8hS+8sL10coBAADgkrKZPAVuhFYOAAAAl2TtsfPxYrkDuD4SOwAAALgkZsWEO6GVAwAAwOVY8gzl5hmSSOzgHmjlAAAAcDnW3jqJxA7ugVYOAAAAl1MgsWNWTLgBWjkAAABcTpbFYvvd7GmqxpoAVYPEDgAAAC7n0oyYHjKZSOzg+kjsAAAA4HKYERPuhpYOAAAAl8Pi5HA3tHQAAAC4HFuPHROnwE3Q0gEAAOByshiKCTdDSwcAAIDL4R07uBtaOgAAAFwOiR3cDS0dAAAALifLttyBZzXXBKgaJHYAAABwOdZZMZk8Be6Clg4AAACXw1BMuBtaOgAAAFxOVq5FEokd3ActHQAAAC6HHju4G1o6AAAAXI41sfPhHTu4CVo6AAAAXI4tsTPzdRfugZYOAAAAl8OsmHA3tHQAAAC4HN6xg7uhpQMAAMDlZJHYwc3Q0gEAAOBybImdp2c11wSoGiR2AAAAcDkMxYS7oaUDAADA5VgnT/EhsYOboKUDAADA5WTnWiTRYwf3QUsHAACAy2EoJtwNLR0AAAAuh6GYcDe0dAAAALicrBwWKId7oaUDAADA5Vh77BiKCXdBSwcAAIDL4R07uBtaOgAAAFyONbHz8WKBcrgHEjsAAAC4nCx67OBmaOkAAABwObZ37Jg8BW6Clg4AAACXk5XDAuVwL7R0AAAAuBzWsYO7ccqWnpGRoZUrV2rMmDFq27atfH19FRAQoA4dOmjKlClKT0+3+1zx8fEymUyl/ixZsqRQWYvFolmzZunaa6+Vn5+f6tWrp2HDhunnn3+uyNsFAABAGTErJtyNV3VXoDwSEhL04IMPSpLat2+vQYMG6dy5c9q2bZsmT56sZcuWacuWLapfv36p52rVqpViYmKK3Hf27FmtXLlSknTTTTcV2JeXl6ehQ4dqxYoVql27tgYMGKDU1FQtX75cn376qTZt2qSuXbs6dqMAAAAos1xLnvKM/N/psYO7cMrEzmw2a+zYsYqNjVX79u1t21NSUjRgwADt3r1bsbGxSkhIKPVcN910U6GkzerNN9/UypUr1aNHD7Vs2bLAvkWLFmnFihVq3bq1vvrqKzVo0ECS9NFHH+muu+7SPffco59//lleXk75EQMAADgt6zBMiR47uA+nbOkxMTGaP39+gaROksLCwjR37lxJUmJiorKzsx26ztKlSyVJ9957b6F9M2fOlCS98sortqROku68804NGjRIv/32m1atWuXQ9QEAAFB21mGYErNiwn24XEvv0KGDJCkrK0unTp0q93kOHjyobdu2ydvbW8OGDSu07+eff5afn58GDBhQqOxdd90lSVq9enW5rw8AAIDysa5h52GSvEjs4CZcrqUfOHBAUv5wzbp165b7PNbeugEDBqhOnToF9v3www+SpGuuuUZms7lQ2c6dO0uS9uzZU+7rAwAAoHyYOAXuyOVa++zZsyVJUVFR8vHxKfd5ShqG+ccff0iSGjduXGRZ6/bDhw+X+/oAAAAoH2uPHcMw4U5camaPNWvW6J133pHZbNbUqVPLfZ4dO3bol19+Ud26dYscamldTsHf37/I8gEBAZKk8+fPl3idlJQUpaSkFHv+nJwc5eTklKnuFc16/equByoWcXU9xNT1EFPXRFyrRsbF/HkWfLw8Kv2zJqauqSbF1d46uExit3//fo0cOVKGYWj69Om2d+3Kw9pbN2zYMHl7e1dUFQuZP3++XnzxxWL3r1u3rtjksaqtX7++uquASkBcXQ8xdT3E1DUR18p1+LwkecmSk6U1a9ZUyTWJqWuqCXHNyMiw6ziXSOyOHj2qqKgopaWlaeLEiZowYUK5z5Wbm6v3339fUtHDMCUpMDBQUvEf8oULFyRJQUFBJV5r3LhxGjRoUKHt6enp6tmzpyIjI1WrVi27614ZcnJytH79evXp06fI9wnhnIir6yGmroeYuibiWjWSDqVJPyUpODBA/fsXvaxVRSGmrqkmxfXcuXN2Hef0id3p06cVGRmpw4cPa/To0ZoxY4ZD51u3bp1OnDihli1bKjw8vMhjmjZtKkn6888/i9xv3d6sWbMSrxUWFqawsLBC263BM5vN1d6QrGpSXVBxiKvrIaauh5i6JuJaufL+bxoJH7NnlX3OxNQ11YS42nt9p36jND09Xf369dO+ffsUHR2thQsXymQyOXRO6zDMkSNHFnuMdZjnTz/9VOSY1127dkmSrrvuOofqAgAAgLLLyrVIYlZMuBenbe1ZWVkaPHiwduzYob59+2rZsmXy9PR06Jzp6em2RcVLSuxatGih9u3bKzMzU59++mmh/cuXL5ck3X777Q7VBwAAAGWXzayYcENO2dotFotGjBihjRs3KiIiQomJiaVOcjJnzhy1a9dOTz/9dLHHJCYmKiMjQ926dVPr1q1LPN/EiRMlSZMmTdKJEycKnOPjjz9Wq1atNHjw4DLcFQAAACpCtoV17OB+nPIduzlz5mjFihWSpNDQUD3yyCNFHjdjxgyFhoZKklJTU5WcnFzk8gJWJa1d93f333+/1qxZoxUrVqhdu3a69dZblZqaqi1btsjPz09Lly6Vl5dTfrwAAABOzbqOnQ+JHdyIU2YeaWlptt+tCV5R4uLibIldaVJSUrRx40aZzWYNHz681OM9PDz04Ycfavbs2Vq0aJE++eQTBQQE6M4779SLL76oq666yq7rAgAAoGLZhmKS2MGNOGVrj4uLk2EYpf40b968UJn4+PgizxkWFqbc3FxlZ2crJCTErnp4enpq4sSJ+umnn5SZmanU1FR9+OGHJHUAAADV6FJi59j8C4AzccrEDgAAAChOFpOnwA3R2gEAAOBSGIoJd0RrBwAAgEvJtuSvY8fkKXAntHYAAAC4lGxmxYQborUDAADApTAUE+6I1g4AAACXYlugnMlT4EZo7QAAAHApWTn02MH90NoBAADgUrIsJHZwP7R2AAAAuBTesYM7orUDAADApVyaFdOzmmsCVB0SOwAAALgUeuzgjmjtAAAAcCnMigl3RGsHAACAS2GBcrgjWjsAAABcSlauRRJDMeFeaO0AAABwKbxjB3dEawcAAIBLsSV2vGMHN0JrBwAAgEuxTp7iY+arLtwHrR0AAAAuJYseO7ghWjsAAABcCu/YwR3R2gEAAOAyDMO41GNHYgc3QmsHAACAy8ixGLbffTw9q7EmQNUisQMAAIDLsE6cItFjB/dS4a09Ly9Pqamp+uOPP2SxWCr69AAAAECxrO/XSSR2cC8V0totFoveeecdRUREyN/fXw0aNFDLli2VnJxc4LhPPvlEkyZN0rRp0yrisgAAAEAB1sTOy8MkTw9TNdcGqDpejp7gxIkTuuOOO/Ttt9/KMIwSj23evLkGDRokk8mkAQMGqGPHjo5eHgAAALBhRky4K4davMVi0e23367t27fLZDJp2LBhmjNnTrHHX3PNNbrxxhslSStWrHDk0gAAAEAhWbn5rwKR2MHdONTi3333XSUlJclsNuvTTz/V//73Pz3yyCMllhk0aJAMw9DXX3/tyKUBAACAQlicHO7KoRa/bNkymUwmjRs3Tn379rWrTKdOnSSp0Pt3AAAAgKOss2LSYwd341CL37Nnj6T8Xjh71a9fX5J06tQpRy4NAAAAFGJ9x86HxA5uxqEWf+bMGUlSSEiI3WWsSyB4smAkAAAAKtilyVP4rgn34lBiV7duXUnSkSNH7C7z66+/SpLq1avnyKUBAACAQpgVE+7KoRZ/9dVXS5KSkpLsLvP+++/LZDLphhtucOTSAAAAQCHWd+x8mDwFbsahFn/HHXfIMAzNmTNHaWlppR6/fPlyrV69WpJ05513OnJpAAAAoBCWO4C7cqjFP/jgg2ratKnOnTunyMhI7du3r8jjTpw4oWeffVZ33323TCaTrrnmGg0bNsyRSwMAAACFMBQT7srLkcI+Pj5atWqVevXqpe+++07XXnut2rZta9s/cuRIpaen68CBAzIMQ4ZhKCQkRB999JFMJpPDlQcAAAAul806dnBTDrf4Dh06KCkpSd27d5dhGNq/f79t3w8//KDffvtNeXl5MgxDXbt21bfffqtWrVo5elkAAACgEOsC5T5mEju4F4d67KxatWqlrVu36uuvv9bHH3+snTt36sSJE7JYLAoJCVGnTp00aNAg9enTpyIuBwAAABTJtkA5PXZwMxWS2FnddNNNuummmyrylAAAAIDdeMcO7ooWDwAAAJeRRWIHN0WLBwAAgMugxw7uyqGhmH/88YdDF2/atKlD5QEAAIDLWRM7FiiHu3EosWvRokW5y5pMJuXm5parbEZGhtatW6fVq1fr66+/1uHDh+Xp6alWrVrpzjvv1MSJExUYGFjm8x46dEgvv/yy1q5dq2PHjikoKEitW7dWdHS0nnrqqULHHzt2TP/5z3/0+eef68iRI7Y6DBkyRE8++aSCgoLKdX8AAAAoH1tiZ/as5poAVcuhP2VY16Yr7095JSQkaMiQIVq0aJE8PT01aNAgRURE6ODBg5o8ebJuuOEGnThxokzn/Oyzz3T11VdrwYIFCgkJUXR0tDp37qxDhw5p/vz5hY7/9ddf1bFjR82dO1cWi0UDBw7ULbfcoiNHjmjKlCnq1q2bzp49W+57BAAAQNkxKybclUM9dosXLy71mAsXLuiXX37RRx99pKNHj6pHjx564IEHHLmszGazxo4dq9jYWLVv3962PSUlRQMGDNDu3bsVGxurhIQEu863f/9+RUdHKygoSOvXr1d4eLhtX15ennbt2lWozD//+U+dPHlSjzzyiF5//XV5eub/Vejs2bOKiorS9u3bNXPmTL344osO3SsAAADsxzt2cFcOJXYxMTF2Hzt9+nQ9/vjjevPNN9WjRw+9/PLLDl23qGuHhYVp7ty5Cg8PV2JiorKzs+Xt7V3q+SZOnKiLFy/qo48+KpDUSZKHh4e6dOlSqMyXX34pSXr++edtSZ0kBQcHa9KkSYqOjlZSUlJZbw0AAAAOyMq1SCKxg/upshZvNps1Z84c9erVS9OnT9fatWsr5TodOnSQJGVlZenUqVOlHn/kyBGtXbtWLVu2VP/+/e2+jo+PT6nHhISE2H0+AAAAOM623AFDMeFmqrzFjxs3ToZh6I033qiU8x84cEBSfiJZt27dUo/fvHmz8vLyFB4ertzcXH3wwQeaMGGCHn30Ub311ltKS0srslxkZKQkaerUqbJYLLbtZ8+e1SuvvCJJuv/++x29HQAAAJQBQzHhrhwailkerVu3liTt3LmzUs4/e/ZsSVJUVJRdvWr79u2TJAUGBioiIkLbt28vsP/ZZ5/V8uXLdcsttxTY/tJLL+m7777TvHnztGbNGl1//fW6ePGitm7dKl9fXy1durRQGQAAAFQu6+QpPiR2cDNVnthZZ4qsjBkj16xZo3feeUdms1lTp061q4y1R+7tt99WYGCgEhISFBUVpZMnT2rq1KlaunSphgwZor1796pRo0a2cg0bNtTmzZs1YsQIrVu3TocOHbLti46O1vXXX1/qtVNSUpSSklJoe3p6uiQpJydHOTk5dt1HZbFev7rrgYpFXF0PMXU9xNQ1EdfKl5WTP5LK02RUyedMTF1TTYqrvXUwGY6sO1AOo0eP1rvvvqvmzZvbhk1WhP379ys8PFxpaWl67bXXNGHCBLvKjR07VgsXLpQkvf/++xo2bFiB/V27dlVSUpKeeeYZTZs2zbZ9z549GjBggDw9PfX666/r5ptv1oULF7R8+XI9/fTT8vPz07Zt29S2bdtirx0XF1firJkJCQny9/e36z4AAAAgvfS9p45nmvToVRa1Dq7Sr7lApcjIyNDdd9+ts2fPqlatWsUeV2U9dr/++qteffVVvfvuuzKZTGWaqKQ0R48eVVRUlNLS0jRx4kS7kzpJtoXMAwMDNXTo0EL7R48eraSkJG3ZssW2LScnR3fddZeOHTumpKQkde7cWZJUu3ZtTZgwQRaLRU888YReeOEFvf/++8Vee9y4cRo0aFCh7enp6erZs6ciIyNLDF5VyMnJ0fr169WnTx+ZzeZqrQsqDnF1PcTU9RBT10RcK9+ryV9JmZmK6NFdnZvWrvTrEVPXVJPieu7cObuOcyixa9myZanH5OXl6cyZMzp//rxtW/369fXss886cmmb06dPKzIyUocPH9bo0aM1Y8aMMpVv1qyZJKlp06YymUyF9jdv3lySCix4vn37dv3666+68sorbUnd5YYOHaonnnjCtiRCccLCwhQWFlZouzV4ZrO52huSVU2qCyoOcXU9xNT1EFPXRFwrT3Zufi+dv493lX7GxNQ11YS42nt9hxK7y98rs1f37t21aNGiIhOaskpPT1e/fv20b98+RUdHa+HChUUmZyXp1KmTJBU7++Xp06clXerZk6Q///xTUv6adUWxbi/unAAAAKgc1slTmBUT7qbSFyj38PBQUFCQWrRooZ49e6pjx46OXNImKytLgwcP1o4dO9S3b18tW7aswELh9goPD1dISIiOHz+u5OTkQu/EWYdgWhNAKX/iFElKTk7W+fPnFRQUVKCMdWFya28fAAAAqgbLHcBdOZTYLV68uKLqUSYWi0UjRozQxo0bFRERocTERHl7e5dYZs6cOZozZ46GDBmil156ybbdy8tLEydO1LPPPqvx48crMTHR9l7bhg0bFB8fL5PJpHHjxtnKdO/eXfXr19eJEyf06KOPasGCBbalFY4dO6bHH39cknTXXXdV9K0DAACgBNbEjuUO4G6qfLmDijBnzhytWLFCkhQaGqpHHnmkyONmzJih0NBQSVJqaqqSk5OLXF7gqaee0qZNm7Rhwwa1adNG3bp1U2pqqrZv3y6LxaJp06apa9eutuN9fX01f/58DR06VEuWLNEXX3yhLl26KDMzU998843Onz+vzp0761//+lcl3D0AAACKYhgGQzHhtpwysbv83TVrgleUuLg4W2JXErPZrDVr1mjWrFlasmSJ1q5dK29vb/Xs2VOPP/64Bg4cWKjMHXfcoR07dmjGjBn68ssvtWbNGnl7e6t169YaNmyYYmNj5efnV74bBAAAQJlZkzqJxA7uxykTu7i4OMXFxVVoGbPZrEmTJmnSpEl2n7NTp07673//W6Z6AAAAoHJk5V6W2HmS2MG92JXYLVmypFIuft9991XKeQEAAOB+skns4MbsSuxGjRpV5mUESmMymUjsAAAAUGGsiZ3Z0yQPj4r97grUdHYPxTQMozLrAQAAADjk0oyYZV8CC3B2diV2Bw8erOx6AAAAAA5hRky4M7sSu2bNmlV2PQAAAACH2BYn5/06uCFaPQAAAFxCVq5FEj12cE+0egAAALgE63IHJHZwR7R6AAAAuASGYsKdVdgC5b///rs+/vhj/fDDD0pNTVVmZmaJM2maTCZ98cUXFXV5AAAAuLlseuzgxhxO7DIyMjR+/Hi99957hRI5wzAKrX9nPaai18UDAACAe7POiulDYgc35FBiZxiGhgwZog0bNsgwDIWGhqpx48b6/vvvZTKZFBERodOnTys5OVm5ubkymUxq27atGjZsWFH1BwAAACTRYwf35lCr//DDD7V+/XpJ0uTJk3X8+HEtWbLEtn/Lli368ccflZaWppkzZyogIECnT5/W1KlTtWnTJsdqDgAAAFzm0gLlJHZwPw61+oSEBElS9+7dNXnyZHl4eBQ5xDIgIECxsbH64osvdP78eUVHR+vYsWOOXBoAAAAogFkx4c4cavU7d+6UyWTSgw8+aNfxN9xwgx5++GGlpqbq9ddfd+TSAAAAQAHMigl35lCrT01NlSS1bNnSts1sNtt+z8zMLFRmwIABkqRPPvnEkUsDAAAABVgnT6HHDu7IoVbv5ZU/90pQUJBt2+W/Hz9+vFCZ4OBgSdKRI0ccuTQAAABQQJbtHTvPaq4JUPUcSuyuuOIKSdLJkydt2xo2bCg/Pz9J0q5duwqV+fXXXyVJubm5jlwaAAAAKIBZMeHOHGr1HTp0kCT9+OOPtm0mk0k33nijJGnevHkFjs/JydHMmTMlSa1bt3bk0gAAAEABJHZwZw61+t69e8swDH3++ecFtt9///0yDEObN29Wr169NHfuXL3yyivq2rWrbcKVYcOGOVRxAAAA4HJZuRZJTJ4C9+RQqx8yZIhMJpM2bdqkAwcO2LaPHDlSUVFRMgxDX331lf7xj3/o6aef1p49eyRJHTt21MSJEx2rOQAAAHAZeuzgzuxu9RaLpdC2hg0bKicnRxcvXiwwM6YkrVixQs8++6waNGggwzBkGIaCg4M1fvx4bdq0Sb6+vo7XHgAAAPg/1lkxWaAc7sjuVn/FFVcoNjZW3333XcETeHjI07PwzEM+Pj6aOnWqjh07ptTUVB0/flynTp3SG2+8oVq1ajlecwAAAOAy2bkkdnBfdrf6kydP6o033lDXrl11zTXX6JVXXtHRo0ftKlu3bl3Vr19fJpOp3BUFAAAASsJQTLizMvXYWYdU/vzzz3r66afVvHlz9enTR0uXLlVGRkZl1hMAAAAoEQuUw53Z3eqPHDmiDRs2KCYmRgEBATIMQxaLRRs3blRMTIwaNmyoUaNG6YsvvqjM+gIAAABFysr5v8SuiNeEAFdnd2JnMpnUu3dvLV68WH/99Zf++9//KioqSh4eHjIMQ+np6XrvvfcUGRmpJk2a6Omnn9a+ffsqs+4AAACATRY9dnBj5Wr1fn5+GjFihNasWaM///xTr776qjp16mQbqnn06FG98soruvbaa9WlSxe98cYbSk1Nrei6AwAAADa8Ywd35nCrb9CggR5//HF99913+umnnzRp0iQ1btzYluTt3r1bsbGxatSokQYNGqTly5crOzu7IuoOAAAA2GSzQDncWIW2+quuukovv/yyDh8+rC+++EKjRo1SYGCgDMNQTk6OPv30Uw0fPlxhYWF6+OGHK/LSAAAAcHO2dezMJHZwP5XS6k0mk2655RYtWrTI9j5ev3795OnpKcMwlJaWpgULFlTGpQEAAOCmbEMx6bGDG/Kq7Av4+voqMjJSp06d0oEDB/TLL7/IMIzKviwAAADcDAuUw51VWmKXnZ2tVatW6b333tPatWuVm5srSbakrkWLFpV1aQAAALihLCZPgRur8MRuy5YtWrp0qZYvX65z585JupTM1apVS0OHDtV9992niIiIir40AAAA3BizYsKdVUhit3//fr333nv673//qyNHjki6lMx5enoqMjJS9913n+644w75+PhUxCUBAAAAm7w8Q7l5+d8/eccO7qjcid2JEye0bNkyvffee9q9e7ckFXh3rkOHDrrvvvt0zz33qH79+o7XFAAAACiGdUZMSfIxe1ZjTYDqUabE7uLFi1qxYoXee+89bdiwQRZL/loh1oSuYcOGuueee3Tffffp2muvrfjaAgAAAEWwvl8n0WMH92R3Yjdq1CitWLFC6enpki4lc35+fho8eLDuu+8+RUZGysODBwkAAABVK/uyxM7saarGmgDVw+7EbsmSJTKZTDIMQyaTSTfffLPuu+8+DR06VEFBQZVZRwAAAKBEWbn5I8m8vTxkMpHYwf2UaShmq1atdO+99+ree+9Vs2bNKqtOAAAAQJnY1rBjGCbclN2J3bZt29StW7fKrAsAAABQLtbJU1jqAO7K7pZPUgcAAICaytZjR2IHN0XLBwAAgNNjcXK4O6ds+RkZGVq5cqXGjBmjtm3bytfXVwEBAerQoYOmTJlim7mzrA4dOqSHHnpILVq0kI+Pj0JDQ9W9e3dNnz692DI5OTl67bXX1LVrV9WqVUuBgYFq06aN7r//fh09erS8twgAAIAyILGDu3PKlp+QkKAhQ4Zo0aJF8vT01KBBgxQREaGDBw9q8uTJuuGGG3TixIkynfOzzz7T1VdfrQULFigkJETR0dHq3LmzDh06pPnz5xdZ5vTp0+revbsef/xx/fnnn7rtttsUGRkpX19fLV68WAcPHqyI2wUAAEApskjs4ObKNCtmTWE2mzV27FjFxsaqffv2tu0pKSkaMGCAdu/erdjYWCUkJNh1vv379ys6OlpBQUFav369wsPDbfvy8vK0a9euQmUMw9Bdd92l7777TpMnT9Zzzz0nL69LH+eBAwdUq1YtB+4SAAAA9rIldsyKCTfllIldTEyMYmJiCm0PCwvT3LlzFR4ersTERGVnZ8vb27vU802cOFEXL17URx99VCCpkyQPDw916dKlUJkPP/xQmzZt0tChQxUXF1dof8uWLe2/IQAAADiEWTHh7lyu5Xfo0EGSlJWVpVOnTpV6/JEjR7R27Vq1bNlS/fv3t/s6CxculCQ99thj5asoAAAAKsyld+w8q7kmQPVwyh67khw4cEBS/nDNunXrlnr85s2blZeXp/DwcOXm5ioxMVFbt26VxWLRNddco+HDh6tOnToFyuTk5Ojrr7+Wl5eXunbtqj179ujDDz/UiRMn1KhRIw0ePNiWYAIAAKDysdwB3J3LJXazZ8+WJEVFRcnHx6fU4/ft2ydJCgwMVEREhLZv315g/7PPPqvly5frlltusW07cOCALl68qAYNGmjWrFl69tlnlZeXZ9sfFxenCRMmaNasWSVeOyUlRSkpKYW2W2f1zMnJUU5OTqn3UJms16/ueqBiEVfXQ0xdDzF1TcS18mRm53+mZo+q/XyJqWuqSXG1tw4mwzCM8l6kRYsW8vDw0Nq1a9WqVSu7yvzxxx/q1auXTCaTfv/99/Jeukhr1qzRwIED5eXlpaSkJLt6zR566CHNnz9fXl5eCgwM1Lx58xQVFaWTJ09q6tSpWrp0qYKDg7V37141atRIkrR9+3Z1795dXl5eys3N1SOPPKInnnhCwcHBWrVqlR577DFlZGRozpw5Gj9+fLHXjouL04svvljs/oSEBPn7+5f9gwAAAHAzG4+ZtOqwp24IzdPI1nmlFwCcREZGhu6++26dPXu2xMkZHeqxO3z4sEwmk7Kzs+0uk5OTo0OHDslkMjly6UL279+vkSNHyjAMTZ8+3e6hkNaettzcXM2fP1/Dhg2TJNWpU0fvvfeekpOTlZSUpHnz5mnatGmFyvTr109z5861ne/+++/XxYsXNX78eL300kslJnbjxo3ToEGDCm1PT09Xz549FRkZWe0za+bk5Gj9+vXq06ePzGZztdYFFYe4uh5i6nqIqWsirpXn4OYD0uHf1KJZE/Xvf3WVXZeYuqaaFNdz587ZdZxLDMU8evSooqKilJaWpokTJ2rChAl2lw0MDLT979ChQwvtHz16tJKSkrRly5ZCZaz7/27UqFEaP368jh49qt9++63Y3sywsDCFhYUV2m4NntlsrvaGZFWT6oKKQ1xdDzF1PcTUNRHXimf5vzFovt5e1fLZElPXVBPiau/1q/zt0rNnz0pShQ0xPH36tCIjI3X48GGNHj1aM2bMKFP5Zs2aSZKaNm1aZC9i8+bNJanAgufWMpfvv5y/v7/q169fqBwAAAAqRzbr2MHNVXnLX7p0qaSCyVF5paenq1+/ftq3b5+io6O1cOHCMg/x7NSpkyQpLS2tyP2nT5+WVLCXLjg4WC1atCi2XF5ens6cOVOoHAAAACqHdYFyHzOJHdxTmYZi9u7du8jto0ePVkBAQIlls7KydODAAZ04cUImk0mRkZFluXSR5xs8eLB27Nihvn37atmyZfL0LPu6JeHh4QoJCdHx48eVnJystm3bFthvHYJpTQCtBg0apNmzZ2vz5s2F7mX79u3Kzs6Wn59fofMBAACg4tkWKC/H90HAFZQpsdu8ebNMJpMun0jTMAwlJSWV6aItW7bU008/XaYyl7NYLBoxYoQ2btyoiIgIJSYmytvbu8Qyc+bM0Zw5czRkyBC99NJLtu1eXl6aOHGinn32WY0fP16JiYm2CUs2bNig+Ph4mUwmjRs3rsD5YmNj9eabb2rOnDkaNGiQunXrJklKTU1VbGyspPyE154lFwAAAOCYSwuU02MH91SmxO7mm28uMNRxy5YtMplMuv7660vssTOZTPL19VVYWJjCw8P1//7f/yu1h68kc+bM0YoVKyRJoaGheuSRR4o8bsaMGQoNDZWUn3AlJycXuW7cU089pU2bNmnDhg1q06aNunXrptTUVG3fvl0Wi0XTpk1T165dC5Rp3ry53nzzTT3wwAO6+eab1b17dwUHB2vbtm06deqUOnfurP/v//v/yn2PAAAAsF8WiR3cXJl77C7n4ZH/4MTHx+uqq66qsEqV5vL32qwJXlHi4uJsiV1JzGaz1qxZo1mzZmnJkiVau3atvL291bNnTz3++OMaOHBgkeXuv/9+tWzZUi+//LK+/fZbZWZmqmXLlnrsscf05JNPOpS8AgAAwH7ZuRZJJHZwXw4td3DffffJZDKpTp06FVUfu8TFxSkuLq5Cy5jNZk2aNEmTJk0q03l79eqlXr16lakMAAAAKpZ1KKYPs2LCTTmU2MXHx1dQNQAAAIDys06ewqyYcFdVskD577//rtTUVDVv3lwNGjSoiksCAADAjbCOHdydQy3/xIkTmjdvnubNm2dbePxyv/32m66//nq1adNG4eHhatSoke68885i14wDAAAAyoNZMeHuHGr5iYmJevTRRzV79mwFBwcX2JeVlaV+/frp+++/l2EYMgxDeXl5WrlypQYPHuxQpQEAAIDLMSsm3J1DLX/dunUymUwaMmRIoX3x8fH6/fffJV1azPv222+XYRjaunWr3n//fUcuDQAAANgwFBPuzqGWn5ycLEm2xbkvl5CQIEnq3bu3Vq5cqccee0yrVq3SbbfdJsMw9L///c+RSwMAAAA29NjB3TnU8k+ePClJaty4cYHtmZmZ2r59u0wmk8aOHVtg3/333y9J2rVrlyOXBgAAAGyss2KS2MFdOdTyz5w5k38Sj4Kn2b59u3JycmQymXTbbbcV2NeiRQtJ+ROvAAAAABXBto6dl2c11wSoHg4ldoGBgZKk48ePF9i+efNmSdJVV11VaPFys9ksSfLyqpKVFgAAAOAGLiV29NjBPTnU8tu1aydJ+vzzzwts/+ijj2QymdSzZ89CZaxJIOvZAQAAoKIwFBPuzqFuswEDBmj79u1asGCB2rdvr4iICMXHx2vfvn0ymUyKjo4uVMb6bl2jRo0cuTQAAAAgScq15MmSZ0hiVky4L4cSu0cffVTz5s1TSkqKHn300QL7unfvrltuuaVQmdWrV8tkMumGG25w5NIAAACApEu9dRI9dnBfDrX84OBgbdiwQZ07d7YtQm4YhiIiIvTBBx8UOv6HH35QUlKSJKlPnz6OXBoAAACQdOn9OonEDu7L4RlM2rdvr507d+rgwYM6fvy4wsLC1Lx582KPX7x4saT89e0AAAAAR1kTOw+T5OVhqubaANWjwqambNGihW0pg+J06NBBHTp0qKhLAgAAAAUWJzeZSOzgnuirBgAAgFOzzYjJxClwYxXWY5eXl6dNmzbpm2++0fHjx5WRkaFp06YpLCzMdkx2drZyc3Pl6ekpHx+firo0AAAA3FhWjrXHjsXJ4b4qJLH75JNP9I9//EOHDx8usP3JJ58skNi9/fbbeuyxxxQYGKhjx44pICCgIi4PAAAAN2btsWNxcrgzh1v/woULNXjwYB06dEiGYSgkJESGYRR57AMPPKDg4GClp6drxYoVjl4aAAAAsE2ewoyYcGcOtf5ff/1V48ePl5Q/y+W+fft04sSJYo/39vbWnXfeKcMwtG7dOkcuDQAAAEi6lNjRYwd35lDrnzVrlnJzc3X11VdrzZo1ateuXallIiIiJEm7d+925NIAAACAJCnbYpFEjx3cm0Otf+PGjTKZTIqNjZW3t7ddZVq1aiVJOnLkiCOXBgAAACRdNhSTWTHhxhxq/X/++acklWltOuuEKRkZGY5cGgAAAJBUcB07wF051PqtC0CWJUk7deqUJCk4ONiRSwMAAACSSOwAycHErlGjRpKkAwcO2F3m66+/liS1bNnSkUsDAAAAkhiKCUgOJna9evWSYRh699137Tr+7Nmzeuutt2QymdS7d29HLg0AAABIYrkDQHIwsRs3bpxMJpO2bNmi+Pj4Eo89deqU7rjjDh0/flxeXl566KGHHLk0AAAAIOnyBco9q7kmQPVxKLHr1KmTJkyYIMMwNGbMGA0fPlwffPCBbf+2bduUkJCg8ePHq1WrVvryyy9lMpn0/PPPq1mzZg5XHgAAAKDHDpC8HD3Bq6++qqysLL355ptavny5li9fbptUZdy4cbbjDMOQJMXGxuq5555z9LIAAACAJBYoByQHe+yk/Jkx586dq7Vr16pXr14ymUwyDKPAjyR1795dn376qWbOnOlwpQEAAACrrFwWKAcc7rGz6tOnj/r06aPz589r9+7dOnHihCwWi0JCQtSxY0eFhoZW1KUAAAAAG2bFBCowsbMKCgrSzTffXNGnBQAAAIpknTyFHju4M1o/AAAAnFoW79gBlZ/YpaWl6eTJk7Z37QAAAICKxKyYQDkTu9zcXP3000/67rvvdPLkyUL7L168qBdeeEGNGzdWaGioGjZsqKCgIN11113au3evw5UGAAAArEjsgDImdoZh6IUXXlBoaKg6dOigrl27qmHDhrrpppuUlJQkScrOzlbfvn01bdo0paSk2GbGzMjI0IoVK9S1a1d98cUXlXIzAAAAcD9ZTJ4ClG3ylNGjR+u9996TpAJDK7dt26aoqCh9++23mjdvnr766itJUt26ddW6dWvl5uZq3759yszMVGZmpu655x4lJycrODi4Am8FAAAA7ogeO6AMPXabNm3SkiVLJEk+Pj6688479eSTT2ro0KHy8/PTmTNnNGvWLMXHx8tsNmvBggU6efKkvvnmGyUlJSk1NVVPPvmkJOnkyZOKj4+vlBsCAACAe7HOisnkKXBndvfYLV68WJJUv359bdy4Ue3bt7ft279/v3r37q0FCxYoLy9PTz31lB544IEC5f38/PTKK6/oxx9/1Nq1a/Xpp59qwoQJFXQbAAAAcFfZtlkxPau5JkD1sfvPGt9++61MJpMef/zxAkmdJLVr106PP/64LBaLJOnee+8t9jwxMTGSxCQqAAAAqBAMxQTKkNgdO3ZMktS9e/ci91++vVWrVsWep3Xr1pKk06dP23tpAAAAoFgsUA6UIbG7cOGCpPwJUYpSu3Zt2+8+Pj7FnsfX11dS/uyZ5ZWRkaGVK1dqzJgxatu2rXx9fRUQEKAOHTpoypQpSk9PL9d5Dx06pIceekgtWrSQj4+PQkND1b17d02fPt2u8rfddptMJpNMJpP+/PPPctUBAAAAZZOVkz9qjFkx4c7K3PpNJlOZtleGhIQEDRkyRIsWLZKnp6cGDRqkiIgIHTx4UJMnT9YNN9ygEydOlOmcn332ma6++motWLBAISEhio6OVufOnXXo0CHNnz+/1PLx8fH64osvqvRzAAAAAD12gFTG5Q5qCrPZrLFjxyo2NrbA+34pKSkaMGCAdu/erdjYWCUkJNh1vv379ys6OlpBQUFav369wsPDbfvy8vK0a9euEsufPHlSTzzxhCIjI5WcnKzDhw+X78YAAABQZlm8YweUvceuJoiJidH8+fMLTeISFhamuXPnSpISExPtHu45ceJEXbx4UfHx8QWSOkny8PBQly5dSiwfGxurjIwMzZs3rwx3AQAAgIqQzQLlQNl77ObNm6f69esX2n750McpU6YUW76sQyTLqkOHDpKkrKwsnTp1SmFhYSUef+TIEa1du1YtW7ZU//79y3y9zz//XAkJCZo6daquvPLKctUZAAAA5WMYxqV17MwkdnBfZU7s3nzzzWL3Wd8ve/HFF8tfIwcdOHBAUv5wzeImernc5s2blZeXp/DwcOXm5ioxMVFbt26VxWLRNddco+HDh6tOnTpFlr1w4YIefvhhtWvXTpMmTarQ+wAAAEDpcvMMGUb+7z6erGMH91WmxM6wPjU12OzZsyVJUVFRJc7OabVv3z5JUmBgoCIiIrR9+/YC+5999lktX75ct9xyS6GyL7zwgg4dOqTNmzfL29u7zHVNSUlRSkpKoe3WWT1zcnKUk5NT5vNWJOv1q7seqFjE1fUQU9dDTF0Tca14F7Jybb+bDIuq+qMlpq6pJsXV3jqYDDuztS1btjhUoaL07NmzQs+3Zs0aDRw4UF5eXkpKSrINyyzJQw89pPnz58vLy0uBgYGaN2+eoqKidPLkSU2dOlVLly5VcHCw9u7dq0aNGtnK7dq1S127dtXIkSMVHx9v2968eXMdPnxYR44cUePGjUu8dlxcXIm9mwkJCfL39y/9xgEAANxUeo707M78vopZ3XLlwQTlcDEZGRm6++67dfbsWdWqVavY4+zusavoJKyi7d+/XyNHjpRhGJo+fbpdSZ2UP+ulJOXm5mr+/PkaNmyYJKlOnTp67733lJycrKSkJM2bN0/Tpk2TJFksFj3wwAOqXbu2ZsyYUe46jxs3ToMGDSq0PT09XT179lRkZGSJwasKOTk5Wr9+vfr06SOz2VytdUHFIa6uh5i6HmLqmohrxTt+7qK080t5epg0cEDZ50twFDF1TTUprufOnbPrOKdc7uDvjh49qqioKKWlpWnixImaMGGC3WUDAwNt/zt06NBC+0ePHq2kpKQCPZavvfaadu/erXfeeUehoaHlrndYWFiRk7tYg2c2m6u9IVnVpLqg4hBX10NMXQ8xdU3EteIYyh+m5u3pUa2fKTF1TTUhrvZe3+kTu9OnTysyMlKHDx/W6NGjy9yD1qxZM0lS06ZNi1xcvHnz5pIKzua5evVqmUwmvfvuu1qyZEmB448fPy5JGjp0qHx8fPSvf/1LUVFRZaoTAAAA7JNtsUhiRkzAqRO79PR09evXT/v27VN0dLQWLlxYZHJWkk6dOkmS0tLSitx/+vRpSZd69qwMw9CXX35Z7Hmtk7CMGjWqTPUBAACA/bJYww6Q5MSJXVZWlgYPHqwdO3aob9++WrZsmTzLMcVteHi4QkJCdPz4cSUnJ6tt27YF9luHYFoTQCl/iYTilGXyFAAAADjGtji5F4kd3JtTPgEWi0UjRozQxo0bFRERocTExFKXG5gzZ47atWunp59+usB2Ly8vTZw4UYZhaPz48QVeTtywYYPi4+NlMpk0bty4SrkXAAAAlF8WiR0gyUl77ObMmaMVK1ZIkkJDQ/XII48UedyMGTNsk5ukpqYqOTm5yHXjnnrqKW3atEkbNmxQmzZt1K1bN6Wmpmr79u2yWCyaNm2aunbtWnk3BAAAgHLJZigmIMlJE7vL34ezJnhFiYuLs2vWSrPZrDVr1mjWrFlasmSJ1q5dK29vb/Xs2VOPP/64Bg4cWCH1BgAAQMWyJnY+9NjBzTllYhcXF6e4uLgKLWM2mzVp0iRNmjTJobodOnTIofIAAACwX7bFmtiVfa4FwJXwpw0AAAA4LSZPAfLxBAAAAMBpkdgB+XgCAAAA4LSycvMXKGfyFLg7ngAAAAA4LZY7APLxBAAAAMBpWSdPIbGDu+MJAAAAgNPiHTsgH08AAAAAnBbr2AH5eAIAAADgtOixA/LxBAAAAMBp2RYoZ1ZMuDmeAAAAADitrBx67ACJxA4AAABOjFkxgXw8AQAAAHBatnfsGIoJN8cTAAAAAKdlXaDcx+xZzTUBqheJHQAAAJyWbSgmPXZwczwBAAAAcFrZuRZJvGMH8AQAAADAaWWxjh0gicQOAAAATowFyoF8PAEAAABwWtbEjgXK4e54AgAAAOC0rJOn+Jj5Wgv3xhMAAAAAp3VpHTuWO4B7I7EDAACA0+IdOyAfTwAAAACcFrNiAvl4AgAAAOC06LED8vEEAAAAwCkZhmGbPMWbWTHh5ngCAAAA4JSsSZ1Ejx3AEwAAAACnZB2GKUk+JHZwczwBAAAAcEqXJ3YMxYS74wkAAACAU7IOxTR7muThYarm2gDVi8QOAAAATikrh4lTACueAgAAADgl24yYvF8HkNgBAADAObGGHXAJTwEAAACcUtb/JXY+Xp7VXBOg+pHYAQAAwCnRYwdcwlMAAAAAp2R7x47JUwASOwAAADinrByLJHrsAInEDgAAAE6KWTGBS3gKAAAA4JSybZOn8JUW4CkAAACAUyKxAy7hKQAAAIBTYigmcAlPAQAAAJySbbkDZsUESOwAAADgnLJYxw6wccqnICMjQytXrtSYMWPUtm1b+fr6KiAgQB06dNCUKVOUnp5ervMeOnRIDz30kFq0aCEfHx+Fhoaqe/fumj59eoHj8vLy9NVXX2nSpEm6/vrrFRQUJB8fH1155ZV66KGHdPDgwYq4TQAAAJSAxA64xCmfgoSEBA0ZMkSLFi2Sp6enBg0apIiICB08eFCTJ0/WDTfcoBMnTpTpnJ999pmuvvpqLViwQCEhIYqOjlbnzp116NAhzZ8/v8CxBw4c0M0336zp06fr2LFj6t27twYMGKCsrCzNnz9fHTp00Ndff12RtwwAAIC/uTQU07OaawJUP6/qrkB5mM1mjR07VrGxsWrfvr1te0pKigYMGKDdu3crNjZWCQkJdp1v//79io6OVlBQkNavX6/w8HDbvry8PO3atavA8SaTSX369NG//vUv3XLLLTKZTJKkrKwsPfTQQ4qPj9c999yj3377TWazuQLuGAAAAH+XTY8dYOOUT0FMTIzmz59fIKmTpLCwMM2dO1eSlJiYqOzsbLvON3HiRF28eFHx8fEFkjpJ8vDwUJcuXQpsu/LKK7Vu3Tr17t3bltRJko+Pj+bNm6fg4GD98ccf2rZtW3luDwAAAHbItlgksdwBIDlpYleSDh06SMrvPTt16lSpxx85ckRr165Vy5Yt1b9/f4ev7+fnpzZt2kiSjh075vD5AAAAUDR67IBLnHIoZkkOHDggKX+4Zt26dUs9fvPmzcrLy1N4eLhyc3OVmJiorVu3ymKx6JprrtHw4cNVp04du6+fl5enw4cPS5IaNmxYvpsAAABAqVigHLjE5RK72bNnS5KioqLk4+NT6vH79u2TJAUGBioiIkLbt28vsP/ZZ5/V8uXLdcstt9h1/WXLlunEiROqV69eoWGdAAAAqDjMiglc4lKJ3Zo1a/TOO+/IbDZr6tSpdpVJS0uTJL399tsKDAxUQkKCoqKidPLkSU2dOlVLly7VkCFDtHfvXjVq1KjEcx05ckSxsbGSpClTppSaWKakpCglJaXQdutyDTk5OcrJybHrPiqL9frVXQ9ULOLqeoip6yGmrom4VqyLObmSJE8Z1faZElPXVJPiam8dTIZhGJVclyqxf/9+hYeHKy0tTa+99pomTJhgV7mxY8dq4cKFkqT3339fw4YNK7C/a9euSkpK0jPPPKNp06YVe54LFy6oV69e2rlzp+644w6tWLGi1GvHxcXpxRdfLHZ/QkKC/P397boPAAAAdzNvn4eSz3poZCuLbqjnEl9pgUIyMjJ099136+zZs6pVq1axx7lEj93Ro0cVFRWltLQ0TZw40e6kTsofgmn936FDhxbaP3r0aCUlJWnLli3FniMnJ0dDhw7Vzp07ddNNN9m9zMK4ceM0aNCgQtvT09PVs2dPRUZGlhi8qpCTk6P169erT58+LN3gQoir6yGmroeYuibiWrGWpiRJZ9PU9fpO6ndN9cxtQExdU02K67lz5+w6zukTu9OnTysyMlKHDx/W6NGjNWPGjDKVb9asmSSpadOmBZYusGrevLkkFbvgeV5enmJiYvTZZ5+pY8eOWr16tfz8/Oy6dlhYmMLCwgpttwbPbDZXe0Oyqkl1QcUhrq6HmLoeYuqaiGvFyLHk99L5+XhX++dJTF1TTYirvdd36jdN09PT1a9fP+3bt0/R0dFauHBhkclZSTp16iTp0rt2f3f69GlJl3r2/u6xxx7TsmXL1KZNG61du1a1a9cu0/UBAABQPix3AFzitE9BVlaWBg8erB07dqhv375atmyZPD09y3ye8PBwhYSE6Pjx40pOTi603zoE05oAXu65557TvHnz1LRpU61fv17169cv+40AAACgXLJy8xco9/Z02q+0QIVxyqfAYrFoxIgR2rhxoyIiIpSYmChvb+8Sy8yZM0ft2rXT008/XWC7l5eXJk6cKMMwNH78+AJjWDds2KD4+HiZTCaNGzeuQLlZs2Zp2rRpatiwoTZs2KCmTZtW3A0CAACgVNkWeuwAK6d8x27OnDm2WSdDQ0P1yCOPFHncjBkzFBoaKklKTU1VcnJykcsLPPXUU9q0aZM2bNigNm3aqFu3bkpNTdX27dtlsVg0bdo0de3a1Xb8999/ryeeeEKS1KJFi2Jny3zggQd00003OXSvAAAAKBoLlAOXOGVid/n7cCUtKxAXF2dL7EpiNpu1Zs0azZo1S0uWLNHatWvl7e2tnj176vHHH9fAgQMLHH/mzBlZV4n45ptv9M033xR53l69epHYAQAAVBISO+ASp0zs4uLiFBcXV6FlzGazJk2apEmTJpV6rl69eslFlv8DAABwWkyeAlzCUwAAAACnxDt2wCU8BQAAAHA6eXmGbR07ZsUESOwAAADghKy9dRI9doBEYgcAAAAnlJVLYgdcjqcAAAAATif78sSOoZgAiR0AAACcz+UTp5hMpmquDVD9SOwAAADgdGxr2NFbB0gisQMAAIATYg07oCCeBAAAADidrFyLJBI7wIonAQAAAE6HHjugIJ4EAAAAOB1bYsc7doAkEjsAAAA4oaz/mxXTx8zXWUAisQMAAIAToscOKIgnAQAAAE6Hd+yAgngSAAAA4HSybImdZzXXBKgZSOwAAADgdBiKCRTEkwAAAACnk/1/69j5MBQTkERiBwAAACeUbZ0Vk8QOkERiBwAAACfE5ClAQTwJAAAAcDokdkBBPAkAAABwOtYFypk8BcjHkwAAAACnk5VDjx1wOZ4EAAAAOB3r5CkkdkA+ngQAAAA4Hd6xAwriSQAAAIDTsSZ2Pl6e1VwToGYgsQMAAIDToccOKIgnAQAAAE7HtkA5s2ICkkjsAAAA4ISyci2S6LEDrHgSAAAA4HQYigkUxJMAAAAAp2NL7BiKCUgisQMAAIATyrLOimnm6ywgkdgBAADACdkWKKfHDpBEYgcAAAAnxDt2QEE8CQAAAHA6WSR2QAE8CQAAAHA61h47HxI7QBKJHQAAAJzQpVkxPau5JkDNQGIHAAAAp2OdPIVZMYF8PAkAAABwKpY8Q5Y8QxKzYgJWPAkAAABwKtZhmBKTpwBWPAkAAABwKiR2QGE8CQAAAHAqWbkWSZLJJHl5mKq5NkDN4JSJXUZGhlauXKkxY8aobdu28vX1VUBAgDp06KApU6YoPT29XOc9dOiQHnroIbVo0UI+Pj4KDQ1V9+7dNX369GLLxMfHq2vXrgoMDFTdunXVv39/bdu2rby3BgAAgFLY1rDz9JDJRGIHSE6a2CUkJGjIkCFatGiRPD09NWjQIEVEROjgwYOaPHmybrjhBp04caJM5/zss8909dVXa8GCBQoJCVF0dLQ6d+6sQ4cOaf78+UWWiY2N1ejRo/XTTz/ptttuU9euXbV+/XrdfPPNWrlyZQXcKQAAAP7OOiMmwzCBS7yquwLlYTabNXbsWMXGxqp9+/a27SkpKRowYIB2796t2NhYJSQk2HW+/fv3Kzo6WkFBQVq/fr3Cw8Nt+/Ly8rRr165CZTZs2KDZs2crJCRE33zzjVq3bi1J+uabb9SrVy+NHj1avXr1Uu3atR27WQAAABRwaXFy1rADrJzyzxwxMTGaP39+gaROksLCwjR37lxJUmJiorKzs+0638SJE3Xx4kXFx8cXSOokycPDQ126dClUZubMmZKk5557zpbUSVL37t310EMP6cyZM3rnnXfKdF8AAAAo3aXEzim/ygKVwuWehg4dOkiSsrKydOrUqVKPP3LkiNauXauWLVuqf//+dl0jMzNTGzdulCTdddddhfZbt61evdreagMAAMBODMUECnPKoZglOXDggKT84Zp169Yt9fjNmzcrLy9P4eHhys3NVWJiorZu3SqLxaJrrrlGw4cPV506dQqUSU5OVlZWlurVq6fGjRsXOmfnzp0lSXv27KmAOwIAAMDlsnIuTZ4CIJ/LJXazZ8+WJEVFRcnHx6fU4/ft2ydJCgwMVEREhLZv315g/7PPPqvly5frlltusW37448/JKnIpE6SAgICVLt2baWlpen8+fMKCgoq170AAACgsGxL/nIH9NgBl7hUYrdmzRq98847MpvNmjp1ql1l0tLSJElvv/22AgMDlZCQoKioKJ08eVJTp07V0qVLNWTIEO3du1eNGjWSJNtyCv7+/sWeNyAgQGfOnCkxsUtJSVFKSkqh7dbz5+TkKCcnx677qCzW61d3PVCxiKvrIaauh5i6JuJaMTIu5n9+Zk9TtX+WxNQ11aS42lsHl0ns9u/fr5EjR8owDE2fPt32rl1p8vLyu/Jzc3M1f/58DRs2TJJUp04dvffee0pOTlZSUpLmzZunadOmVWid58+frxdffLHY/evWrSsxeaxK69evr+4qoBIQV9dDTF0PMXVNxNUx36WaJHnq/JnTWrNmTXVXRxIxdVU1Ia4ZGRl2HecSid3Ro0cVFRWltLQ0TZw4URMmTLC7bGBgoO1/hw4dWmj/6NGjlZSUpC1bthQqU9KHfOHCBUkqcRjmuHHjNGjQoELb09PT1bNnT0VGRqpWrVr23UglycnJ0fr169WnTx+ZzeZqrQsqDnF1PcTU9RBT10RcK0bmrqPSr3t1RYP66t+/c7XWhZi6ppoU13Pnztl1nNMndqdPn1ZkZKQOHz6s0aNHa8aMGWUq36xZM0lS06ZNZTKZCu1v3ry5JBVY8Lxp06aSpD///LPIc164cEFnzpxRnTp1SkzswsLCFBYWVmi7NXhms7naG5JVTaoLKg5xdT3E1PUQU9dEXB1jUf53Nh+zZ435HImpa6oJcbX3+k79xml6err69eunffv2KTo6WgsXLiwyOStJp06dJF161+7vTp8+LelSL50ktW3bVj4+Pjp58qSOHj1aqIx1QfPrrruuTHUBAABA6WyzYjJ5CmDjtE9DVlaWBg8erB07dqhv375atmyZPD09y3ye8PBwhYSE6Pjx40pOTi603zoE05oASpKfn5969+4tSfrwww8LlVm+fLkk6fbbby9zfQAAAFAy1rEDCnPKp8FisWjEiBHauHGjIiIilJiYKG9v7xLLzJkzR+3atdPTTz9dYLuXl5cmTpwowzA0fvz4AmNYN2zYoPj4eJlMJo0bN65AuYkTJ0qS/v3vf+vXX3+1bf/mm280f/581a5dW2PGjHH0VgEAAPA32bn5iZ0PiR1g45Tv2M2ZM0crVqyQJIWGhuqRRx4p8rgZM2YoNDRUkpSamqrk5OQilxd46qmntGnTJm3YsEFt2rRRt27dlJqaqu3bt8tisWjatGnq2rVrgTK33XabJkyYoNmzZ6tjx47q06ePsrOztX79ehmGocWLF6t27doVe+MAAAC4LLEr+2gtwFU5ZWJ3+ftw1gSvKHFxcbbEriRms1lr1qzRrFmztGTJEq1du1be3t7q2bOnHn/8cQ0cOLDIcq+99po6duyoOXPmaP369fL29tZtt92m559/XuHh4WW/MQAAAJSKoZhAYU6Z2MXFxSkuLq5Cy5jNZk2aNEmTJk0q03lHjRqlUaNGlakMAAAAys/aY+ftSWIHWPE0AAAAwKlk5dJjB/wdTwMAAACcSlauRRKJHXA5ngYAAAA4FYZiAoXxNAAAAMCpZDMUEyiEpwEAAABOxTorJuvYAZfwNAAAAMCp0GMHFMbTAAAAAKdyaYFyvsoCVjwNAAAAcCosdwAUxtMAAAAAp3JpVkzPaq4JUHOQ2AEAAMCpWCdPoccOuISnAQAAAE6Fd+yAwryquwKo2db8eFxb/zLpbNIReTLcwWVYLBb9RFxdCjF1PcTUNRHXinEuM0cSPXbA5UjsUKI5m3/Xryc89cGBn6u7KqhwxNX1EFPXQ0xdE3GtKAHefJUFrHgaUKLuLUPkl3teDRo0lIeHqbqrgwqSl2for7+OE1cXQkxdDzF1TcS14lwVFqymIf7VXQ2gxiCxQ4meH9BOa0wH1L9/R5nN5uquDipITk6O1qxZQ1xdCDF1PcTUNRFXAJWFgckAAAAA4ORI7AAAAADAyZHYAQAAAICTI7EDAAAAACdHYgcAAAAATo7EDgAAAACcHIkdAAAAADg5EjsAAAAAcHIkdgAAAADg5EjsAAAAAMDJkdgBAAAAgJMjsQMAAAAAJ0diBwAAAABOjsQOAAAAAJwciR0AAAAAODkSOwAAAABwciR2AAAAAODkSOwAAAAAwMl5VXcFUJhhGJKkc+fOVXNNpJycHGVkZOjcuXMym83VXR1UEOLqeoip6yGmrom4uh5i6ppqUlytOYE1RygOiV0NdP78eUlSkyZNqrkmAAAAAGqC8+fPKzg4uNj9JqO01A9VLi8vT8eOHVNQUJBMJlO11uX7779Xz549tWXLFnXs2LFa64KKQ1xdDzF1PcTUNRFX10NMXVNNiqthGDp//ryuuOIKeXgU/yYdPXY1kIeHhxo3blzd1ZAkBQYG2v63Vq1a1VwbVBTi6nqIqeshpq6JuLoeYuqaalpcS+qps2LyFAAAAABwciR2AAAAAODkSOwAAAAAwMmR2AEAAACAkyOxQ4nCwsI0efJkhYWFVXdVUIGIq+shpq6HmLom4up6iKlrcsa4stwBAAAAADg5euwAAAAAwMmR2AEAAACAkyOxAwAAAAAnR2KHImVmZuqFF15QmzZt5OvrqyuuuEL333+/jh49Wt1VQwm+++47vfzyy4qOjlbjxo1lMplkMplKLRcfH6+uXbsqMDBQdevWVf/+/bVt27YqqDFKkpGRoZUrV2rMmDFq27atfH19FRAQoA4dOmjKlClKT08vtiwxrdlmzpyp6OhotW7dWsHBwfLx8VGzZs1033336ccffyy2HHF1HqdOnVL9+vVlMpnUqlWrEo8lrjVXr169bP9fWtTP559/XmQ5YlrznTx5Uk8++aTatm0rPz8/1a1bV507d9ZTTz1V5PGrV69Wz549VatWLdWqVUu9evXSp59+WsW1LoUB/E1mZqbRrVs3Q5IRFhZmDBs2zOjatashyahXr57x+++/V3cVUYzBgwcbkgr9lGTChAmGJMPPz88YPHiw0bdvX8PLy8vw9PQ0VqxYUTUVR5EWLlxoi2H79u2NoUOHGn379jWCgoIMSUa7du2Mv/76q1A5YlrzhYSEGL6+vkbXrl2NIUOGGEOGDDHatGljSDLMZrOxevXqQmWIq3OJiYkxTCaTIcm48soriz2OuNZsPXv2NCQZd955pxETE1PoZ8+ePYXKENOab+fOnUZISIghybj66quN4cOHG/369TOaNWtmeHp6Fjp+1qxZhiTDy8vLiIqKMgYPHmz4+fkZkow33nijGu6gaCR2KOTZZ581JBndu3c3zp8/b9v+6quvGpKMnj17Vl/lUKKXX37ZeP75542PP/7YSElJMXx8fEpM7NavX29IMkJCQoxffvnFtn3btm2Gt7e3Ubt2bSMtLa0Kao6ixMfHG2PHjjX27dtXYPuxY8eMTp06GZKMESNGFNhHTJ3D119/bWRmZhbaPnfuXEOS0aBBAyMnJ8e2nbg6lw0bNhiSjLFjx5aY2BHXms+a2B08eNCu44lpzXfixAkjNDTU8Pf3N1atWlVo/7ffflvgv/fv3294enoaPj4+xrZt22zbk5OTjZCQEMPLy8v49ddfK73e9iCxQwFZWVlGcHCwIcnYtWtXof3XXXedIcnYuXNnNdQOZVVaYtevXz9DkjFr1qxC+/7xj38YkowZM2ZUYg1RXtu2bTMkGT4+PkZWVpZtOzF1fldeeaUhyfjhhx9s24ir88jIyDCuvPJK46qrrjJ++eWXEhM74lrzlTWxI6Y138MPP2xIMubOnVum4ydMmFBo38yZMw1JxqOPPlrBtSwf3rFDAVu3btXZs2d15ZVXqlOnToX233XXXZLyxxnDuWVmZmrjxo2SLsX1csS6ZuvQoYMkKSsrS6dOnZJETF2F2WyWJHl7e0sirs7mxRdf1IEDB/TWW2/ZYlkU4up6iGnNl5mZqaVLlyogIECjR4+2q4z1PTpniKlXdVcANcsPP/wgSercuXOR+63b9+zZU2V1QuVITk5WVlaW6tWrp8aNGxfaT6xrtgMHDkjKTwLq1q0riZi6gvfee0/Jyclq3bq1WrduLYm4OpM9e/bo1Vdf1ejRoxUREaFDhw4VeyxxdS7vvPOOTp06JQ8PD7Vp00Z33HGHmjZtWuAYYlrz7dy5U+fPn9dNN90kPz8/ffbZZ1q/fr0uXryoNm3aaNiwYbriiitsx585c0Z//PGHJBXZ4dGkSROFhobq8OHDOnfunGrVqlVl91IUEjsUYG28Rf2DdPn2w4cPV1mdUDlKi3VAQIBq166ttLQ0nT9/XkFBQVVZPZRi9uzZkqSoqCj5+PhIIqbOaPr06dq7d68uXLign3/+WXv37tUVV1yhZcuWydPTUxJxdRZ5eXl64IEHVLt2bb3yyiulHk9cncu///3vAv/95JNP6vnnn9fzzz9v20ZMa759+/ZJkurXr6877rhDq1atKrD/mWee0TvvvKMRI0ZIuhTTOnXqKCAgoMhzNm7cWKmpqTp8+LCuvfbaSqx96RiKiQKs06f7+/sXud/aqM+fP19ldULlKC3WEvGuqdasWaN33nlHZrNZU6dOtW0nps5n7dq1evfdd7V8+XLt3btXzZo107Jly3T99dfbjiGuzuGNN95QUlKSpk+frpCQkFKPJ67O4eabb9Z7772n33//XRkZGUpOTta0adPk5eWlF154wfZHNomYOoO0tDRJ0scff6zPP/9cc+fO1YkTJ3To0CE9+eSTyszMVExMjL7//ntJzhdTEjsAcCL79+/XyJEjZRiGpk+fbnvXDs5pw4YNMgxDaWlp+vLLL9W6dWv17NlT06ZNq+6qoQz++OMPPffcc+rZs6dGjRpV3dVBBZoyZYpGjhypli1bys/PT23atNEzzzyjlStXSpLi4uKUmZlZvZWE3fLy8iRJubm5mjJlih555BHVq1dPzZo10/Tp0zV06FDl5ORo+vTp1VzT8iGxQwGBgYGS8hdGLsqFCxckieEDLqC0WEvEu6Y5evSooqKilJaWpokTJ2rChAkF9hNT51W7dm1FRERozZo1uv766/X8888rKSlJEnF1BuPHj1d2drbeeustu8sQV+cWGRmpLl266MyZM/r2228lEVNnYI2RpCInT7Fu27JlS4HjnSWmvGOHAqwvAv/5559F7rdub9asWZXVCZWjtFhfuHBBZ86cUZ06dWrEP1bu7vTp04qMjNThw4c1evRozZgxo9AxxNT5mc1mDR8+XN99951Wr16tG264gbg6gU8++US1a9fWQw89VGD7xYsXJeX/UaZXr16SpP/9739q2LAhcXUBrVu31s6dO5WSkiKJf4OdgfX7q7+/v+rVq1dof/PmzSVJJ06ckHQppmlpabpw4UKR79nVpO/GJHYowDqsa9euXUXut26/7rrrqqxOqBxt27aVj4+PTp48qaNHj6pRo0YF9hPrmiM9PV39+vXTvn37FB0drYULF8pkMhU6jpi6htDQUEnSyZMnJRFXZ3HmzBnbX/n/7uLFi7Z91mSPuDo/6/ta1i/7xLTms85smZmZqaysLNvkY1anT5+WdKmnrnbt2mratKn++OMP7d69WzfddFOB448cOaLU1FQ1a9as2mfElBiKib/p0aOHgoOD9fvvv9teHL3c8uXLJUm33357FdcMFc3Pz0+9e/eWJH344YeF9hPrmiErK0uDBw/Wjh071Ldv3wKzJf4dMXUN1gTgyiuvlERcnYFhGEX+HDx4UFJ+LK3brD0CxNW5nTx5Ul999ZWkS8sYENOar2nTpurQoYMMwyjyDzHWbZcvbTBgwABJl+J3uRoX02paGB012LPPPmtIMsLDw4309HTb9ldffdWQZPTs2bP6Kocy8fHxMUp6zNevX29IMkJCQoxffvnFtn3btm2Gj4+PUbt2bSMtLa0Kaoqi5ObmGkOGDDEkGREREcaFCxdKLUNMa76vv/7a+OyzzwyLxVJge3Z2tvH6668bHh4ehp+fn/HHH3/Y9hFX53Tw4EFDknHllVcWuZ+41mxbt241VqxYYeTm5hbYfvDgQaNHjx6GJGPQoEEF9hHTmu+///2vIcm49tprjWPHjtm2796926hbt64hyfjggw9s2/fv3294enoaPj4+xjfffGPb/ssvvxghISGGl5eX8euvv1bpPRSHxA6FZGZmGjfeeKMhyQgLCzOGDRtm++969eoZv//+e3VXEcX45JNPjBtvvNH2YzKZDEkFtn3yyScFykyYMMGQZPj7+xuDBw82+vXrZ3h5eRmenp7GihUrqudGYBiGYbz22muGJEOSMWTIECMmJqbIn5MnTxYoR0xrtsWLFxuSjNDQUKNv377G3XffbURGRhphYWGGJMPX19d4//33C5Ujrs6ntMTOMIhrTWZ9Vhs2bGj079/fuPvuu40ePXoYvr6+hiTj6quvNv76669C5YhpzRcTE2NIMmrXrm3079/fuOWWW2x/DH/wwQcLHT9z5kxDkuHl5WX069fPGDx4sOHn52dIMl5//fVquIOikdihSBkZGcbzzz9vXHnllYa3t7fRsGFDY9SoUcaRI0equ2oogfX/hEr6Wbx4cZHlrr/+esPf39+oXbu2ERUVZWzdurXqbwAFTJ48udR4SjIOHjxYqCwxrbkOHDhgPPPMM0aPHj2MsLAww2w2GwEBAcbVV19tPPbYYyX+5Ze4Ohd7EjvDIK411b59+4yHH37Y6Ny5s1GvXj3Dy8vLCA4ONrp162a8+uqrRkZGRrFliWnNlpeXZyxYsMAWo4CAAKN79+5GfHx8sWU+/vhjIyIiwggMDDQCAwONiIgIY/Xq1VVY69KZDMMwKnp4JwAAAACg6jB5CgAAAAA4ORI7AAAAAHByJHYAAAAA4ORI7AAAAADAyZHYAQAAAICTI7EDAAAAACdHYgcAAAAATo7EDgAAAACcHIkdAKDKmEwmmUwmxcXFVXdVaiyLxaLZs2era9euqlWrlu0zu+OOO6q7ai6refPmMplMGjVqVHVXBQDKjcQOAKrA5s2bbV/QTSaThg8fXmqZUaNG2Y6H+xgxYoRiY2OVlJSk8+fPl+scl7c3kmgAcA8kdgBQDT788EP9+OOP1V0N1DDbtm3Thx9+KEkaMGCA1q9frz179ujHH3/U66+/XmHXOXTokC3xi4+Pr7Dz1iS9evWSyWRSr169qrsqAFAlvKq7AgDgjgzD0OTJk5WYmFjdVUENsmHDBkmSp6enEhISVKtWrWqukXs4dOhQdVcBABxGjx0AVLHQ0FBJ0ooVK7R79+5qrg1qkqNHj0qSGjRoQFIHACgTEjsAqGL/+Mc/5OPjI0l64YUXqrk2qEmysrIkSWazuZprAgBwNiR2AFDFmjRporFjx0qSPvnkE+3YsaNc57F3Jj/rJCzNmzcvtK+od60SExMVGRmp+vXrKyAgQB06dNAbb7yhnJwcWznDMJSQkKBevXqpfv368vf3V+fOnfXWW2/JMAy772HDhg0aNGiQwsLC5Ovrq5YtW+rRRx+19VyVZteuXXrooYfUtm1bBQYGKiAgQG3bttXDDz+sX375pdhy8fHxtvs+dOiQsrKy9Nprr6lbt24KDQ11aNKRH3/8UWPHjlXr1q3l7++voKAgXX311Xr88ceLHfJnrcu7774rSTp8+HCByXYqcgIdk8mkFi1a2P579OjRha5V3L1X1eednZ2t1atX69FHH9UNN9ygOnXqyGw2KyQkRDfeeKPi4uKUmppa5LWs7X3Lli2SpC1bthS6v78/C/Y+S6tXr9Zdd92lxo0by8fHRyEhIerevbtefvllpaen233/eXl5WrBggcLDw1WnTh0FBATouuuu07Rp05SRkVFiHb777juNGTNGbdq0UUBAgHx9fdWkSRNdf/31Gj9+vD7++OMyPYMAXIgBAKh0mzZtMiQZkozFixcbx44dM/z8/AxJRmRkZJFlYmJibGWK0qxZM0OSERMTU+K1redp1qxZoX0HDx4sUK+HH37Y9t9//4mOjjZyc3ONixcvGnfddVexxz344IPF1sV6zOTJk424uLhizxEcHGx8+eWXxZ7HYrEYjz/+uGEymYo9h5eXlzF//vwiyy9evNh2XFJSktGxY8dC5SdPnlzi51qU//znP4aHh0exdfLx8THefffdYj+Xkn7K4vL29vf7sOdafy9T1Z/35W2/uJ+QkBDj66+/LnQte8r+/Vko7VnKzMw0hgwZUuI5r7jiCmP37t2l3v/evXuNW2+9tdjzdO3a1UhPTy/yPDNnziyxfVl/zp8/X2R5AK6NyVMAoBqEhYXp4Ycf1syZM7Vu3Tp9/fXXuummm6q1Tm+99Za+/fZb9e/fXw888ICaNWumI0eO6KWXXtK3336rxMRELV68WHv27NHy5ct199136+6771ZYWJh+/fVXxcXFaf/+/Vq4cKGio6MVFRVV7LU+/fRT7dy5U23bttWkSZN03XXX6ezZs/rwww+1cOFCnT17VgMHDtRPP/2kJk2aFCr/2GOPad68eZKkm2++WaNGjVLLli3l7++vH374Qa+99pr27t2rcePGqWHDhho0aFCxdRkzZox+/PFH3XfffRo+fLgaNmyoP/74wzZc1l7z5s3TM888I0mqV6+e/vnPf6pHjx6yWCzasGGDpk+frgsXLmjUqFEKDQ1V//79bWWtM6Q+99xzWrVqla644gqtXbu2TNe3148//qhjx46pb9++kqR///vfGjx4cIFj6tevX+C/q/rzzs3NVcuWLTVkyBB17dpVTZs2lZeXlw4fPqwNGzZo0aJFOnXqlIYMGaKffvqpQH2nTZumJ598UqNHj9bOnTvVpUsXLV68uEAdvL29y/SZxcTEaMWKFZKkDh066IknnlD79u11+vRp/e9//1N8fLyOHTumW2+9VXv27FGjRo2KPdeDDz6o7du3KyYmRsOGDbPd/yuvvKJvvvlGO3bs0L///W+99NJLBcrt2bNHTz75pPLy8tSiRQs9+uij6tixo+rWravz588rOTlZmzZt0qpVq8p0bwBcSHVnlgDgDv7eY2cYhvHXX38ZAQEBhiTjlltuKVSmqnvsJBmxsbGFjrlw4YLtWiEhIYbJZDJee+21QselpKQYQUFBhiRj0KBBRdbl8mt17ty5yJ6FJUuW2I4ZOnRoof3r1q2z7X/77beLvE5mZqbRu3dv233n5OQU2H95D0pJ57HXiRMnDH9/f1vPzR9//FHomF27dtni3ahRIyM7O7vQMSXFqixK6rEzjMI9tSWpjs/7t99+M/Ly8ordv2fPHiMwMNCQZDz33HNFHtOzZ09DktGzZ88Sr2UYJT9Ln3zyia3et956q5GVlVXomAULFtiOGTZsWKH9f7//9957r9AxFy9eNK655hrbc/b3z/D55583JBkBAQHG8ePHi72XM2fOGBaLpdR7BuB6eMcOAKpJ/fr19eijj0qSNm3apE2bNlVrfZo0aaJXXnml0HZ/f3/FxMRIkk6dOqUbb7xREyZMKHRcw4YNNWTIEEnSV199Ver1FixYoMDAwELb7733XvXr109S/syhx48fL7D/5ZdfliTdeeedGjNmTJHn9vX11Zw5cyTlv69W0mfbu3fvYs9jr8WLF9vejZo5c2aRvYydOnXS008/LSl/9suVK1c6dM2qUh2f95VXXlnie4XXXnutHnjgAUmq9M9x7ty5kvIntFm8eHGRvX0PPvigbrvtNkn576impKQUe77o6GiNHDmy0HYfHx/bvwenTp3Svn37Cuy3Pgdt2rRRgwYNij1/cHCwPDz4ege4I558AKhGTz31lIKCgiRJzz//fLXWJTo6utjZGDt06GD7ffjw4cWew3pcWlqazpw5U+xx1157ra6//vpi999///2S8ofkbd682bb93Llztv++6667ii0vSe3bt7ctLfHNN98Ue9w999xT4nnsYV1/rnbt2oqOji72OGsycnmZmqymfN5paWn6/ffftXfvXv3000/66aefVLt2bUnSvn37CkzsU5Fyc3Ntk7BERkYWmbBbPfjgg7Yyl7fZvyvp/i9/Jg4cOFBgX1hYmKT8+y3vhEsAXBuJHQBUo5CQEMXGxkqStm7dWmnvVdmjTZs2xe6zfokuy3Hnz58v9rgbbrihxLp07drV9rv1/TNJ2r17t/Ly8iRJI0aMKDTb4d9/rDMn/r3X73LXXXddiXWxx08//SRJ6ty5c4lLFTRo0MA2I6O1TE1WnZ/3jz/+qPvvv19hYWGqW7euWrVqpWuuuUbXXnutrr32Wtssmnl5eUpLS3PsRotx4MABW0/sjTfeWOKxl+8vKbbt2rUrdl/dunVtv//9+RkxYoTMZrOysrLUo0cP3X777Xrrrbf0008/MQsmAEkkdgBQ7SZOnGhLiCZPnlxt9fD39y923+VDu+w9zmKxFHvc3yfn+LvLh5qdPn3a9vuJEydKLFeckqaQr1OnTrnOeTlrHUu7Lyl/yOrlZWqy6vq833nnHXXu3FmLFy8uMUm0yszMLFP97HV5jEqLrTWufy/3d+V9ftq1a6dly5apTp06ys3N1SeffKKHH35Y1157rerXr697773XriHQAFwXs2ICQDWrXbu2Jk6cqBdeeEHffvutPvnkEw0cOLC6q1Wpyrsu2+VfdufPn6/w8HC7ypWUTHh6eparLkWpyPXmaoLq+Lz379+vhx56SLm5uapfv76eeuop9e7dW82bN1dQUJCtR3TRokW2d/WqoseqJsT2zjvv1G233ab3339fa9eu1VdffaWTJ08qNTVVS5cu1dKlSxUTE6NFixbxnh3ghkjsAKAGiI2N1ezZs3Xq1ClNnjzZrsTO+sXNOlSuOBcuXKiQOlakv/76y+79lw9PCwkJsf3u7++va665puIrVw5169ZVSkpKqfclXRqmePl91VTV8XnHx8crNzdXnp6e2rJlS7FDF6uix/PyGJUW28t7FisztsHBwRo7dqzGjh0rSfr555+1atUqvfHGGzp27JjeffddderUqcgJjgC4Nv6cAwA1QFBQkJ566ilJ0q5du2xrZpVWRlKp7xf98ssvjlewgiUlJdm9//JkomPHjraek61bt1ZO5crBWsddu3YpNze32ONOnDihw4cPFyhTHeztfaqOz3vv3r2S8ifiKel9tJ07d5Z4noroYbOu1SdJ3377bYnHXj6hSVXGtn379vrXv/6l7du3KyAgQJL0wQcfVNn1AdQcJHYAUEM8+uijtvd4Jk+eXOrwshYtWkjKTyaKO3bv3r3as2dPxVa0Avz444/avXt3sfsXLVokKX/YXq9evWzb69Wrp27dukmSEhISdPLkyUqtp72sU92fOXNGiYmJxR73zjvv2GJlLVMdfH19bb9nZWUVe1x1fN7WxLiknuaUlBR9/PHHJZ7Heo8l3V9pvLy81LNnT0nS+vXr9eeffxZ77Ntvv20rc3mbrSpNmjSxTWxkncQGgHshsQOAGiIgIED//Oc/JeUnPmvWrCnxeOsXzmPHjmnZsmWF9p8/f97h9dkq09ixY4v88p6QkGC79zvuuMM2zbvVc889Jyl/Kv677rqrxGUVsrKyNHfuXF28eLHiKl6E0aNH23p2nnjiCR09erTQMT/88IP+85//SJIaNWqkO+64o1LrVJKQkBDbemy///57icdW9efdunVrSdKvv/6qbdu2FdqfkZGhu+++u9QJU6zt5sCBAw69gzd+/HhJUnZ2tsaMGVPk0gqLFi3SunXrJOUvG/L3NlsRVq5cWeJnf+TIEe3fv1/SpT/6AHAvJHYAUIM8/PDDti+Fpf3VfeTIkapVq5YkacyYMZoyZYq+/fZb7dixQ2+++aY6d+6sH374QZ06dar0epdVly5dtHPnTnXp0kXx8fH67rvvtHHjRj3yyCO69957JeUPNZ0xY0ahsv3797e9P/Tll1+qffv2evHFF/XFF1/o+++/19atW/Xuu+/qgQceUFhYmB599NESh0dWhHr16mn69OmSpD///FPXX3+9XnvtNe3YsUPbtm3TlClTdNNNNyk9PV0mk0kLFiwocVmEyubl5WVbcmLRokVatmyZfv75Z/3222/67bffCry/VtWftzX+eXl5GjBggP7zn//oyy+/tLXrjh07avPmzerRo0eJ57FO9HLixAlNnDhR3333ne3+rMNh7TFgwAANHTpUkrRu3Tp169ZN//3vf/Xdd99pw4YNeuCBB2zrE9atW1czZ84sz22X6rXXXlOjRo00bNgwvfXWW9qyZYu+//57bdq0SdOnT1ePHj1sye5DDz1UKXUAUMMZAIBKt2nTJkOSIclYvHhxice+8cYbtmOtP8X54IMPDE9Pz0LHSzL8/PyMDz/80IiJiTEkGc2aNStU/uDBg3bV6/L6b9q0qdjjFi9ebDvu4MGDhfZb902ePNmYPHlykfWWZNSqVcvYvHlzsdfJy8szXnzxRcPLy6vYc1h/AgICjIyMjDLVs7ymTZtmeHh4FFsXHx8f49133y22fEmxKovL4zV58uQij/nkk08Mk8lUZD3/XqaqP+8XX3yxxGs88cQTpZ7z/PnzRsuWLYss//fPt1mzZoYkIyYmpsj6ZGZmGkOGDCmxTldccYWxe/fuIsvbe/8lPY89e/Ys9bP38PAwpk6dWvwHC8Cl0WMHADXMgw8+qCZNmth17NChQ7Vt2zYNGTJE9erVk7e3t5o0aaKYmBglJSXprrvuquTall9cXJw+//xzDRgwQA0aNJC3t7eaN2+uRx55RHv37rUNNS2KyWTSCy+8oF9++UWTJk1Sly5dVLduXXl6eiooKEhXXXWV7rnnHr377rtKSUmRn59fldzTM888o927d+vBBx/UlVdeKT8/PwUEBKh9+/aaMGGC9u/fr/vuu69K6lKaAQMG6IsvvtDgwYN1xRVXlNiDWNWf9wsvvKBPP/1UkZGRqlOnjry9vdW4cWNFR0dr3bp1Rfbk/l1gYKC2bdumCRMmqH379iWuH1caX19fJSYm6uOPP1Z0dLSuuOIKeXt7q06dOrrxxhv10ksvKTk5WR07diz3NUqzbNkyLViwQHfffbc6duyohg0bysvLS4GBgbr66qv18MMPa/fu3bahswDcj8kwqmDxFwAAAABApaHHDgAAAACcHIkdAAAAADg5EjsAAAAAcHIkdgAAAADg5EjsAAAAAMDJkdgBAAAAgJMjsQMAAAAAJ0diBwAAAABOjsQOAAAAAJwciR0AAAAAODkSOwAAAABwciR2AAAAAODkSOwAAAAAwMmR2AEAAACAkyOxAwAAAAAn9/8DzpNLotza0TYAAAAASUVORK5CYII=", "text/plain": [ - "
" + "(4, 3, 306)" ] }, + "execution_count": 16, "metadata": {}, - "output_type": "display_data" + "output_type": "execute_result" } ], "source": [ - "# Plot the best value obtained vs number of iterations\n", - "plt.figure(figsize=(10, 6))\n", - "for i in range(num_cluster):\n", - " plt.plot(best_observed_all_clusters[i], label=f'Cluster {i}')\n", - "plt.xlabel('Number of Iterations')\n", - "plt.ylabel('Best Value')\n", - "plt.legend()\n", - "plt.grid()\n", - "plt.show()\n" + "best_observed_all_clusters_all_trials = []\n", + "for trial in range(1,5):\n", + " # Load files from the trial\n", + " path = f'../bo_output/results_with_three_clusters/trial_{trial}'\n", + " # Load the joblib files\n", + " best_observed_all_clusters_all_trials.append(load(os.path.join(path, 'best_observed_all_clusters.joblib')))\n", + "\n", + "# Convert to a 3D numpy array\n", + "best_observed_all_clusters_all_trials = np.array(best_observed_all_clusters_all_trials)\n", + "best_observed_all_clusters_all_trials.shape" ] }, { "cell_type": "code", - "execution_count": null, - "id": "ca4fcc52", + "execution_count": 22, + "id": "321e5285", "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/plain": [ + "Text(0.5, 0, 'Number of Iteration')" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0gAAAIjCAYAAAA5ozv2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAABueUlEQVR4nO3deXxU9b3/8feZNXtCEtAgCIJsdUERNyzFpWWRViRuP6yt5Vq15dZC8dbWWgW1tr0/qdUW+7vUoqi3UKsFWxFFqYIKKriCULAKokCULWQny8z390dmhslKJpmZkzl5PR8PFGbmnHxmPvme+X7O93u+xzLGGAEAAAAA5LI7AAAAAADoLiiQAAAAACCEAgkAAAAAQiiQAAAAACCEAgkAAAAAQiiQAAAAACCEAgkAAAAAQiiQAAAAACDEY3cAiRIMBrVnzx5lZ2fLsiy7wwEAAABgE2OMKioq1LdvX7lc7Y8RObZA2rNnj/r37293GAAAAAC6ic8++0z9+vVr9zWOLZCys7MlNX4IOTk5NkcDAAAAwC7l5eXq379/pEZoj2MLpPC0upycHAokAAAAAB269IZFGgAAAAAghAIJAAAAAEIokAAAAAAghAIJAAAAAEIokAAAAAAghAIJAAAAAEIokAAAAAAghAIJAAAAAEIokAAAAAAghAIJAAAAAEIokAAAAAAghAIJAAAAAEIokAAAAAAghAIJAAAAAEIokAAAAAAghAIJAAAAAEIokAAAAAAgxGN3AEg9ZbVldodgK7flVpYvq8lj1fXVqg/W2xQR4sHr8irDm2F3GOiChmCDquqrmjyW7kmXz+1r8lhPP4Y5QaY3Ux7XkS5MIBhQZX2ljREhHnL9uU3+XReoU01DjU3RIB5clkvZvmy7w4gZBRJiVlJVooAJ2B2GbdLcaS0KpIOHD6qsjk5XKsvz51EgpbjaQK12Ve5q8tixGceqIL0g8m9jTIvXIPUMzBnYpEBqLfdIPVneLLld7si/q+qrtKdqj40RoavS3GkpWSAxxQ4xC5qg3SHYqrX3b4yxIRLEU0//vXaC1nLY/DHy7AzN82jEMdgJgmqa1558Mhb2okBCTIwxPf6LqNVOmOh0pTqK3NTXoQKJtuoIFL7O1Pw4zHEZdqFAQkz4EupYJwyphzOVqa/V0d1mJ3Roq87QvONMXp2h+XGYvMIuFEiICWdfW/8MOIinPs5Upj6m2PUcdKSdqcUIUg+fsQL7UCAhJnQiG9Hpch5ymPpa60w170hzDHMGRgadqXkeGdmHXSiQEBO+hBpRIDkPo6OpLxBs2ZliKpYzsUiDMzU/DtNeYRcKJMSEg1UjpgE4D7/bqa+1dsgZaWdqkddWimOknmCwWeHLiC9sQoGEmFAINGreyaLTlfookFJfR1aYpMPlDIwgORMjSOguKJAQEwqBRtEHbQ7gzmBk6DynuFYLpGZnpJlK6QxMc3Ym8oruggIJMaED2Sj6bCUHcOcgl6mtIyNI5NgZuLbMmcgrugsKJMSEg1Wj6M+BotE5+P1Oba3eB4kOlyMxddKZuLEzugsKJMSEzkWjJlPsOIA7BrlMbR25RxnHMGfgOlBnanFtGYUvbEKBhJhwIWyj6IM4qyc5B53n1NZaZ4qRBmdqMTLIyQ1HaJ5HCl/YhQIJMaED2ajJFDuKRseg85za2upMRZ/EoCPtDIw0OFPz6et8v8IunSqQ7rvvPhUXF2vIkCHKzc2V3+/XgAED9O1vf1ubNm2KeX+lpaWaOXOmBgwYENnXrFmzdOjQoc6EhwSiQGrEIg3OxNnK1NZWJzm6KGq+qh1SE1MnnSm6DZNT2KlTBdIvf/lLPffcc8rPz9dFF12kyZMnKy0tTY8//rjOOOMMLV++vMP72r9/v8466yz97ne/k8fj0aWXXqrs7Gw98MADOvvss3Xw4MHOhIgE4SxdoyZnpDmIOwa5TG1t5a9Jp4sRJEegQHKm6JNUtFXYqVMF0t///neVlpbqzTff1NKlS7V06VJt27ZNDz74oOrr6/Xd735XDQ0NHdrXrFmz9NFHH6m4uFjbtm3TE088oQ8++EA33XSTPvzwQ82ePbszISJBOGA1YgTJmTgBkNraOj5x3zLnYfl2Z4o+BnM8hp06VSCdd955SktLa/H4jBkzNHjwYH3xxRfasmXLUfdTUlKiJUuWyOfz6Q9/+IM8Hk/kuXvvvVe9e/fW//7v/2rv3r2dCRMJwPSURnS4nIkTAKmrvcVSWJbfmVhN1HmaLIDElGfYKO6LNHi9XkmSz+c76muff/55BYNBjR07Vsccc0yT5/x+v77xjW8oEAhoxYoV8Q4TncSXUCMWaXAmit3U1d6xiU6XM4XzSrt1juh2zMkM2CmuBdLjjz+ubdu2aciQIRoyZMhRX//+++9LkkaNGtXq8+HHN27cGL8g0SV8ETViBMmZyGXqaq8z1WSRBnLsGOGcU/Q6B9+t6C48R39J2+69915t3rxZVVVV+te//qXNmzerb9++WrJkidxu91G3//TTTyVJ/fr1a/X58OM7d+5scx8lJSUqKSlp8XhlZWVH3gJixBmdRhzEnYlcpq72ctfkugZGfB0jYALyysv3koMwbRLdRZcKpJUrV+qf//xn5N8DBgzQY489pjPOOKND24eLmIyMjFafz8zMlCRVVFS0uY8FCxbozjvv7GjI6CI6kI04I+1MdLRSV3vtsMnKWLRXx2CKnfMYGRljZFkW1zzDVl2aYrdq1SoZY1RaWqpXXnlFQ4YM0bhx43TPPffEK76juvHGG/X222+3+LNmzZqkxdCTcEanEfdqcCam6qSu9tph+DluPOks4VxyDHaWSOFLfwM26tIIUlheXp7Gjh2rFStW6Nxzz9Xtt9+u8ePH68wzz2x3u6ysLElSdXV1q89XVVVJkrKzs9vcR1FRkYqKilo8Xl5e3tHwEQPOsDeK7kjzmTgHuUxd7XWmuFbFmRhBcqagCcotN3mFreK6SIPX69VVV10lY4yeeeaZo77++OOPlyTt2rWr1efDjw8YMCB+QaJLOGA1iu5I0+lyDs5Ypq52F2mIGkGCczDS4EzhfNJeYae4L/NdWFgoSdq3b99RXzty5EhJ0jvvvNPq8+HHTz311DhFh67ii6gRF5I6EycAUld7JyroSDsTha8zhfPKyUfYKe4FUvjan8GDBx/1tRMnTpTL5dKrr77a4mawtbW1euaZZ+R2u3XxxRfHO0x0Ap3HI7hXgzPxO5662r0GSUzFciKm2DlT+DuV71bYKeYCae3atZEbvEarr6/X73//ez3++ONKT0/XVVddFXlu/vz5Gj58uG699dYm2xQVFWnatGmqq6vTjBkz1NDQEHnulltu0b59+3TNNdeoT58+sYaJBOBLqCm+nJ2HXKaujkyxI7/OEl6kgZEGZwnnkxFf2CnmRRr+/e9/a/r06SosLNQZZ5yhgoIC7d+/X5s2bVJJSYnS0tK0aNEi9e/fP7LN/v37tW3btlbvV3T//ffrjTfe0N/+9jcNHz5co0eP1ubNm/XBBx9oyJAhuu+++7r2DhE3nM1pKmACclkuOl0OQi5TV0fug0R+nSUQbOxI893kLJzQQHcQ8wjSuHHj9LOf/UzDhg3Txo0b9eSTT2rt2rXKz8/XTTfdpE2bNunKK6/s8P4KCwu1fv163XTTTaqrq9OyZctUVlamH/7wh1q/fr3y8/NjDREJwlm6puh0OQ9LQKeujtwHibbqLCzz7Ux8t6I7sIxDT72Ul5crNzdXZWVlysnJsTucI6r2S+W77Y6iU6obDmtHdctRwB7Hkybl9dfg3MFK86Rp84HNdkeEODqp4CS7Q0An7K7crUO1h1p9Lt2drkF5g3To8CHtrorj8bfqgFRzMH77Q8fl9levrCL1zeqrz6s+14HKPdKhT+2OCl2Vf6KKsvsqPy1fO8p2qLryc6nyC7ujQhekebM1eNBFdochKbbaIC73QUIMggEpRc+KGJO6scdV8MjZaM5wOU/QBOWy4r5+DRKsIzeKjfsoOMdE+wQDTfMaDJILJ4j6XjXGNOaUvKa2FM0fvYBkS9FfFEkKOnOwMXZR86MpkJyHnKamdhdpCN9XJd5TKPldsY8xTa9VIRfOEJXXgAlI9DtgEwqkZEvhg3iQ6zMaRd1Thc608zh01rHjdeg+SPFur/yu2Kj5SAO5cIYghS+6BQqkZEvhxm5SOPb4OnKPBgok52Fp2dRkyzLftH/7GNPs/lbkwhFM8Mh9kGRoY7ANBVKypXBjZwQphCl2jhZeOhippd1lvmUSc0KDUQv7GNN0tTNy4QzGNF11ku9Y2IQCKdlSuLFzDVKUYONBnGWhnYecpqajjfwl5oQGvyu2McFmHWly4QihESRjTGgEibzCHhRIyZbCBRIdxyihgzj3hnIeRgVT09HyFlQw/teX8btin6ipWEEx0uAYoamTke9W8gqbUCAlWwqfDQlwoDoiNF2HC/qdhwIpNR21QDLB+F9fRvu3kVGL5aDhAMGm363kFTahQEq2FG7sLNIQLSgjFmlwInKaeiLTcdoRjJqSFb8fzO+KbaIK3sa8Uqw6QujkI4vlwG4USMmWwl+oLNIQJdTZojPtPOQ09XQkZybqov644XfFPqFcBoIBVjtzknCBFM4neYVNKJCSLYUbO4s0RGl+EIdjcK1d6unI2WYWaXCYUC4bTEPo3+TCEULtlAIJdqNASrYUbux0HKOE5khTIDkPOU09HclZQkZ8+V2xT+izjyzLTy6cwTSbvk7hC5tQICVbCh/E6ThGCS/SQNHoOPyep56OTrGLa26DtP3uoD5Y3/gX2q1DMMUO3QMFUrKlcGPnGqRoQabYORQ5TT0dubYoqDhf+M3vib3CI0jcasFZQicej+SVfgfsQYGUTMYolRs7S1pHCZ2N5svZeSiQUk9HctYQbIjvD+V4aK/wNUjhvNJunSHUrsgr7EaBlEwp3tAZQYoSupkdRaPzkNPU05ETFZFrVeImtY/nKS9qFbvGf5MPR4i+tsxIqXxSGamNAimZUvwAzpn1KM1X2oFjMCqYejpS1MZ/BIm2b6sWI0h0pB0hOq+0MdjIY3cAPUqKN3aKgSjhAomzyI5jV04DQaNPDlTZ8rNTXVlthfYdrmn1ubx0r7LSPEeWg5a0q7T118bCqq+Rp6K2y/tB5+RnWvLqyDLfDYGADpCPlHdMQYakUF5NUDX1AZXXxPnkBpIqw+/TYLuD6AQKpGRK8QKDKXbRTOOFpHGftgO72TXFLhA0qq7l96kzqmsbdLiu9c+uwdf4NXdkpEFtvjYWrvoGBRtS+5ieygIm2FgghfIaCAZURz5SXjAYlEvhESSjQMCQ1xTn9aRm/phil0wpXCAxetQMKyg5ll2/69yIufPaG/ULF7zhthq/1bk5JtrJmKBkjuSVawedIRiM/m7ltCzswwhSMqVwkUHnrZlmnS44R32wXtsPbU/6zz1cH9Duquqk/1wnCLRzfVH4qHukQIrPsczikGgrE2z8TyC0OKxJ4e9XRAm104AJSMZQ+MI2FEjJlMIHcG6I2kwK5xLtMzKqCXT9GpVYHW4IqDZwOOk/1+maH7vi1+HiGGCnxgXOgpJcjffOIR2O0CSPJsj9mGEbptglUwofwZli1wyfB+KMUdrEaP6xxu1T5hhgK2PMkeQapmI5RTB6VgZtDDZiBCmZulkHqDZQ1+EZvnXB+gRHA/Rs3evo4BzBZqeg4zfFjozZqfHTD3WgmYrlGE3yaEx36zahB6FASqZudDYkaILaXrWHSyA7qxvlEs5ARyAxWowgxetz5hhgK6OoESS+yRwjvPiGLEkyZBa2YYpdMnWjL9Ty+ioOPF3RjXIJZ6BASozmH2v8pjKSMDsZoyPHYWO4TtYhmuY1yHERtqFASqZu1Kkuq+eGlF3CURtxxjVIidH8c43fx0y+7ERH2qGaFb40M9iFAimZukmBVB9sUBWrZXVNN8klnIMOXmIk7HPlGGCrxgIpapEG2o8jBKUmeeXEEexCgZRM3eQLtay+iukIXcbnh/jiIvPEaP65xm3ZYPJlq8Yp4uGONBfzO0Vje41efMPWcNCDUSAlU7B73FS0vL7S7hBSXzcpduEc/EYlRvOCiFXsnKHJFDsu5neMlotvkFfYg1XsksmGTvUXhw+qOlDb5LGaYF3S43AeE7XSDtB1jCAlBjeKdSim2DlSi2vLbI0GPRkFUjLZUCDVBGpVzfVGiWGCksUgLOKDufaJkahlvhlBslfQGC7mdyBWJ0R3Qe8umWz4Qg0yFSxx+GwRR/S3E6N54Rm/c9K0fzsZiYv5HajF4hs0M9iEAimZbGjpDFAnEF/IiCN+nRIjcTeKJWG2anIxP1PsnCKopiOD5BV2oUBKJhsKpACnXxKIzxbxQz8gcaIPg8E4LWPHFDt7NY4gHSmQaEEOEmlbTLCDfSiQksmGYoXDSwLRQUIcMUUocaI/2/h9zJwgsVMwerUzwzedU0Rfg2QC3WPlX/RMFEjJxAiSs3STZdvhDNRHiZOQz5Zjq60aB42OjCAxndwZogskrqGGnSiQkinJjZ2DS4LRo0Ucscx34kSPL8TlqEiqbMfF/M7UONrbmFdDUmEjCqRkSnJjZ/Qo0fh8ET9xujQGrYj+aOMzlZG2bzcT9V+J81WO0Xz5dsAmFEjJEnVWJGk/ktOcicXBG3FEe02cJkURBZJzhKc5c0NRx4hevt0EaWewDwVSsnD9kfNQICGOWKQhcaI/2niM1LGCXfcQXpGQjrRzRC/zzWUCsBMFUrLYsYIdX+KJxcEbcURzTZzo4jMuhSjJ6haCpnEEiWtVnCOy+EaQeyDBXhRIycJNYp2HL2XEEZ2BxIn7DDvafvcQZKTBaY7cAinItGPYigIpWZhi50AcvBEfNNXEajrFruvtlil23UNk5Igpds5igo33tiKtsBEFUrLYMsWOo0tC8fkiTrj+KLHi//nS9ruDcF75rnOOxkUagmIODOxGgZQsdowgcXhJLDq1iBN+lRLryLSdeH3WJKw7MEZSMMgJBocJBoONU+zIK2xEgZQsjCA5D58v4oRzpYkVvpYhXveaYopd92BkpGCAEwwOEwxPsbM7EPRoFEjJwjVIzsPnizihg5dY4c83biMNtP1uoXHFswAX8ztNeASJvMJGFEjJYkMPiIMLkBpoq4kVHqGL15QdiwKp+wg2UK86jAmPIHFYhI0okJKFESTn4fNFnNARSKzw5xu3j5m23y0EpdBIA5ykcaQ3yHERtoq5QKqurtbTTz+t6667TsOGDVNaWpoyMzM1cuRI3XXXXaqsrIw5iBdffFGTJ09W79695fV6VVBQoPHjx2vZsmUx76vbsuM+SBxdEotOEuKEtppYwWD4GqR4fc7kqzswJnQNkt2BIL6MuLYMtou5QFq8eLGmTp2qhx9+WG63W5dcconGjh2rHTt2aM6cOTrzzDO1d+/eDu/v/vvv1/jx4/Xcc89p6NChuuyyyzR8+HCtWrVKxcXFuu2222INsXuypUCiA59QHL0RJ/wqJVb0KnZx2mO8doQuOLJIA/lwknBeGRuEnWIukLxer2644QZt2bJFW7Zs0V//+lc9//zz2rZtm04//XRt3bpVs2bN6tC+9u3bp5/+9Kfyer16+eWXtXbtWv3lL3/R2rVrtXr1avn9fv3qV7/S9u3bYw2z+7GjQOLgklgUoIgT+neJFf54WaTBWYyRZBq4hs9hguG8klbYKOYC6dprr9WCBQs0YsSIJo8XFRXpwQcflCQtXbpUdXV1R93Xm2++qdraWl144YUaN25ck+e+8pWvaMKECTLG6K233oo1zO6HESQH4uiN+GCKXWKFP994LfNNz617MEzFcqZQXvmKhZ3iukjDyJEjJUm1tbU6cODAUV/v9/s7tN+CgoIuxdUtMILkPBSgiBM6eIkV78+X+yB1D40jSFzM7zThxTc4cQQ7xbVACk+F83q9ys/PP+rrzzrrLOXl5emll17SmjVrmjz3yiuvaOXKlRoyZIjGjh0bzzDtwQiSAxnOcCEuuIYi0eI8giSOrd1BUEYKMBXLaYwhr7CfJ547e+CBByRJEydO7NDoUG5urhYuXKirr75aF1xwgcaMGaN+/fpp165dWrdunc477zw99thj8vl8be6jpKREJSUlLR7vzGp6CUWB5EwmKFmslo+uoaUmVrgwitcZaUaQuofGESQu5ncaIyOZgDgDCTvFrUBasWKFFi5cKK/Xq7vvvrvD2xUXF+u5557TlVdeqbVr10Yez8nJ0fjx43Xccce1u/2CBQt05513djrupEnyF6oxhi+NZKj4nAIJXeaqrpWnpsHuMBzL5bIkb4ZUUydPdX2X92cFu74PxInhGiSnCRe+pBV2ikuBtHXrVl1zzTUyxujee++NXIvUEb/5zW90yy236NJLL9XcuXM1aNAgbd++XXfccYfuuOMOvfnmm1q+fHmb299444265JJLWjxeWVnZYuGHnoTiKEnqKuyOAE5wuF7uOgqkRLEsSbX1smr5nJ0k/su3ozsI55V+DOzU5QJp9+7dmjhxokpLSzV79mzNnDmzw9uuXr1a//Vf/6VRo0bpySeflMvVeCb+lFNO0VNPPaXRo0fr2Wef1XPPPadJkya1uo+ioiIVFRW1eLy8vLxzb8ghmF4HpA46AokV7nAx0uAs4SmTpNVhwu2Vbgxs1KW5QQcPHtT48eO1c+dOTZ8+XfPmzYtp+8cff1ySNHXq1EhxFOZ2u1VcXCypccEGxIYCCUgddNwTy7T4Cxwhsnw7iXWS8Akj0go7dbpAqqys1KRJk7RlyxYVFxfroYcekmVZMe1j165dkhoXa2hN+PHS0tLOhtljcUYaSCE014QzQY6LTnOk8CWvTsLMSXQHnSqQamtrNWXKFK1fv14TJkzQkiVL5Ha7Y97PscceK0lt3gh2w4YNkqSBAwd2JswejREkIHVwBjzxjOhwOU1kpMHmOBBfxjC9DvaLuUAKBAKaNm2aXnrpJY0dO1ZLly5tdxluSZo/f76GDx+uW2+9tcnjl156qSTpz3/+c4uFGP7+979r8eLFcrlcmjp1aqxh9nh0uIDUQXNNvKAMn7PDhDvRjAw6i+EWg+gGYl6kYf78+Vq2bJkkqbCwUDNmzGj1dfPmzVNhYaEkaf/+/dq2bVuL+xVdeumluuKKK/Tkk0/qG9/4hkaPHq0TTjhBO3bsiIwq3XPPPRo2bFisYfZ4Qe6sAqQMQ3cg8QyFqNNEFt/g685RgtymBN1AzAVS9PVA4UKpNXPnzo0USG2xLEtPPPGEJk6cqEcffVQbN27Ue++9p7y8PF188cW66aabNHHixFhDhBhBAlIJrTXxjDguOhVpdRiGkNANWMY489BSXl6u3NxclZWVKScnx+5wpAMfS7XJW3p8f22Zvqg9mLSfB6Dz9pTVKBCwOwpn65Pj16GqOtUFHPmV12P175Wuz0pr7A4DceR1WyrI8uvzssN2h4I4yEzL1thRLe9XaodYaoMuLfON7ospO0DqYIpQ4nFS2pkCzjzH26Mx2ovugALJoQL0uICUQV8g8Yy4rsGJgqTUcYxhQRXYjwLJoQwFEpAy6AskngmKD9qBglRIjhM0zIKB/SiQHCrAwQVICZzLSI7GaTt2R4F4Y4qdA3EfJHQDFEgOxQgSkBqY9pUcJvIfOAlFr/MYiRuVwHYUSA7FNUhAauAEeHIYw6QdJ3LoQrw9HnmF3SiQHIqz0gBwRIChBkcir85EXmE3CiSHCjKCBKQEugHJwbLBzkRenYm8wm4USA7FwQVIDbTV5OCEtDORV2cir7AbBZJDBbnEEUgJ1EfJwXLQzkRenYm8wm4euwPoKQImoGCwIWk/j7PSQGpg6YDk4JjoTOTVmcgr7EaBlCS7qr5QZc0+u8MA0M1wuWBy0OFyJvLqTOQVdqNAAoB4MFJDJ6aFcKPL5AhSiDoSeXUm8gq7USABQBzUNAS0v6LO7jDQBspQZyKvzkReYTcWaQCAOOCMJwAAzkCBBABxEOBiIgAAHIECCQDigBEkAACcgQIJAOKAxRYAAHAGCiQAiANGkAAAcAYKJACIgyAVEgAAjkCBBABxEGBhWgAAHIECCQDiIBiwOwIAABAPFEgA0EXBIDc2BADAKSiQAKCLgqxgBwCAY1AgAUAXBYIUSAAAOAUFEgB0EfdAAgDAOSiQAKCLGEACAMA5KJAAoIu4BxIAAM5BgQQAXUR9BACAc1AgAUAXcZNYAACcgwIJALooEKBAAgDAKSiQAKCLuA8SAADOQYEEAF1EgQQAgHNQIAFAF7FIAwAAzkGBBABdEAgalmgAAMBBKJAAoAu4SSwAAM5CgQQAXRCgQgIAwFEokACgC4IUSAAAOAoFEgB0ASvYAQDgLBRIANAFAQokAAAcxWN3AECimKBUVlOvqroGu0OBg1EfAQDgLBRISVLbENDh+oDdYfQYQSOV19SrPkDvFQAAAB1HgZQkB6vrdKiizu4wAAAAALSDa5AAAAAAIIQCCQAAAABCKJAAAAAAIIQCCQAAAABCKJAAAAAAIIQCCQAAAABCKJAAAAAAICTmAqm6ulpPP/20rrvuOg0bNkxpaWnKzMzUyJEjddddd6mysrJTgXzyySf63ve+pxNOOEF+v1+FhYU699xzde+993ZqfwAAAAAQK8sYY2LZ4E9/+pOuv/56SdKIESN08sknq7y8XOvWrVNFRYWGDx+uNWvWqE+fPh3e53PPPafLL79cNTU1GjVqlIYMGaIDBw5o06ZNyszM1EcffRTbu5JUXl6u3NxclZWVKScnJ+bt4+2NTS/oUMU+u8MAAAAAkiIzLVtjR11idxiSYqsNPLHu3Ov16oYbbtCsWbM0YsSIyOMlJSWaPHmy3n33Xc2aNUuLFy/u0P62bt2q4uJiZWdn68UXX9SYMWMizwWDQb3zzjuxhggAAAAAnRLzCFJ7Xn/9dY0ZM0Z+v1/l5eXy+XxH3ebiiy/Wc889p2effVYXX3xxvEJhBAkAAACwUaqOIMV1kYaRI0dKkmpra3XgwIGjvv6zzz7TypUrNWjQoLgWRwAAAADQGTFPsWvP9u3bJTVOw8vPzz/q61evXq1gMKgxY8aooaFBS5cu1dq1axUIBHTyySfrqquuUq9eveIZIgAAAAC0Ka4F0gMPPCBJmjhxovx+/1Ffv2XLFklSVlaWxo4dqzfeeKPJ87fddpueeuopXXDBBfEMEwAAAABaFbcCacWKFVq4cKG8Xq/uvvvuDm1TWloqqXFlvKysLC1evFgTJ07Uvn37dPfdd+t///d/NXXqVG3evFnHHXdcq/soKSlRSUlJi8c7u9w4AAAAgJ4rLgXS1q1bdc0118gYo3vvvTdyLdLRBINBSVJDQ4MWLFigK6+8UpLUq1cvPf7449q2bZs2bNigP/zhD7rnnnta3ceCBQt05513xuNtAAAAAOjhulwg7d69WxMnTlRpaalmz56tmTNndnjbrKysyP+vuOKKFs9Pnz5dGzZs0Jo1a9rcx4033qhLLmm5OkZlZaXGjRvX4VgAAAAAoEsF0sGDBzV+/Hjt3LlT06dP17x582LafsCAAZKk448/XpZltXh+4MCBkqS9e/e2uY+ioiIVFRW1eLy8vDymWAAAAACg08t8V1ZWatKkSdqyZYuKi4v10EMPtVrktOf000+XdORapOYOHjwo6chIEwAAAAAkUqdGkGprazVlyhStX79eEyZM0JIlS+R2u2Pez5gxY1RQUKDPP/9c27Zt07Bhw5o8H55aFy6kYL/Xy97VxoqtdodhK8uydE7OaTo1e7gk6dPDe/TigbVqMA02R4au8Foefa3gy+qf1nJEGt1fQ7BBS/e9oNL6siaPZ7jTdEnvryrXky1JevngG/qweocdISKO8r25Ku4zQW7LraAJatm+F7W/7qDdYaGLBqUfr68VnCdJqmyo0tP7VqkqUG1zVOgKt8ujZ6vW6ddjf213KDGJuUAKBAKaNm2aXnrpJY0dO1ZLly6Vz+drd5v58+dr/vz5mjp1qn71q18d+eEej2bPnq3bbrtN//mf/6mlS5dG7my7atUqLVq0SJZl6cYbb4w1TCTI8n0vqc7U2x2G7coaKiIF0qulG7S1+mObI0I8pLn9uvrY7nHHb8Tm45rPtKF8Y6vP9U/rq/N7na26YL2eO7BaJsmxIQFqpNOyv6QhGQP16eESvVH2rt0RIQ4+qtmpMXmjlOlO18bKbXq/8l92h4Q4qNx72O4QYhZzgTR//nwtW7ZMklRYWKgZM2a0+rp58+apsLBQkrR//35t27at1eW4f/zjH+vll1/WqlWrNHToUJ1zzjnav3+/3njjDQUCAd1zzz0666yzYg0TCVAXrI8UR1cec7E8Vlxvo5USyhsqtHz/y6psqIo8VhFo/PuX887Q8WmtL0eP7m1nzS6tLXtHFVF5RWqpCDTe2qHI10cX5J8jSXq3YrP+VfVxpL1WBqpkJLktt646ZrJdoaKLXjr4uj6v2xfJazj3vb35+lrBl+0MDV3wt73PqzZYp8qGKmW60yN5HZpxgkbnnGJzdOgsvzdNp5+Yeu0y5h5u9PVC4UKpNXPnzo0USO3xer1asWKFfvvb3+qxxx7TypUr5fP5NG7cOP3oRz/S17/+9VhDRIKEh7ndcml09ikxX3PmBJUN1Vq+/2VVBw8rYAJyW25Vhj6XL2UO0ZCMgfYGiE7JcKVrbdk7kVwi9YRzd6y/t07P/pIk6VB9eWOBFHou/P9sd2bkNUg9myv/3VggNctrb18+eU1hLx54TfuCB1UZqNYxavy+laQBaX3JawrLTMvW2OMvsDuMmMW8SMPcuXNljDnqn/AKdNHbLFq0qNV9er1e3XLLLfrggw9UU1OjsrIy/fOf/6Q46maqAjWSpEx3Ro8sjqTG6xnC7zz8eYS/nDPdGTZFha7KcqdLEnPdU1g4d+FcSlJms7xWRdpqupC6wjmubJbXLI7BKS2cv+aFL9+tsEOnV7FDz1MdKggyenDnwmW5Iu+/KlCtoDGRz4Uv59QV/gKuCtQoaLhCJRVVtdKZosPlTEfaa/PCl7ymsrbyyncr7ECBhA6rCoZHkHpugSRJWe5MSY2drZpgjUzoku+e/rmksvAXcFBBHQ6m3sWkODIdJ9w+JSnLc6StNn0NHa5UFn0MlsirU7R1QoO8wg4USOgwztI1ip62E/5iTnelyW3FvtQ9ugePy6M0l1+SuA4pRVW2Mn2OM9LOlNlsih0jg86Q6WlaINHngJ0okNBhR65B6tkjJdFnuTjD5RzNr1dBammt+AnntN40qC5YR0faIbI8FL5OlBV1QiNggqoOjeaTV9iBAgkdRoHUKDNqegdfzM7RfNoOUsuRkxVHptj5LV/kdgS0V+doMcWOvDpCdIEUbquWevZ1z7APBRI6rDp0wMpw9eyDVWTFs4Zqzkg7SPNpO0gd9VH3aIs+gWNZFiO+DhQ+3h4O1qoh2NBkhVWkrsyothoukDLc6XJZdFWRfPzWocP4EmoUffaSDpdzhPPKFLvUE26HbssduZYsrLWz0uHFG5Ca0l1pcoVuuLC/vlRBBSVxHE51rRVI0SPCQDJRIKHDmGLXKLOVDlf44lKkrshIQwMFUqqJPlHR/B5tmVF5ZcTXGVyWpYxQDr+o2y9JSnP55XF57AwLXRQ+BlcHalTRUCWJ/gbsQ4GEDqsOMoIkNb1JIdc0OAeLNKSu9m4AG37sYEOZ6k1Dm69Dagkfcz8PFUjkNPWFc2hktLf+oCS+W2EfCiR0iDGGu9CHMMXOmVikIXW1tkBDWPix8EiDx/LIb/mSFxwSIqvZCBJTsVKf23Ir3ZUm6UheM8krbEKBhA6pM/VqMAFJFEjhEbSa4GGVN1Q2eQypK3r+O1JLe1PnWnakW07DQ+rJbJbXnv695BQt2yt5hT0okNAh4dEjj+WR1/LaHI29MtxpCnevDtQfkkSB5ARZTLFLWUemurY9xW5/XWmTfyO1ZTXLK6P4zhDO45H2Sl5hDwokdEj0Ag09/eyry3JFLhBm9STnCK9sVhWoUdAYm6NBLNpb8Sr82JG2ypQdJ8gkr46U2eK7lbzCHhRI6JBwgdTT74EU1rwg4qx06ssM/W4HFdTh0B3ckRrCKw+2OsXOQ1t1Io7BztS8DTPFDnahQEKHsEBDU9GfQ7orTW7LbWM0iAePyxO5hw7XIaWWynZXsWve4WK01wma55qpWM7Q8oQGeYU9KJDQIdXcJLaJ6GF/OlzOEX1TUaSO9qfYNS+QmLLjBM1v9ktenYH2iu6CAgkdUhXkJrHRogtFCiTnYCW71NTecvs+yyuPdeQGohzDnKFlR5q8OkH0d6ulxkWRADtQIKFDohdpQNMvY0bVnCMz6ibASA31wXrVmXpJrR+fLMtq0pnmhIYzND/uchx2huj2meFOl8uimwp78JuHDqlu5z4jPRFT7JwpnFem2KWOcDHrttyRa8iaa1IgeZiy4wTprjS5dGRFVY7DztB0dgZtFfahQEKHRFaxYwRJUtODeKaHL2anCHeywquiofuLnl7X1i0ImrRXOtKO4LKsyO0W0lx+eVyeo2yBVJDVpK3S34B9KJDQIZEpdizzLanpFDvOXDpHJjeLTTkdWWEzs8mUWI5hThE+9pJT58jkuxXdBAUSOoRlvptiip0zhfPKNUipo7KdFezCws95LI/8li8pcSHxwsdepmI5h9tyK93VuDBDJnmFjRiTxlEZY1jmuxmm7DhT9Cp2DcEGm6NBR1Q0VElqvx0e6Ui3PQ0PqSeTESRHynJnqCZ4mJUJYSsKJBxVbbBOAQUlcQ1SWIY7TZYkIwokJwl/IX9et0+3fjzP5mgQi/Y6U+EONB1pZwnnnFF8Z8l0Z2hf/UG+W2ErptjhqML3QPJaXvlcXpuj6R5clkvDMwar0NtLvb35doeDOOnjK1CBN8/uMBAjt+XWkIyBbT5/Qnp/pbn8GpE5OHlBIeGGZpwgn+XV0IwT7A4FcTQic7DSXH4NSu9vdyjowSxjjLE7iEQoLy9Xbm6uysrKlJOTY3c4emPTCzpUsc/uMCL+sW+V1pW9K3Ug/UZGQRnleXJ02wkzkhBdajDGyMhwnwaHCZqgaoN1doeBGHgst7xHOXkTNEHaqgORV2cir86RmZatsaMusTsMSbHVBkyx64H+Xf2JXj30VszbDU4/PgHRpC7LsmSJ6xmcxmW5lM7d2x2HzpYzkVdnIq+wGwVSD1MfbNDSvSslSefknq6v5o/p0HaWLGWzogwAAAAcjgLJweqDDdp5eJcCUdPoNlf9W/vrS5XjztLFBeM4Uw4AAABEoUBysGX7VmpD+aZWn5vS+6sURwAAAEAzFEgO9nntfklSvjdPaa4jN0ccknGCTskaZldYAAAAQLdFgeRg4eW5/88xX9cJ6f1sjgYAAADo/lgmxMFqAo0FEjdHBAAAADqGAsmhAiaommCtJCnDRYEEAAAAdAQFkkPVBA5H/s5iDAAAAEDHUCA5VHXo+qM0l19ubrgGAAAAdAg9Z4eqDl1/lMH1RwAAAECHUSA5VKRAcjG9DgAAAOgoCiSHCi/xzQgSAAAA0HEUSA5VHVqkgSW+AQAAgI6jQHKoI1PsKJAAAACAjqJAcqgjizRwDRIAAADQURRIDlXNNUgAAABAzCiQHCpyDRJT7AAAAIAOo0ByqKrQFLt0RpAAAACADqNAcqjwFDtWsQMAAAA6jgLJocJT7LgGCQAAAOg4CiQHqg82qN7US5IyXKxiBwAAAHQUBZIDhafXuWQpzeW3ORoAAAAgdXjsDgDxF71Ag2VZNkcDoCP2Vbn0eSXnrOItw2t0Qq+AXKFDYWmNpV3lbnuDQpflpwd1XE4w8u89FS4dqKb9pLrjsgPKzzCSJGOkTw65VVlHPyaVpXkld85+jTmx0O5QYhJzgVRdXa0XXnhBzzzzjF577TXt3LlTbrdbJ554oi677DLNnj1bWVlZnQ7o3//+t0499VQdPnxYF110kVatWtXpffVUNaECiSW+gdRQUy/9YnW2agN0BBLhu2dUafRx9Qoa6VevZKu8lo60E8y9sFzHZgV1oNrS3S9ny4j2k+oyvEH99/hyed3Spi88+sP6zvcn0X3027xRr/3kQrvDiEnMBdLixYt1/fXXS5JGjBihSy65ROXl5Vq3bp3mzJmjJUuWaM2aNerTp0+nArrhhhtUW1vbqW3R6MgIEtcfAangQLVLtQFLbsuoX27A7nAco7TGpfJal/ZUuCXVq7LWihRHA/Ia7A0OnfZ5hVu1AUslFW4dmxXU55VuGVnyuY2Ksmk/qerTQ25V17tUdtilwsxgqN1K2b6g8jOCR9ka3ZXb5dbgY3PsDiNmMRdIXq9XN9xwg2bNmqURI0ZEHi8pKdHkyZP17rvvatasWVq8eHHMwSxcuFCrV6/WDTfcoD/+8Y8xb49G1cHQTWJZwQ5ICZV1jZ32PllB3fqVSpujcY5ntqbp2Q/TVFnbOLJQEZqqk+njc05lf3gzUxu/8B7Ja+j/g3o1aNaYKjtDQxfc+mKOSmssVdRZKsw8ktdz+tfpspMO2xwdOiszLVtjR422O4yYxTzP4Nprr9WCBQuaFEeSVFRUpAcffFCStHTpUtXV1cW03y+++EI//vGP9bWvfU3Tpk2LNSxEqQ6NIGUwxQ5ICeGOe7aPs6TxlO1v/DzDn2+4EM32GdtiQte1mVc/eU1l4eNfJXlFNxDXidgjR46UJNXW1urAgQMxbTtz5kzV1NToD3/4QzxD6pEiBRIjSEBKCJ8pzaIjEFdZoUKoIjStLvw50+FKbeTVmcL5a5FXThzBBnEtkLZv3y6pcRpefn5+h7dbsWKFnnjiCf3sZz/TiSeeGM+QeqTwMt8ZXIMEpARGNhIj3OEKn5EOjzhk0eFKaeTVmY4Uvs3ySuELG8S1QHrggQckSRMnTpTf37H771RVVWnGjBkaNmyYfvKTn8QznB6rihEkIKUcOQNOBy+ewmeeIx2uWqbsOAF5dabw8S9S+IbymkNeYYO43QdpxYoVWrhwobxer+6+++4Ob/fzn/9cO3fu1Msvvyyfzxfzzy0pKVFJSUmLxysre+4FuDWB0CINXIMEpITIFDtGkOIqfOa5qt6lQFCRi/oZqUtt4bxWhEZeK+vIqxNkR02dNOZIXhkZhB3iUiBt3bpV11xzjYwxuvfeeyPXIh3NW2+9pd/97nf69re/rfPPP79TP3vBggW68847O7VtsvlryzT0w2Xy1lcn9OcEMioltzRy+ws6PfBSQn9WT2Qsl3b1O0/7ep9idyhwCC5GTowsn5ElIyNLVXVW1JSdrne4Mqr3ashHy+VuYHWtZBvRYKnAU6Tf1X5TklR52Ojnnsd10ae7lFlCG0pVA2tcKnSfrHV1E3S4QcoKVuo2z5819l8HIzd6Rupxuz3SZy9IU+bbHUpMulwg7d69WxMnTlRpaalmz56tmTNndmi7hoYGXX/99crLy9O8efM6/fNvvPFGXXLJJS0er6ys1Lhx4zq930QoKlmv40rWJ/znVB5/nCS3BpZ+rD519Qn/eT1RRvU+CiTEDavYJYbLkjJ9RpWh4igyFSsOIw39P3tVRZ+/1eX9oHO+49mkF+pGy5h+Glj/ib7reU4qszsqdEUfSbd5N+rrtV9WRa1Lk91v6grPK1Jsa36hO6o9aHcEMetSgXTw4EGNHz9eO3fu1PTp02MqdHbt2qX33ntPxx57rK644oomzx06dEiS9Pbbb0dGllavXt3qfoqKilRUVNTi8fLy8g7Hkmj3v32/3vrsDfnrPlFGn0LV+XJU60/cTbNKg42/iCWDp6jO6ti1YOgYf125hn70jPx13ef3C6mPVewSJ8tnVFnXOG0nMhUrDp+zv7bxGFBy7Gjtzx/W5f2h4wbufEnZVSUqVLnKay31Ch6SJFWm99aOE8bbGxw6bdjWp+QL1spfW6HKujz1sQ5Jkg7kD9OeY1PvPjpo5PemaeiQ8+wOI2adLpAqKys1adIkbdmyRcXFxXrooYdkWbGPgX7++ef6/PPPW33u0KFDWrNmTWdD7Dbe3fuu3i/bLFmSMjMkNUjBxFbTfpdPlcd9RYddcbvMDJJ8tRUa+tEz8tVXyQoGZFxuu0NCigsEpep6VrFLlGy/0eeViowiSfG5psFXVyFJ2ld4kvb0PbvL+0PHFR74l7KrSlRglamkwq0Cq7FYrco8Vrv7pV5HDI2O375KvsNfKK2hQhV1vXRcaEiwNG8weU1hmWnZGnpyy5le3V2nes+1tbWaMmWK1q9frwkTJmjJkiVyu2PrKA4cOFDGtN4ZWL16tS644AJddNFFWrVqVWdC7FamnzxdJ300WL23/UPZVXu0u++5OpQ3KKE/s19akTwUR3FX58uUkSVLRt76KtUlcCQQPUN4VMOSUSYFUtyFpy2WHXapKo7XevnqGxcCqvNldXlfiE2dL1uSlG9V6LMKl3qrsVit95OLVNbgz5IOf6GcYIUOVLt0itWYV9oY7BBzDzoQCGjatGl66aWXNHbsWC1duvSoq8/Nnz9f8+fP19SpU/WrX/2q08GmqvP7n6+0Q3Ua8c5flVtRpbcyhmp/7sl2h4XOsFyq92bKV18pX10lBRK6LHoFOy5Ejr/wtMXPKxuLI0smLqsFhkeQwp11JE/4My9QudZXuDU8NIJELlJbfagQKrDKtTNqZJC8wg4xF0jz58/XsmXLJEmFhYWaMWNGq6+bN2+eCgsLJUn79+/Xtm3bWl2Ouyc58oXK2ZBUVufLChVIFXaHAgcIr2DH9UeJEZ62uKeicZZDRjwKUWPkq2MEyS51UR3pPdEdaS8d6VRW7z9S+L5e4VaBwgUSbQzJF3OBVFpaGvl7uFBqzdy5cyMFEtT4hRqaklHP2ZCUVufLkqokXz0FEroucpNYVrBLiPDNJ0sq4nedl6ehRi4TkESn3A513sYOc75VrpIKl/LpSDtCi7y6KHxhH1esG8ydO1fGmKP+GThwYIttFi1a1KGfcf7558sY44jrj8JcDYflDjZIOnIQQGoKD/eHzyADXXHk3jyMICVCeDpdZCGMeFx/FGr7De40Bd3eLu8PsYmeYldd71IhU7Ec4cjIYIVq643yrfAoLXlF8sVcIKFzvLWNq7E0uHwKeFh6O5WFC1ym2CEeIjeJZYGGhGheEMVzBTtGLOwRPcVOahxxiH4cqSmy+IbK1UuNxZGRpTpfpp1hoYeiQEoSb+ieGfUcwFMeI0iIp8gUOz9T7BKh+dTF+KxgR4Fkp/AxONeqllcNyhf5cILIyKBVHil6D3syJIuuKpKP37ok8YQKpFqGilMeBRLiKXoVO8Rf86mL8RipO7JAA8dzO9R7MxRUY7sZYH0ur8X1YE4QLnDzrXIW3oDtKJCSJDzFjjNcqS+cQ6bYIR4q43hvHrSU5TOydOSzjcdIHVPsbGa5dNjT+NkPsXZLkupcXA+W6o5MsatQ79BNYrm3FexCgZQk3rrwFDvOhqS6SIHEKnaIA1axSyyXpSY34I3PPZBCI0ic3bZN+Dg81NolSapl8aOUF76+120ZDXLtkSQ10GeCTSiQkuTIFDsO4qmOKXaIJ1axS7zooiieq9gxxc4+DaF75gxxNRZIdX5ykeqMy63D7gxJ0onhkUH6TLAJBVKSRKbYccYx5YVz6KuvkhUM2BwNUlkgeGT56RyuQUqY6KIoHiN1TLGzX3jBo/AUuwZy4Qi1oe/X8MggJyFgFwqkJGEVO+eo82XKhC4Q9tZX2RwNUlllaPTIklEGBVLCRBdF8RipC9/0mwLJPuGO8yCrRBLXqjhFQyiPJ1ifS6KNwT4USEnCKnYOYrlU7228LwPT7NAV0SvYuSybg3GwcFFkycTpGqTwCBLHc7uEP/vICnbkwhHCJ5HJK+xGgZQk4REkzoY4AyvZIR5YwS45wkt7Z8ajEDUm6hokjud2af7ZM33dGZrPsqGNwS4euwPoEYyJmmLHQdwJ6nxZUhUr2aFR0Egr/+1X6eHYzjntr2p8fRYr2CVUeGnveIweeRpq5DLcd8dudV460k7UfMSINga7UCAlQ12lXMH6xr+yFKkjHFnJjgIJ0ua9Hv19a3qnty/IoEBKpPDnG4/POdzmG9zcd8dOLTrSnHx0hBYjg+QVNqFASoaq/ZKkBpdPAY/f5mAQD+FCl2uQIEmfHnJLkgbkNejkPg0xbet1G53dry4RYSHkpD4N+tbIag0piC03rWF6XffQsiNNPpwguiAyslTny7QxGvRkFEjJECqQWMHOObgXEqJ9VtZYIJ11XL0uGlxrczRozmVJ5w2ITxEanlZLh9xeLUeQyIcTROe13pshWVwqD3vwm5cM1Y0FEivYOQdT7BBtV3ljgdQvl/tiOR03ie0e6r0ZkdstSFyr4hTRhS5tDHaiQEqG0AgSZ7ic48gqdowg9XQ19dL+6lCBlEOB5HTcJLabsFyRHHA9mHNEX6dNG4OdKJCSoWqfJFawc5JIgcQqdj1eePQoPz2oTG726niRESRGLGwX7kzTkXaOJiNItDHYiAIpGaoPSJJqOYg7BtcgIWxXGaNHPQk3ie0+wjkgF85hXB7VexpXBKXwhZ0okJIhPMWOsyGOEc6lr75KVpCOcU/G9Uc9C6vYdR/hHJALZ6HwRXfAKnbJEJlix0HcKep8mTKyZMlo/Kof2h0ObPQ1SfJL1qeNf+BslhqnUdIpt9+RjjS5cJI6X7Yyq/eSV9iKAikZWMXOeSyXDuQPVeHBbZEOE3omt6SoxbTQA9R5MlSefbzdYfR4B/OHqv+u13Sg11C7Q0EcHcgfppzyT1WaN9juUNCDWcYYR/buysvLlZubq7KyMuXk5NgbTH2N3nlnqQ7U1nKjWCcxQZb57uG+qHDpvnXZ8rmN7ryoXC4KpR6hwZPBqmndhBVskHFxrtdpyKtzZKZla+yoS+wOQ1JstQG/fcngTVddRm8FAvvsjqTHqaqzdOhwonqtLkm9ErRvpILNh7zap3QNzm1QQxrVEZBsdKKdibzCbvwGJsELmz/Xmx8GVFPrszuUHqO2wdLmvR59eMCjoKHjisRiBTsAAJyDAikJ/vTqDq3/JCgpw+5QeqRMX5DlGpEwaR6jc/rX2R0GAACIEwqkJDjrhHyZQKnq62vtDqXHsCxpYF5Ao/rWq3dm0O5wAAAAkCIokJLgvyYM0xt9d+pQRandoQAAAABoBzOPAAAAACCEAgkAAAAAQiiQAAAAACCEAgkAAAAAQiiQAAAAACCEAgkAAAAAQiiQAAAAACCEAgkAAAAAQiiQAAAAACCEAgkAAAAAQiiQAAAAACCEAgkAAAAAQiiQAAAAACCEAgkAAAAAQiiQAAAAACCEAgkAAAAAQiiQAAAAACCEAgkAAAAAQiiQAAAAACCEAgkAAAAAQiiQAAAAACCEAgkAAAAAQiiQAAAAACCEAgkAAAAAQmIukKqrq/X000/ruuuu07Bhw5SWlqbMzEyNHDlSd911lyorKzu8r0OHDmnx4sWaNm2aTjjhBPl8PmVnZ+vss8/WAw88oPr6+ljDAwAAAIBOs4wxJpYN/vSnP+n666+XJI0YMUInn3yyysvLtW7dOlVUVGj48OFas2aN+vTpc9R9/fznP9c999wjy7J02mmnaejQodq3b5/Wrl2r2tpaffnLX9bKlSuVkZER8xsrLy9Xbm6uysrKlJOTE/P28fbGphd0qGKf3WEAAAAASZGZlq2xoy6xOwxJsdUGMY8geb1e3XDDDdqyZYu2bNmiv/71r3r++ee1bds2nX766dq6datmzZrVoX1lZmbqlltu0SeffKJ33nlHf/nLX/TPf/5TmzZt0vHHH6/XXntNv/jFL2INEQAAAAA6JeYRpPa8/vrrGjNmjPx+v8rLy+Xz+Tq9ryVLlujqq6/WwIEDtWPHjpi3ZwQJAAAAsE+PGUFqz8iRIyVJtbW1OnDgQFz2tWfPni7HBQAAAAAdEdcCafv27ZIap+Hl5+fHZV/HHntsl+MCAAAAgI7wxHNnDzzwgCRp4sSJ8vv9cdnXlClT2n1dSUmJSkpKWjwey2p6AAAAACDFsUBasWKFFi5cKK/Xq7vvvrtL+/qf//kfrVq1Snl5efrpT3/a7msXLFigO++8s0s/DwAAAACkOBVIW7du1TXXXCNjjO69997I9UOd8eqrr2rmzJmyLEsPP/yw+vbt2+7rb7zxRl1yScuLvyorKzVu3LhOxwEAAACg5+lygbR7925NnDhRpaWlmj17tmbOnNnpfX3wwQeaMmWK6urq9Lvf/U5Tp0496jZFRUUqKipq8Xh5eXmn4wAAAADQM3VpkYaDBw9q/Pjx2rlzp6ZPn6558+Z1el87duzQ+PHjVVpaqrlz5+qmm27qSmgAAAAAELNOF0iVlZWaNGmStmzZouLiYj300EOyLKtT+yopKdHXvvY1lZSUaObMmZozZ05nwwIAAACATutUgVRbW6spU6Zo/fr1mjBhgpYsWSK3292pAEpLSzVhwgR9/PHHmj59un772992aj8AAAAA0FUxF0iBQEDTpk3TSy+9pLFjx2rp0qXy+XztbjN//nwNHz5ct956a5PHq6urNXnyZG3atElXXnlll0ahAAAAAKCrYl6kYf78+Vq2bJkkqbCwUDNmzGj1dfPmzVNhYaEkaf/+/dq2bVuL+xXddtttev311+V2u+XxeHTddde1uq9FixbFGiYAAAAAxCzmAqm0tDTy93Ch1Jq5c+dGCqSj7SsQCGjx4sVtvo4CCQAAAEAyWMYYY3cQiVBeXq7c3FyVlZUpJyfH7nD0xqYXdKhin91hAAAAAEmRmZatsaNa3q/UDrHUBl1a5hsAAAAAnIQCCQAAAABCKJAAAAAAIIQCCQAAAABCKJAAAAAAIIQCCQAAAABCKJAAAAAAIIQCCQAAAABCKJAAAAAAIIQCCQAAAABCKJAAAAAAIIQCCQAAAABCKJAAAAAAIIQCCQAAAABCKJAAAAAAIIQCCQAAAABCKJAAAAAAIIQCCQAAAABCKJAAAAAAIIQCCQAAAABCKJAAAAAAIIQCCQAAAABCKJAAAAAAIIQCCQAAAEDcWbLsDqFTPHYHgNSS7k5TX38fu8OwlTFGe2r36nCwVpLksdzql3as3Jbb5sjQFQ0moF2HSxQwQbtDQSf18RUo25PZ5LHaYJ12H/5CRkaSlOPJUm9fvh3hIY4qGiq1t+5g5N+9ffnK8WTZGBHiYX9dqcoaKiL/7uvvo3R3mo0Roat8/gy7Q+gURpDQYS7LUm8vHQvLsnSMv1BuyyVLlo7196Y4cgCP5daxvt4pe7arp8v1ZLcojiTJ7/Kpt6+XJCnN5Veht1eyQ0MCZHuylBsqiHI8WRRHDlHgzVO62y9JKvT1ojiCbRhBQocVevPldfErIx3pTNebBvldPrvDQZykuf3q7eul0oZyu0NBDHyWTwXevDafz/ZkqcEElO3JkmVRADtFgbeXLMulfE+u3aEgTizL0jG+QpU1VCrXk213OOjB6O32UNmeTGW40tXRvoIlSxnu9MQGlWLS3H6lyW93GIizbE+Wsjkb7Ti9vHSincayrHYLY6Qmt+VWPu0VNqNAcrh8b658zUY40lw+poQBAAAAraBAcrgsd4a8Lq/dYQAAAAApgUUaHI7iCAAAAOg4CiQHY0EFAAAAIDYUSA7msxg9AgAAAGJBgeRgTK8DAAAAYkOB5GBeiyl2AAAAQCwokByMAgkAAACIDQWSgzHFDgAAAIgNBZJDuSxLHm4GCwAAAMSEAsmhvKxgBwAAAMSMAsmhuP4IAAAAiB0FkkNx/REAAAAQOwokh2IECQAAAIgdBZJD+RhBAgAAAGLGMEOS+Fw++ZNYtDCCBAAAAMSOXnSS9PYXKLueZbcBAACA7owpdgAAAAAQQoEEAAAAACEUSAAAAAAQQoEEAAAAACEUSAAAAAAQEnOBVF1draefflrXXXedhg0bprS0NGVmZmrkyJG66667VFlZGXMQpaWlmjlzpgYMGCC/368BAwZo1qxZOnToUMz7AgAAAIDOsowxJpYN/vSnP+n666+XJI0YMUInn3yyysvLtW7dOlVUVGj48OFas2aN+vTp06H97d+/X+eee64++ugjDRo0SKNHj9bmzZu1efNmDR06VK+//rry8/NjfmPl5eXKzc1VWVmZcnJyYt4+3j77aJMOVx6yOwwAAAAgKXz+DA0YcYbdYUiKrTaIeQTJ6/Xqhhtu0JYtW7Rlyxb99a9/1fPPP69t27bp9NNP19atWzVr1qwO72/WrFn66KOPVFxcrG3btumJJ57QBx98oJtuukkffvihZs+eHWuIAAAAANApMY8gtef111/XmDFj5Pf7VV5eLp/P1+7rS0pK1K9fP3k8Hn366ac65phjIs/V1taqf//+OnjwoPbs2dPhEakwRpAAAAAA+/SYEaT2jBw5UlJjcXPgwIGjvv75559XMBjU2LFjmxRHkuT3+/WNb3xDgUBAK1asiGeYAAAAANCquBZI27dvl9Q4Da8j1w29//77kqRRo0a1+nz48Y0bN8YpQgAAAABoW1wLpAceeECSNHHiRPn9/qO+/tNPP5Uk9evXr9Xnw4/v3LkzThECAAAAQNs88drRihUrtHDhQnm9Xt19990d2ia8JHhGRkarz2dmZkqSKioq2txHSUmJSkpK2tw3AAAAAHRUXAqkrVu36pprrpExRvfee2/kWqRkWLBgge68886k/TwAAAAAztXlAmn37t2aOHGiSktLNXv2bM2cObPD22ZlZUlqvPlsa6qqqiRJ2dnZbe7jxhtv1CWXXNLi8crKSo0bN67DsQAAAABAlwqkgwcPavz48dq5c6emT5+uefPmxbT98ccfL0natWtXq8+HHx8wYECb+ygqKlJRUVGLx8vLy2OKBQAAAAA6vUhDZWWlJk2apC1btqi4uFgPPfSQLMuKaR/hqXjvvPNOq8+HHz/11FM7GyYAAAAAdFinCqTa2lpNmTJF69ev14QJE7RkyRK53e6Y9zNx4kS5XC69+uqr2rt3b4uf8cwzz8jtduviiy/uTJgAAAAAEJOYC6RAIKBp06bppZde0tixY7V06VL5fL52t5k/f76GDx+uW2+9tcnjRUVFmjZtmurq6jRjxgw1NDREnrvlllu0b98+XXPNNerTp0+sYQIAAABAzGK+Bmn+/PlatmyZJKmwsFAzZsxo9XXz5s1TYWGhJGn//v3atm1bq8tx33///XrjjTf0t7/9TcOHD9fo0aO1efNmffDBBxoyZIjuu+++WEMEAAAAgE6JuUAqLS2N/D1cKLVm7ty5kQKpPYWFhVq/fr3mzp2rp59+WsuWLdMxxxyjH/7wh7rzzjuVl5cXa4gAAAAA0CmWMcbYHUQilJeXKzc3V2VlZcrJybE7HH320SYdrjxkdxgAAABAUvj8GRow4gy7w5AUW23Q6VXsAAAAAMBpKJAAAAAAIIQCCQAAAABCKJAAAAAAIIQCCQAAAABCKJAAAAAAICTm+yChc7LTPEoLeu0Oo0cxRgoaI2cuZI/uoiEQ1OGGoN1hAACAOKFASpK8dJ/k8tsdBoA4M0YqKTusqroGu0MBAABxwBQ7AOgCy5KKctOU4eN8EwAATsA3OgB0kWVJx+WlKdiJmXaHGwLafagm/kGhiUyfh1E+ByKvzkReYTdGkAAgTlyu2P9k+NzyuCy7Q3e8Xhle+T185TmJJal3tl+0Hmfxul3Kz/TZHQZ6OL4tAMBmmX4G8xPJbVlK97n5nB0m3eeR122RV4fJ8nuU5nXJ46b0hX0okADAZpl+t90hOFr4882iI+0omT536P/k1Uki7ZW8wkYUSABgswyvR8yyS5ystMaOlt/jktfN155TZIbymuF3M83OITwuS+leTmjAfnxTAIDNLItpdonishoL0DBG65whzeuWN3RWweOylOYlr04QfRxM97nltih9YQ8KJADoBiiQEiPT71F0H4uz0s7QPI/k1Rma55ETGrALRxQA6AayfB4dl5dudxiO03xKXbrXzefsAH5P045zdppXPlYpTHnpzUYC87P8yk7z2hQN4sHy+u0OoVMokACgG7CsxiW/kXh8zs7jdpFXJ/K6LHnJa2rzpGapwekWAAAAAAihQAIAAACAEAokAAAAAAihQAIAAACAEAokAAAAAAihQAIAAACAEAokAAAAAAihQAIAAACAEAokAAAAAAihQAIAAACAEAokAAAAAAihQAIAAACAEAokAAAAAAihQAIAAACAEAokAAAAAAihQAIAAACAEAokAAAAAAihQAIAAACAEAokAAAAAAihQAIAAACAEAokAAAAAAihQAIAAACAEAokAAAAAAihQAIAAACAEAokAAAAAAihQAIAAACAEAokAAAAAAihQAIAAACAEAokAAAAAAihQAIAAACAEI/dAfQYlmV3BEgEX7bdEQBItoYaKdhgdxSQJMstmYDdUSDeyCtsRoGULNl9pdpKGryTZBdJ2cfaHQWAZKurkvb/W5KxO5KezZMmFZwoHfi4sWiFM/hzpNx+0r5t9Jlgm05NsXv77bf161//WsXFxerXr58sy5LVhRGSF198UZMnT1bv3r3l9XpVUFCg8ePHa9myZZ3eZ7fjTZPyjrc7CsRLWi7FEdBT+TIbO3Cwj+WSep0gub1S/gmNIw5IfW6/1Gug5An9H7CJZYyJ+RTYpZdeqr///e8tHu/ErnT//ffrRz/6kSzL0rnnnqv+/fvrs88+0+uvvy5jjH72s5/pnnvuiXm/5eXlys3NVVlZmXJycmLePmEq90n1VXZHga6wXFLOcZKLL2SgR6v4XGo4bHcUPVN6r8YTVWGHy6Wag/bFg/jIOkbyph/5d9UBqa7CvnjQdW6flNPX7igkxVYbdKpA+u///m9VVVXpzDPP1JlnnqmBAweqtrY25gJp37596t+/v4LBoF588UWNGzcu8twrr7yi8ePHq66uTh999JEGDRoU0767bYEEAAAAIKliqQ06dQ3ST37yk04F1tybb76p2tpaTZgwoUlxJElf+cpXNGHCBP3jH//QW2+9FXOBBAAAAACxsnWZb7/f36HXFRQUJDgSAAAAALC5QDrrrLOUl5enl156SWvWrGny3CuvvKKVK1dqyJAhGjt2rE0RAgAAAOhJbC2QcnNztXDhQrlcLl1wwQX68pe/rP/zf/6PvvzlL+v888/XmWeeqZUrV8rn89kZJgAAAIAewvb7IBUXF+u5557TlVdeqbVr10Yez8nJ0fjx43Xccce1u31JSYlKSkpaPF5ZWRn3WAEAAAA4m60jSJL0m9/8Rl/96lf1la98RRs3blRlZaU2btyoCy+8UHfccYeKi4vb3X7BggU644wzWvxpvugDAAAAABxNp5b5bi4tLa1Ty3yvXr1aF1xwgUaNGqUNGzbI5TpSrwUCAY0ePVrvvfeeVqxYoUmTJrW6j/ZGkMaNG8cy3wAAAEAPl/BlvuPl8ccflyRNnTq1SXEkSW63W8XFxXrvvff0yiuvtFkgFRUVqaioqMXj5eXl8Q8YAAAAgKPZOsVu165dkhoXa2hN+PHS0tKkxQQAAACg57K1QDr22GMlSW+99Varz2/YsEGSNHDgwGSFBAAAAKAHS0qBNH/+fA0fPly33nprk8cvvfRSSdKf//xnLV++vMlzf//737V48WK5XC5NnTo1GWECAAAA6OE6dQ3Ss88+q7vvvjvy77q6OknSOeecE3ns9ttv1+TJkyVJ+/fv17Zt21ospnDppZfqiiuu0JNPPqlvfOMbGj16tE444QTt2LEjMqp0zz33aNiwYZ0JEwAAAABi0qkCad++fXrzzTdbPB792L59+466H8uy9MQTT2jixIl69NFHtXHjRr333nvKy8vTxRdfrJtuukkTJ07sTIgAAAAAELO4LPPdHcWylB8AAAAA54qlNrD9RrEAAAAA0F1QIAEAAABACAUSAAAAAIRQIAEAAABACAUSAAAAAIR0apnvVBBenK+8vNzmSAAAAADYKVwTdGQBb8cWSBUVFZKk/v372xwJAAAAgO6goqJCubm57b7GsfdBCgaD2rNnj7Kzs2VZlq2xvPfeexo3bpzWrFmj0047zdZY0HXk0znIpbOQT+cgl85CPp0jlXNpjFFFRYX69u0rl6v9q4wcO4LkcrnUr18/u8OQJGVlZUX+z01rUx/5dA5y6Szk0znIpbOQT+dI9VwebeQojEUaAAAAACCEAgkAAAAAQiiQAAAAACCEAgkAAAAAQiiQkqCoqEhz5sxRUVGR3aEgDsinc5BLZyGfzkEunYV8OkdPyaVjl/kGAAAAgFgxggQAAAAAIRRIAAAAABBCgQQAAAAAIRRICVRTU6M77rhDQ4cOVVpamvr27av/+I//0O7du+0ODa04//zzZVlWm3+ef/75VrdbtGiRzjrrLGVlZSk/P18XX3yx1q1bl+Toe6a3335bv/71r1VcXKx+/fpFcnU0ncnZ2rVrdfHFFys/P19ZWVk666yz9Nhjj8XrrfR4seZy7ty57bbXn/70p21uSy4Tq7q6Wk8//bSuu+46DRs2TGlpacrMzNTIkSN11113qbKyss1taZvdS2dySdvs3u677z4VFxdryJAhys3Nld/v14ABA/Ttb39bmzZtanO7Htc2DRKipqbGnHPOOUaSKSoqMldeeaU566yzjCTTu3dv8/HHH9sdIpoZN26ckWQuu+wyc+2117b4s3HjxhbbzJw500gy6enpZsqUKWbChAnG4/EYt9ttli1blvw30cNMmTLFSGrxpz2dydlTTz1l3G63sSzLjBs3zlx22WUmLy/PSDI333xzAt5ZzxNrLufMmWMkmfPOO6/V9vrXv/611e3IZeI99NBDkfyNGDHCXHHFFWbChAkmOzvbSDLDhw83X3zxRYvtaJvdT2dySdvs3goKCkxaWpo566yzzNSpU83UqVPN0KFDjSTj9XrNM88802Kbntg2KZAS5LbbbjOSzLnnnmsqKioij//mN78xksy4cePsCw6tChdIO3bs6NDrX3zxRSPJFBQUmA8//DDy+Lp164zP5zN5eXmmtLQ0McHCGGPMr3/9a3P77bebf/zjH6akpMT4/f52O9WdydmBAwdMTk6OkWT+9re/RR7//PPPzYknnmgkmZdffjneb63HiTWX4U7YI4880uGfQS6TY9GiReaGG24wW7ZsafL4nj17zOmnn24kmWnTpjV5jrbZPXUml7TN7u21114zNTU1LR5/8MEHjSRzzDHHmPr6+sjjPbVtUiAlQG1trcnNzTWSzDvvvNPi+VNPPdVIMm+99ZYN0aEtsRZIkyZNMpLMb3/72xbP/fCHPzSSzLx58+IbJNp1tE51Z3L23//930aSmTJlSottli5daiSZr3/9610NHc0kokAil/Zbt26dkWT8fr+pra2NPE7bTD1t5ZK2mboGDx5sJJn3338/8lhPbZtcg5QAa9euVVlZmQYPHqzTTz+9xfOXX365JOmZZ55JdmiIk5qaGr300kuSjuQzGjnufjqbs2effbbNbSZPnqy0tDStWrVKhw8fjnfIiDNyab+RI0dKkmpra3XgwAFJtM1U1VouO4tcdg9er1eS5PP5JPXstumxOwAnev/99yVJo0aNavX58OMbN25MWkzouIULF+rAgQNyuVwaOnSoLr30Uh1//PFNXrNt2zbV1taqd+/e6tevX4t9kOPup7M5a689+3w+nXzyyXrrrbf04Ycf6tRTT01A5GjPSy+9pPfee0+HDx9Wv379NGnSJJ1xxhmtvpZc2m/79u2SGjti+fn5kmibqaq1XEajbaaWxx9/XNu2bdOQIUM0ZMgQST27bVIgJcCnn34qSa3+MkU/vnPnzqTFhI77xS9+0eTf//Vf/6Xbb79dt99+e+Sxo+U4MzNTeXl5Ki0tVUVFhbKzsxMXMDqkMzkrLy9XWVlZu9v169dPb731lnbu3NltD/RO9vjjjzf59+23367LLrtMixYtUlZWVuRxctk9PPDAA5KkiRMnyu/3S6JtpqrWchmNttm93Xvvvdq8ebOqqqr0r3/9S5s3b1bfvn21ZMkSud1uST27bTLFLgHCy15mZGS0+nxmZqYkqaKiImkx4ei+8pWv6PHHH9fHH3+s6upqbdu2Tffcc488Ho/uuOOOyJeBdPQcS+S5u+lMzqKXsKU9dy8nnnii5s2bp82bN6uyslKfffaZ/vznP+u4447T3/72N33rW99q8npyab8VK1Zo4cKF8nq9uvvuuyOP0zZTT1u5lGibqWLlypV69NFH9dRTT2nz5s0aMGCAlixZ0mSUrye3TQokIOSuu+7SNddco0GDBik9PV1Dhw7Vz372Mz399NOSGu/tUFNTY2+QACRJ11xzjW6++WZ96UtfUmZmpvr166err75aGzZsUEFBgZ5++mm98cYbdoeJkK1bt+qaa66RMUb33ntv5PoVpJ6j5ZK2mRpWrVolY4xKS0v1yiuvaMiQIRo3bpzuueceu0PrFiiQEiA8dFxdXd3q81VVVZLEtKsUMX78eI0ePVqHDh3Sm2++KenoOZbIc3fTmZxFTwOhPaeGoqIiTZ8+XZKa3NyZXNpn9+7dmjhxokpLSzV79mzNnDmzyfO0zdRxtFy2h7bZPeXl5Wns2LFasWKFzjjjDN1+++3asGGDpJ7dNimQEiB8Qf+uXbtafT78+IABA5IWE7omfMFiSUmJpKPnuKqqSocOHVKvXr269QGgJ+lMznJycpSbm9vudrTn7qd5e5XIpV0OHjyo8ePHa+fOnZo+fbrmzZvX4jW0zdTQkVweDW2z+/J6vbrqqqtkjImsSteT2yYFUgKEh5vfeeedVp8PP95dL0xDS6WlpZKOzJsdNmyY/H6/9u3bp927d7d4PTnufjqbs/bac319vT744AOlpaVp6NChCYgandG8vYaRy+SqrKzUpEmTtGXLFhUXF+uhhx6SZVktXkfb7P46msujoW12b4WFhZKkffv2SerZbZMCKQHOO+885ebm6uOPP9Z7773X4vmnnnpKkvSNb3wjyZGhM/bt26dXX31V0pElK9PT03XhhRdKkp588skW25Dj7qezOZs8eXKT56MtX75chw8f1le/+lWlpaXFO2R0gjFGy5Ytk9RyiVlymTy1tbWaMmWK1q9frwkTJjRZGas52mb3Fksu20Pb7P7WrFkjSRo8eLCkHt427bxLrZPddtttRpIZM2aMqaysjDz+m9/8xkgy48aNsy84tLB27VqzbNky09DQ0OTxHTt2mPPOO89IMpdcckmT51588UUjyRQUFJgPP/ww8vi6deuM3+83eXl5prS0NBnhI8Tv95v2DmudydmBAwdMTk6OkWT+9re/RR7/4osvzIknnmgkmZdffjneb6XHay+Xe/fuNfPnzzfl5eVNHq+oqDA33nijkWSOPfZYU1VV1eR5cpkcDQ0NZurUqUaSGTt2bIs8tIa22T3FmkvaZvf22muvmeeee84EAoEmj9fV1Znf/e53xuVymfT0dPPpp59GnuupbZMCKUFqamrM2WefbSSZoqIic+WVV0b+3bt3b/Pxxx/bHSKiPPLII5ED98UXX2yuvvpqc95555m0tDQjyZx00knmiy++aLHdzJkzjSSTkZFhpkyZYiZNmmQ8Ho9xu91m2bJlyX8jPczy5cvN2WefHfljWZaR1OSx5cuXN9mmMzl76qmnjMvlMpZlmQsuuMBcfvnlJi8vz0gys2fPTsI7db5Ycrljxw4jyWRlZZkLLrjAXH311eZrX/uaKSgoMJJMXl6eee2111r9OeQy8e6//34jyUgyU6dONddee22rf/bt29dkO9pm9xNrLmmb3Vu4r1NYWGgmTJhgrr76ajN+/HhTVFRkJJm0tDTzxBNPtNiuJ7ZNCqQEqq6uNrfffrsZPHiw8fl85thjjzXf+c53zGeffWZ3aGhmy5Yt5vvf/74ZNWqU6d27t/F4PCY3N9ecc8455je/+Y2prq5uc9tHHnnEnHHGGSYjI8Pk5eWZiRMnmrVr1yYx+p4rfLBv788jjzzS6nax5uy1114zEydONHl5eSYjI8OMHj3aLFq0KEHvrOeJJZfl5eXmJz/5iRk3bpw57rjjjN/vNxkZGeakk04yN998s9m1a1e7P4tcJtacOXOOmktJZseOHS22pW12L7HmkrbZvW3fvt387Gc/M+edd54pKioyXq/XZGZmmpNOOsncdNNN5t///neb2/a0tmkZY0ys0/IAAAAAwIlYpAEAAAAAQiiQAAAAACCEAgkAAAAAQiiQAAAAACCEAgkAAAAAQiiQAAAAACCEAgkAAAAAQiiQAAAAACCEAgkAHMayLFmWpblz59odSrcVCAT0wAMP6KyzzlJOTk7kM7v00kvtDs2xBg4cKMuy9J3vfMfuUACgXRRIABxj9erVkY6uZVm66qqrjrrNd77zncjr0XNMmzZNs2bN0oYNG1RRUdGpfUT/vlGMAoBzUCABcKwnn3xSmzZtsjsMdDPr1q3Tk08+KUmaPHmyXnzxRW3cuFGbNm3S7373u7j9nE8++SRSQC1atChu++1Ozj//fFmWpfPPP9/uUAAgbjx2BwAAiWKM0Zw5c7R06VK7Q0E3smrVKkmS2+3W4sWLlZOTY3NEPcMnn3xidwgA0CGMIAFwpMLCQknSsmXL9O6779ocDbqT3bt3S5KOOeYYiiMAQAsUSAAc6Yc//KH8fr8k6Y477rA5GnQntbW1kiSv12tzJACA7ogCCYAj9e/fXzfccIMkafny5Vq/fn2n9tPRlbfCiz0MHDiwxXOtXYuydOlSjR8/Xn369FFmZqZGjhyp3//+96qvr49sZ4zR4sWLdf7556tPnz7KyMjQqFGj9D//8z8yxnT4PaxatUqXXHKJioqKlJaWpkGDBukHP/hBZCTlaN555x1973vf07Bhw5SVlaXMzEwNGzZM3//+9/Xhhx+2ud2iRYsi7/uTTz5RbW2t7r//fp1zzjkqLCzs0uIGmzZt0g033KAhQ4YoIyND2dnZOumkk/SjH/2ozalc4VgeffRRSdLOnTubLOoRz4U6LMvSCSecEPn39OnTW/ystt57sj7vuro6PfPMM/rBD36gM888U7169ZLX61VBQYHOPvtszZ07V/v372/1Z4V/39esWSNJWrNmTYv317wtdLQtPfPMM7r88svVr18/+f1+FRQU6Nxzz9Wvf/1rVVZWdvj9B4NB/fGPf9SYMWPUq1cvZWZm6tRTT9U999yj6urqdmMA0MMZAHCIl19+2Ugykswjjzxi9uzZY9LT040kM378+Fa3ufbaayPbtGbAgAFGkrn22mvb/dnh/QwYMKDFczt27GgS1/e///3Iv5v/KS4uNg0NDebw4cPm8ssvb/N1119/fZuxhF8zZ84cM3fu3Db3kZuba1555ZU29xMIBMyPfvQjY1lWm/vweDxmwYIFrW7/yCOPRF63YcMGc9ppp7XYfs6cOe1+rq355S9/aVwuV5sx+f1+8+ijj7b5ubT3JxbRv2/N30dHflbzbZL9eUf/7rf1p6CgwLz22mstflZHtm3eFo7WlmpqaszUqVPb3Wffvn3Nu+++e9T3v3nzZnPRRRe1uZ+zzjrLVFZWtrofAKBAAuAYzQskY4yZPXt25LFXX321xTbJLpDOPvtsI8lcfPHFZunSpebtt982Tz/9dORxSeahhx4yN910k5Fkrr76arN8+XLz9ttvm7/85S9m+PDhkdc999xzrcYSfn706NFGkhk2bJhZuHCh2bBhg1m1apW58cYbIwVGTk6O+fTTT1vdz4wZMyL7+spXvmIefvhhs3r1arN+/Xrz0EMPmZNOOiny/N///vcW20d3WE899VRjWZb59re/bZ599lnz9ttvm2XLlpkVK1a0+7k29+CDD0b22bt3bzNv3jzz+uuvm9dee83MnTvXZGZmGknGsizz7LPPNtl206ZNZtOmTWbKlCmRznb4sfCfWLRXIG3atMmsXLky8vwvfvGLFj/riy++aLJNsj/vb37zm2bQoEHm5ptvNk888YR5/fXXzYYNG8xTTz1lvve97xmfzxf5nJvHumvXLrNp06bI79jo0aNbvL9t27Y12eZobenKK6+MxD9y5Ejz2GOPmQ0bNpiVK1ea6dOnRwrH/Px8s2vXrnbf/5gxY4zL5TLXXnttk/d/7rnnRl7z05/+tL30AujBKJAAOEZrBdIXX3wR6TRfcMEFLbZJdoEkycyaNavFa6qqqiI/q6CgwFiWZe6///4WryspKTHZ2dlGkrnkkktajSX6Z40aNcpUVFS0eM1jjz0Wec0VV1zR4vkXXngh8vyf/vSnVn9OTU2NufDCCyPvu76+vsnz0R3W9vbTUXv37jUZGRmR4qa1wu6dd96J5Pu4444zdXV1LV7TXq5i0V6BZEzLkcP22PF5f/TRRyYYDLb5/MaNG01WVpaRZH7+85+3+ppx48YZSWbcuHHt/ixj2m9Ly5cvj8R90UUXmdra2hav+eMf/xh5zZVXXtni+ebv//HHH2/xmsOHD5uTTz450s6af4YAYIwxXIMEwNH69OmjH/zgB5Kkl19+WS+//LKt8fTv31//9//+3xaPZ2Rk6Nprr5UkHThwQGeffbZmzpzZ4nXHHnuspk6dKkl69dVXj/rz/vjHPyorK6vF49/61rc0adIkSY0r/X3++edNnv/1r38tSbrssst03XXXtbrvtLQ0zZ8/X1Lj9TztfbYXXnhhm/vpqEceeSRy7ch9992n/v37t3jN6aefrltvvVVS42p1Tz/9dJd+ZrLY8XkPHjy43euuTjnlFH33u9+VpIR/jg8++KCkxoUzHnnkEfl8vhavuf766/XVr35VUuM1fCUlJW3ur7i4WNdcc02Lx/1+f+R4cODAAW3ZsiUe4QNwGAokAI734x//WNnZ2ZKk22+/3dZYiouL21w9beTIkZG/X3XVVW3uI/y60tJSHTp0qM3XnXLKKTrjjDPafP4//uM/JEkNDQ1avXp15PHy8vLIvy+//PI2t5ekESNGRJZUf/3119t83Te/+c1299MR4fsX5eXlqbi4uM3XhTv10dt0Z93l8y4tLdXHH3+szZs364MPPtAHH3ygvLw8SdKWLVuaLCASTw0NDZHFHsaPH99q4Rt2/fXXR7aJ/p1trr33H90mtm/fHmO0AHoCCiQAjldQUKBZs2ZJktauXauVK1faFsvQoUPbfC7cGY3ldRUVFW2+7swzz2w3lrPOOivy902bNkX+/u677yoYDEqSpk2b1mJ1suZ/wiudNR+Finbqqae2G0tHfPDBB5KkUaNGtbtE9zHHHBNZQS28TXdm5+e9adMm/cd//IeKioqUn5+vE088USeffLJOOeUUnXLKKZFV74LBoEpLS7v2Rtuwffv2yMjg2Wef3e5ro59vL7fDhw9v87n8/PzI39trPwB6LgokAD3C7NmzI4XFnDlzbIsjIyOjzedcLlfMrwsEAm2+rk+fPu3Gcswxx0T+fvDgwcjf9+7d2+52bWlv6eRevXp1ap/RwjEe7X1JjVMRo7fpzuz6vBcuXKhRo0bpkUceabfYCqupqYkpvo6KztHRchvOa/PtmotH+wHQc3nsDgAAkiEvL0+zZ8/WHXfcoTfffFPLly/X17/+dbvDSqjO3tcnutO4YMECjRkzpkPbtdcpd7vdnYqlNfG8X1F3YMfnvXXrVn3ve99TQ0OD+vTpox//+Me68MILNXDgQGVnZ0dG6B5++OHItUwmhntvdZbTcgsgNVEgAegxZs2apQceeEAHDhzQnDlzOlQghc82h6dAtaWqqiouMcbTF1980eHno6cdFRQURP6ekZGhk08+Of7BdUJ+fr5KSkqO+r6kI9PPot9Xd2XH571o0SI1NDTI7XZrzZo1bU5JS8YIXHSOjpbb6JGuVMgtgNTEFDsAPUZ2drZ+/OMfS5LeeecdLVu2rEPbSDrq9Rcffvhh1wOMsw0bNnT4+ehO+WmnnRY5k7927drEBNcJ4RjfeecdNTQ0tPm6vXv3aufOnU22sUNHR0Ps+Lw3b94sqXHBj/au13nrrbfa3U88RnwGDRoUmRL35ptvtvva9evXR/7eXQp3AM5DgQSgR/nBD34Quc5hzpw5R502dMIJJ0hq7JS39drNmzdr48aN8Q00DjZt2qR33323zecffvhhSY3Tsc4///zI471799Y555wjSVq8eLH27duX0Dg7KrzE86FDh7R06dI2X7dw4cJIrsLb2CEtLS3y99ra2jZfZ8fnHS4w2xv5LCkp0T/+8Y929xN+j+29v6PxeDwaN26cJOnFF1/Url272nztn/70p8g20b+zABBPFEgAepTMzEz95Cc/kdRYQKxYsaLd14c7bnv27NGSJUtaPF9RUdHl+/sk0g033NBqJ3jx4sWR937ppZeqqKioyfM///nPJTUuQX355Ze3u5x4bW2tHnzwQR0+fDh+gbdi+vTpkZGGm2++Wbt3727xmvfff1+//OUvJUnHHXecLr300oTG1J6CgoLI/Xw+/vjjdl+b7M97yJAhkqR///vfWrduXYvnq6urdfXVVx91YYbw78327du7dI3Sf/7nf0qS6urqdN1117W6pPjDDz+sF154QVLjcvnNf2cBIF4okAD0ON///vcjnavwksltueaaa5STkyNJuu6663TXXXfpzTff1Pr16/X//t//06hRo/T+++/r9NNPT3jcsRo9erTeeustjR49WosWLdLbb7+tl156STNmzNC3vvUtSY1TCOfNm9di24svvjhyo9pXXnlFI0aM0J133ql//vOfeu+997R27Vo9+uij+u53v6uioiL94Ac/aHfaWzz07t1b9957ryRp165dOuOMM3T//fdr/fr1Wrdune666y59+ctfVmVlpSzL0h//+Md2lwNPNI/HE1lq/eGHH9aSJUv0r3/9Sx999JE++uijJtf3JPvzDuc/GAxq8uTJ+uUvf6lXXnkl8nt92mmnafXq1TrvvPPa3U94QYm9e/dq9uzZevvttyPvLzzNsSMmT56sK664QpL0wgsv6JxzztGf//xnvf3221q1apW++93vRu5vlZ+fr/vuu68zbxsAOsYAgEO8/PLLRpKRZB555JF2X/v73/8+8trwn7b89a9/NW63u8XrJZn09HTz5JNPmmuvvdZIMgMGDGix/Y4dOzoUV3T8L7/8cpuve+SRRyKv27FjR4vnw8/NmTPHzJkzp9W4JZmcnByzevXqNn9OMBg0d955p/F4PG3uI/wnMzPTVFdXxxRnZ91zzz3G5XK1GYvf7zePPvpom9u3l6tYROdrzpw5rb5m+fLlxrKsVuNsvk2yP+8777yz3Z9x8803H3WfFRUVZtCgQa1u3/zzHTBggJFkrr322lbjqampMVOnTm03pr59+5p333231e07+v472h4B9FyMIAHoka6//nr179+/Q6+94oortG7dOk2dOlW9e/eWz+dT//79de2112rDhg26/PLLExxt582dO1fPP/+8Jk+erGOOOUY+n08DBw7UjBkztHnz5sgUwtZYlqU77rhDH374oW655RaNHj1a+fn5crvdys7O1pe+9CV985vf1KOPPqqSkhKlp6cn5T397Gc/07vvvqvrr79egwcPVnp6ujIzMzVixAjNnDlTW7du1be//e2kxHI0kydP1j//+U9NmTJFffv2bXdEK9mf9x133KFnn31W48ePV69eveTz+dSvXz8VFxfrhRdeaHVksbmsrCytW7dOM2fO1IgRI9q9/9DRpKWlaenSpfrHP/6h4uJi9e3bVz6fT7169dLZZ5+tX/3qV9q2bZtOO+20Tv8MAOgIy5gk3NgAAAAAAFIAI0gAAAAAEEKBBAAAAAAhFEgAAAAAEEKBBAAAAAAhFEgAAAAAEEKBBAAAAAAhFEgAAAAAEEKBBAAAAAAhFEgAAAAAEEKBBAAAAAAhFEgAAAAAEEKBBAAAAAAhFEgAAAAAEEKBBAAAAAAhFEgAAAAAEPL/Ac+0sVidPudiAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "mean_best_observed = []\n", + "std_best_observed = []\n", + "\n", + "for i in range(3):\n", + " mean_best_observed.append(np.mean(best_observed_all_clusters_all_trials[:][i][:], axis=0))\n", + " std_best_observed.append(np.std(best_observed_all_clusters_all_trials[:][i][:], axis=0))\n", + "\n", + "# Plot the mean and standard deviation of the best observed values\n", + "plt.figure(figsize=(10,6))\n", + "for i in range(3):\n", + " plt.plot(mean_best_observed[i], label=f'Cluster {i+1}')\n", + " plt.fill_between(range(len(mean_best_observed[i])), mean_best_observed[i] - std_best_observed[i], mean_best_observed[i] + std_best_observed[i], alpha=0.2)\n", + "plt.xlabel('Number of Iteration') " + ] } ], "metadata": { diff --git a/src/BO_2.ipynb b/src/BO_2.ipynb deleted file mode 100644 index 3ae176c..0000000 --- a/src/BO_2.ipynb +++ /dev/null @@ -1,945 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "a45452e9-567c-4658-bfa1-f9a6f6b70bd1", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Using cpu device\n" - ] - }, - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# This file costructs surrogate models for the input datasets\n", - "import numpy as np \n", - "import pandas as pd\n", - "import os\n", - "import shutil\n", - "import json\n", - "import math\n", - "import time\n", - "import warnings\n", - "from sklearn.model_selection import train_test_split\n", - "from sklearn.metrics import mean_squared_error\n", - "from joblib import Parallel, delayed, dump\n", - "\n", - "# Torch specific module imports\n", - "import torch\n", - "import gpytorch \n", - "\n", - "# botorch specific modules\n", - "from botorch.fit import fit_gpytorch_model\n", - "from botorch.models.gpytorch import GPyTorchModel\n", - "from botorch.optim import optimize_acqf, optimize_acqf_discrete\n", - "from botorch import fit_gpytorch_mll\n", - "from botorch.acquisition.monte_carlo import (\n", - " qExpectedImprovement,\n", - " qNoisyExpectedImprovement,\n", - ")\n", - "from botorch.sampling.normal import SobolQMCNormalSampler\n", - "from botorch.exceptions import BadInitialCandidatesWarning\n", - "from botorch.acquisition import UpperConfidenceBound, ExpectedImprovement\n", - "\n", - "# Plotting libraries\n", - "import matplotlib as mpl\n", - "import matplotlib.pyplot as plt\n", - "%matplotlib inline\n", - "%load_ext autoreload\n", - "%autoreload 2\n", - "\n", - "# Tick parameters\n", - "plt.rcParams['xtick.labelsize'] = 15\n", - "plt.rcParams['ytick.labelsize'] = 15\n", - "plt.rcParams['xtick.major.size'] = 5\n", - "plt.rcParams['xtick.major.width'] = 1\n", - "plt.rcParams['xtick.minor.size'] = 5\n", - "plt.rcParams['xtick.minor.width'] = 1\n", - "plt.rcParams['ytick.major.size'] = 5\n", - "plt.rcParams['ytick.major.width'] = 1\n", - "plt.rcParams['ytick.minor.size'] = 5\n", - "plt.rcParams['ytick.minor.width'] = 1\n", - "\n", - "plt.rcParams['axes.labelsize'] = 15\n", - "plt.rcParams['axes.titlesize'] = 15\n", - "plt.rcParams['legend.fontsize'] = 15\n", - "\n", - "# User defined python classes and files\n", - "import input_class \n", - "import code_inputs as model_input\n", - "import utils_dataset as utilsd\n", - "import surrogate_models\n", - "import kmeans as km\n", - "\n", - "# Set the random seeds\n", - "np.random.seed(0)\n", - "torch.manual_seed(0)" - ] - }, - { - "cell_type": "markdown", - "id": "8b29bdc5", - "metadata": {}, - "source": [ - "#### K means clustering" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "a8de62ac", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
num carbonnum fluorinenum hydrogennum nitrogennum oxygennum sulfurnum silicon
036002161447200
1360021614414400
243203601447200
3360014421621600
4360014421621600
........................
69835996057696000
698361020057648000
698371360076864000
6983818880115212812800
69839536028832000
\n", - "

69840 rows × 7 columns

\n", - "
" - ], - "text/plain": [ - " num carbon num fluorine num hydrogen num nitrogen num oxygen \\\n", - "0 360 0 216 144 72 \n", - "1 360 0 216 144 144 \n", - "2 432 0 360 144 72 \n", - "3 360 0 144 216 216 \n", - "4 360 0 144 216 216 \n", - "... ... ... ... ... ... \n", - "69835 996 0 576 96 0 \n", - "69836 1020 0 576 48 0 \n", - "69837 1360 0 768 64 0 \n", - "69838 1888 0 1152 128 128 \n", - "69839 536 0 288 32 0 \n", - "\n", - " num sulfur num silicon \n", - "0 0 0 \n", - "1 0 0 \n", - "2 0 0 \n", - "3 0 0 \n", - "4 0 0 \n", - "... ... ... \n", - "69835 0 0 \n", - "69836 0 0 \n", - "69837 0 0 \n", - "69838 0 0 \n", - "69839 0 0 \n", - "\n", - "[69840 rows x 7 columns]" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "Input = input_class.inputs(input_path='../datasets/')\n", - "XX_prop, YY, descriptors = Input.read_inputs()\n", - "XX_comp_df, YY_df = Input.get_comp()\n", - "XX_comp_df" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "fe95ea1f", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
num carbonnum fluorinenum hydrogennum nitrogennum oxygennum sulfurnum silicondeliverable capacity [v STP/v]
08320448384000165.565439
1115208321286400152.524690
2137608962566400115.996501
38640720192000143.024802
410880768128000153.528996
...........................
6983515360960160000110.196985
698361440013682160036137.095297
6983725600153638438400169.809763
698382784018245769600110.963253
69839192001200964800166.323232
\n", - "

69840 rows × 8 columns

\n", - "
" - ], - "text/plain": [ - " num carbon num fluorine num hydrogen num nitrogen num oxygen \\\n", - "0 832 0 448 384 0 \n", - "1 1152 0 832 128 64 \n", - "2 1376 0 896 256 64 \n", - "3 864 0 720 192 0 \n", - "4 1088 0 768 128 0 \n", - "... ... ... ... ... ... \n", - "69835 1536 0 960 160 0 \n", - "69836 1440 0 1368 216 0 \n", - "69837 2560 0 1536 384 384 \n", - "69838 2784 0 1824 576 96 \n", - "69839 1920 0 1200 96 48 \n", - "\n", - " num sulfur num silicon deliverable capacity [v STP/v] \n", - "0 0 0 165.565439 \n", - "1 0 0 152.524690 \n", - "2 0 0 115.996501 \n", - "3 0 0 143.024802 \n", - "4 0 0 153.528996 \n", - "... ... ... ... \n", - "69835 0 0 110.196985 \n", - "69836 0 36 137.095297 \n", - "69837 0 0 169.809763 \n", - "69838 0 0 110.963253 \n", - "69839 0 0 166.323232 \n", - "\n", - "[69840 rows x 8 columns]" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "clustered_dfs = km.k_means(XX_comp_df, YY_df, model_input.NUM_CLUSTER)\n", - "sample_dfs = km.draw_samples(clustered_dfs, sample_fraction = 1.00)\n", - "samples = km.concat(sample_dfs)\n", - "samples" - ] - }, - { - "cell_type": "markdown", - "id": "35ed4601", - "metadata": {}, - "source": [ - "#### Acquisition function " - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "955bc734-96c5-4d3a-9325-920c041e256b", - "metadata": {}, - "outputs": [], - "source": [ - "## TODO: TO BE Check\n", - "bounds = torch.tensor([[-10.0], [12.0]])\n", - "\n", - "batch_size = 1\n", - "num_restarts= 10 \n", - "raw_samples = 512\n", - "\n", - "def optimize_acqf_and_get_observation(acq_func, X_test, Y_test):\n", - " \"\"\"Optimizes the acquisition function, and returns a new candidate\"\"\"\n", - " # print(X_test)\n", - " # print(Y_test)\n", - " # optimize\n", - " candidates, _ = optimize_acqf_discrete(\n", - " acq_function=acq_func,\n", - " choices=X_test,\n", - " q=batch_size,\n", - " max_batch_size=2048,\n", - " num_restarts=num_restarts,\n", - " raw_samples=raw_samples, # used for intialization heuristic\n", - " options={\"batch_limit\": 5, \"maxiter\": 200},\n", - " unique=True\n", - " )\n", - " \n", - " print(candidates)\n", - " # observe new values\n", - " new_x = candidates.detach()\n", - " b = [1 if torch.all(X_test[i].eq(new_x)) else 0 for i in range(0,X_test.shape[0]) ]\n", - " b = torch.tensor(b).to(torch.int)\n", - " index = b.nonzero()[0][0]\n", - " new_y = torch.reshape(Y_test[0,index],(1,1))\n", - " \n", - " X_test_new = X_test[torch.arange(0, X_test.shape[0]) != index, ...]\n", - " Y_test_new = Y_test[..., torch.arange(0, Y_test.shape[1]) != index]\n", - " \n", - " return new_x, new_y, index, X_test_new, Y_test_new" - ] - }, - { - "cell_type": "markdown", - "id": "f93f668d", - "metadata": {}, - "source": [ - "#### GP Train Function" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "id": "72bb9112-1749-44fd-bc9d-7c0edb2e59a6", - "metadata": {}, - "outputs": [], - "source": [ - "def create_train_test_data(cluster_dataXX, cluster_dataYY, random_seed):\n", - " if model_input.STANDARDIZE:\n", - " cluster_dataXX, scalerX_transform = utilsd.standardize_data(cluster_dataXX)\n", - " cluster_dataYY, scalerY_transform = utilsd.standardize_data(cluster_dataYY.reshape(-1,1))\n", - " else:\n", - " scalerX_transform = None\n", - " scalerY_transform = None\n", - " \n", - " ## TODO : Incase for feature selection\n", - " # ....\n", - " # ....\n", - " # ....\n", - "\n", - " # Create train and test sets\n", - " X_train, X_test, Y_train, Y_test = train_test_split(cluster_dataXX, cluster_dataYY, test_size=model_input.TEST_SIZE, random_state=random_seed)\n", - "\n", - " # Convert to tensors\n", - " X_train = torch.tensor(X_train, dtype=torch.float32)\n", - " X_test = torch.tensor(X_test, dtype=torch.float32)\n", - " Y_train = np.transpose(Y_train) # IMP : Has to have only one row for GP training\n", - " Y_train = torch.tensor(Y_train, dtype=torch.float32)\n", - " Y_test = np.transpose(Y_test)\n", - " Y_test = torch.tensor(Y_test, dtype=torch.float32)\n", - "\n", - " return X_train, X_test, Y_train, Y_test, scalerX_transform, scalerY_transform\n", - "\n", - "def train_gp(X_train, X_test, Y_train, Y_test, model=None):\n", - " best_observed = []\n", - " # Finding best value in initial data\n", - " if model_input.MAXIMIZATION:\n", - " best_observed_value = Y_train.max()\n", - " optimal_solution = torch.cat([Y_train[0],Y_test[0]]).max()\n", - " else:\n", - " best_observed_value = Y_train.min()\n", - " optimal_solution = torch.cat([Y_train[0],Y_test[0]]).min()\n", - " \n", - " # If optimal value is present in the initial dataset sample remove it \n", - " if (best_observed_value.eq(optimal_solution)) and model_input.MAXIMIZATION:\n", - " print('Max in training set, removing it before training models.')\n", - " optimal_position = torch.argmax(Y_train)\n", - " \n", - " # Add max value to test/exploration set\n", - " X_add_toTest = torch.reshape(X_train[optimal_position,:],(1,X_train.shape[1]))\n", - " X_test = torch.cat([X_test,X_add_toTest])\n", - " Y_add_toTest = torch.reshape(optimal_solution,(1,1)) \n", - " Y_test = torch.cat((Y_test,Y_add_toTest),1)\n", - " \n", - " # Remove max value from training set\n", - " X_train = X_train[torch.arange(0, X_train.shape[0]) != optimal_position, ...]\n", - " Y_train = Y_train[..., torch.arange(0, Y_train.shape[1]) != optimal_position]\n", - " \n", - " # Update best observed value\n", - " best_observed_value = Y_train.max()\n", - " \n", - " elif (best_observed_value.eq(optimal_solution)) and not model_input.MAXIMIZATION:\n", - " print('Min in training set, removing it before training models.')\n", - " optimal_position = torch.argmin(Y_train)\n", - " \n", - " # Add min value to test/exploration set\n", - " X_add_toTest = torch.reshape(X_train[optimal_position,:],(1,X_train.shape[1]))\n", - " X_test = torch.cat([X_test,X_add_toTest])\n", - " Y_add_toTest = torch.reshape(optimal_solution,(1,1)) \n", - " Y_test = torch.cat((Y_test,Y_add_toTest),1)\n", - " \n", - " # Remove min value from training set\n", - " X_train = X_train[torch.arange(0, X_train.shape[0]) != optimal_position, ...]\n", - " Y_train = Y_train[..., torch.arange(0, Y_train.shape[1]) != optimal_position]\n", - " \n", - " # Update best observed value\n", - " best_observed_value = Y_train.min()\n", - " \n", - " # Initialize data for training gp-0 and gp-l models\n", - " X_train0, Y_train0, X_test0, Y_test0 = X_train, Y_train, X_test, Y_test\n", - " \n", - " n_batch = int(model_input.N_BATCH_PER_TRIAL/model_input.N_SEARCH)\n", - " \n", - " # Initialize likelihood, GP model and acquisition function for the models\n", - " #--------------------------- GP-0 ---------------------------#\n", - " likelihood_gp0 = gpytorch.likelihoods.GaussianLikelihood()\n", - " if model is None:\n", - " model_gp0 = surrogate_models.ExactGPModel(X_train0, Y_train0, likelihood_gp0) \n", - " else:\n", - " model_gp0 = model\n", - " AcqFunc_0 = ExpectedImprovement(model=model_gp0, best_f=best_observed_value, maximize=model_input.MAXIMIZATION)\n", - " best_observed.append(best_observed_value) # Appending to best_observed list for the given trial\n", - " \n", - " # run N_BATCH rounds of BayesOpt after the initial random batch\n", - " for iteration in range(1, n_batch + 1):\n", - " # Time start of iteration and end\n", - " t0 = time.monotonic()\n", - " if ((iteration-1)%model_input.N_UPDATE==0):\n", - " # fit the models every 10 iterations\n", - " model_gp0, likelihood_gp0 = surrogate_models.train_surrogate_gp0(X_train0, Y_train0)\n", - " \n", - " # optimize and get new observation using acquisition function\n", - " new_x0, new_y0, index, X_test_new0, Y_test_new0 = optimize_acqf_and_get_observation(AcqFunc_0, X_test0, Y_test0)\n", - " \n", - " # Update remaining choices tensor\n", - " X_test0 = X_test_new0\n", - " Y_test0 = Y_test_new0\n", - "\n", - " # Update training points\n", - " X_train0 = torch.cat([X_train0, new_x0])\n", - " Y_train0 = torch.cat([Y_train0[0], new_y0[0]])\n", - " Y_train0 = torch.reshape(Y_train0,(1,Y_train0.shape[0]))\n", - "\n", - " # update progress\n", - " if model_input.MAXIMIZATION:\n", - " best_value_ei0 = Y_train0.max()\n", - " elif not model_input.MAXIMIZATION:\n", - " best_value_ei0 = Y_train0.min()\n", - " best_observed.append(best_value_ei0)\n", - "\n", - " # AcqFunc_0 = UpperConfidenceBound(model_gp0, beta=0.1) \n", - " AcqFunc_0 = ExpectedImprovement(model=model_gp0, best_f=best_value_ei0, maximize=model_input.MAXIMIZATION)\n", - "\n", - " # Time end of iteration\n", - " t1 = time.monotonic()\n", - " \n", - " if model_input.VERBOSE:\n", - " print(\n", - " f\"\\nBatch {iteration:>2}: best_value (GP-0) = \",\n", - " f\"({best_value_ei0:>4.2f}\",\n", - " end=\"\",)\n", - " print(f'Iteration time = {t1-t0:>4.2f}.')\n", - "\n", - " # t1 = time.monotonic()\n", - " # print(f\"time = {t1-t0:>4.2f}.\")\n", - "\n", - " return [best_observed, X_train0, X_test0, Y_train0, Y_test0, model_gp0] " - ] - }, - { - "cell_type": "markdown", - "id": "17fbf21d", - "metadata": {}, - "source": [ - "#### Main Function" - ] - }, - { - "cell_type": "code", - "execution_count": 47, - "id": "bfb03e8a", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - " -------------------- Trial 1 of 1 --------------------\n", - "(1, 3, 1)\n", - "\n", - " -------------------- Cluster 0 of 3 --------------------\n", - "\n", - " -------------------- Cluster 1 of 3 --------------------\n", - "\n", - " -------------------- Cluster 2 of 3 --------------------\n", - "Using cpu deviceUsing cpu device\n", - "\n", - "Using cpu device\n", - "tensor([[-0.6042, -0.0295, -0.7947, 1.3747, 1.5369, -0.0630, -0.1566]])\n", - "\n", - "Batch 1: best_value (GP-0) = (1.92Iteration time = 1.61.\n", - "tensor([[ 1.0603, -0.0367, -1.6548, 0.8714, -0.7460, -0.0717, -0.1818]])\n", - "\n", - "Batch 1: best_value (GP-0) = (2.55Iteration time = 17.92.\n", - "tensor([[-0.6552, -0.0414, -1.2317, 3.5658, -0.7353, -0.0663, -0.1789]])\n", - "\n", - "Batch 1: best_value (GP-0) = (2.71Iteration time = 84.47.\n" - ] - }, - { - "ename": "ValueError", - "evalue": "too many values to unpack (expected 2)", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[47], line 53\u001b[0m\n\u001b[1;32m 50\u001b[0m \u001b[38;5;66;03m# Initially training of GPs done in parallel by using joblib\u001b[39;00m\n\u001b[1;32m 51\u001b[0m results_cluster_0, results_cluster_1, results_cluster_2 \u001b[38;5;241m=\u001b[39m Parallel(n_jobs\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)(delayed(train_gp)(X_train_all_clusters[i], X_test_all_clusters[i], Y_train_all_clusters[i], Y_test_all_clusters[i]) \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(model_input\u001b[38;5;241m.\u001b[39mNUM_CLUSTER))\n\u001b[0;32m---> 53\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i, results_i_cluster \u001b[38;5;129;01min\u001b[39;00m [results_cluster_0, results_cluster_1, results_cluster_2]:\n\u001b[1;32m 54\u001b[0m best_observed_all_clusters[trial\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m][i][\u001b[38;5;241m0\u001b[39m:\u001b[38;5;28mint\u001b[39m(model_input\u001b[38;5;241m.\u001b[39mN_BATCH_PER_TRIAL\u001b[38;5;241m/\u001b[39mmodel_input\u001b[38;5;241m.\u001b[39mN_SEARCH)] \u001b[38;5;241m=\u001b[39m results_i_cluster[\u001b[38;5;241m0\u001b[39m] \n\u001b[1;32m 55\u001b[0m X_train_all_clusters\u001b[38;5;241m.\u001b[39mappend(results_i_cluster[\u001b[38;5;241m1\u001b[39m])\n", - "\u001b[0;31mValueError\u001b[0m: too many values to unpack (expected 2)" - ] - } - ], - "source": [ - "warnings.filterwarnings(\"ignore\", category=BadInitialCandidatesWarning)\n", - "warnings.filterwarnings(\"ignore\", category=RuntimeWarning)\n", - "\n", - "# Create a new directory if it does not exist\n", - "isExist = os.path.exists(model_input.OUT_FOLDER)\n", - "if not isExist:\n", - " os.makedirs(model_input.OUT_FOLDER)\n", - " print(\"The new directory is created!\", model_input.OUT_FOLDER)\n", - "\n", - "shutil.copy2('surrogate_models.py',model_input.OUT_FOLDER)\n", - "\n", - "# Train each GP model sequentially first then apply the epsilon greed algorithm\n", - "for trial in range(1, model_input.N_TRIALS + 1):\n", - " t0 = time.monotonic()\n", - " if model_input.RANDOM_SEED == 'time':\n", - " random_seed = int(t0)\n", - " elif model_input.RANDOM_SEED == 'iteration':\n", - " random_seed = trial\n", - "\n", - " print(f\"\\n -------------------- Trial {trial:>2} of {model_input.N_TRIALS} --------------------\\n\", end=\"\")\n", - "\n", - " best_observed_all_clusters = np.zeros((model_input.N_TRIALS, model_input.NUM_CLUSTER, model_input.N_BATCH_PER_TRIAL))\n", - " print(best_observed_all_clusters.shape)\n", - " X_train_all_clusters = []\n", - " X_test_all_clusters = []\n", - " Y_train_all_clusters = []\n", - " Y_test_all_clusters = []\n", - " model_gps_all_clusters = []\n", - "\n", - " # Creating the initial training and test sets for each cluster\n", - " for cluster_idx in range(model_input.NUM_CLUSTER):\n", - " print(f\"\\n -------------------- Cluster {cluster_idx:>2} of {model_input.NUM_CLUSTER} --------------------\\n\", end=\"\")\n", - " XX_desc = list(sample_dfs[cluster_idx].columns[:-1])\n", - " YY_desc = sample_dfs[cluster_idx].columns[-1]\n", - " (\n", - " X_train_idx,\n", - " X_test_idx,\n", - " Y_train_idx,\n", - " Y_test_idx,\n", - " scalerX, \n", - " scalerY\n", - " ) = create_train_test_data(sample_dfs[cluster_idx][XX_desc].to_numpy(), sample_dfs[cluster_idx][YY_desc].to_numpy(), random_seed)\n", - " dump(scalerX, os.path.join(model_input.OUT_FOLDER, f'scalerX_{cluster_idx}.joblib'))\n", - " dump(scalerY, os.path.join(model_input.OUT_FOLDER, f'scalerY_{cluster_idx}.joblib'))\n", - " X_train_all_clusters.append(X_train_idx)\n", - " X_test_all_clusters.append(X_test_idx)\n", - " Y_train_all_clusters.append(Y_train_idx)\n", - " Y_test_all_clusters.append(Y_test_idx)\n", - " \n", - " # Initially training of GPs done in parallel by using joblib\n", - " results_cluster_0, results_cluster_1, results_cluster_2 = Parallel(n_jobs=-1)(delayed(train_gp)(X_train_all_clusters[i], X_test_all_clusters[i], Y_train_all_clusters[i], Y_test_all_clusters[i]) for i in range(model_input.NUM_CLUSTER))\n", - "\n", - " for i, results_i_cluster in [results_cluster_0, results_cluster_1, results_cluster_2]:\n", - " best_observed_all_clusters[trial-1][i][0:int(model_input.N_BATCH_PER_TRIAL/model_input.N_SEARCH)] = results_i_cluster[0] \n", - " X_train_all_clusters.append(results_i_cluster[1])\n", - " X_test_all_clusters.append(results_i_cluster[2])\n", - " Y_train_all_clusters.append(results_i_cluster[3])\n", - " Y_test_all_clusters.append(results_i_cluster[4])\n", - " model_gps_all_clusters.append(results_i_cluster[5])\n", - "\n", - " # print(f'\\n')\n", - " # print(f'Starting the epsilon greedy search')\n", - " # print(f'\\n') \n", - "\n", - " # # Now apply the epsilon greedy algorithm and choose which GP to train next\n", - " # for i in range(model_input.N_SEARCH):\n", - " # random_number = np.random.rand()\n", - " # epsilon = model_input.EPSILON\n", - " # # Explore using the Epsilon Greedy Exploration Strategy\n", - " # if random_number <= epsilon:\n", - " # # Selecting a number between 1,2 and 3\n", - " # cluster_idx = np.random.choice(model_input.NUM_CLUSTER)\n", - " # else:\n", - " # # Exploit best known action\n", - " # cluster_idx = np.argmax(best_observed_all_clusters[i][-1] for i in range(model_input.NUM_CLUSTER))\n", - " # print(f'Iteration {i} : Cluster {cluster_idx} is selected for training')\n", - " # results = train_gp(X_train_all_clusters[cluster_idx], \\\n", - " # X_test_all_clusters[cluster_idx], \\\n", - " # Y_train_all_clusters[cluster_idx], \\\n", - " # Y_test_all_clusters[cluster_idx], \\\n", - " # model=model_gps_all_clusters[cluster_idx])\n", - " \n", - " # # Add the best values\n", - " # best_observed_all_clusters[cluster_idx].extend(best_observed_idx)\n", - " # X_train_all_clusters[cluster_idx] = results[1]\n", - " # X_test_all_clusters[cluster_idx] = results[2]\n", - " # Y_train_all_clusters[cluster_idx] = results[3]\n", - " # Y_test_all_clusters[cluster_idx] = results[4]\n", - " # model_gps_all_clusters[cluster_idx] = results[5]\n" - ] - }, - { - "cell_type": "code", - "execution_count": 48, - "id": "04617178", - "metadata": {}, - "outputs": [ - { - "ename": "ValueError", - "evalue": "could not broadcast input array from shape (2,) into shape (1,)", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[48], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i, results_i_cluster \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m([results_cluster_0, results_cluster_1, results_cluster_2]):\n\u001b[0;32m----> 2\u001b[0m \u001b[43mbest_observed_all_clusters\u001b[49m\u001b[43m[\u001b[49m\u001b[43mtrial\u001b[49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m[\u001b[49m\u001b[43mi\u001b[49m\u001b[43m]\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m:\u001b[49m\u001b[38;5;28;43mint\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mmodel_input\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mN_BATCH_PER_TRIAL\u001b[49m\u001b[38;5;241;43m/\u001b[39;49m\u001b[43mmodel_input\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mN_SEARCH\u001b[49m\u001b[43m)\u001b[49m\u001b[43m]\u001b[49m \u001b[38;5;241m=\u001b[39m results_i_cluster[\u001b[38;5;241m0\u001b[39m] \n\u001b[1;32m 3\u001b[0m X_train_all_clusters\u001b[38;5;241m.\u001b[39mappend(results_i_cluster[\u001b[38;5;241m1\u001b[39m])\n\u001b[1;32m 4\u001b[0m X_test_all_clusters\u001b[38;5;241m.\u001b[39mappend(results_i_cluster[\u001b[38;5;241m2\u001b[39m])\n", - "\u001b[0;31mValueError\u001b[0m: could not broadcast input array from shape (2,) into shape (1,)" - ] - } - ], - "source": [ - "for i, results_i_cluster in enumerate([results_cluster_0, results_cluster_1, results_cluster_2]):\n", - " best_observed_all_clusters[trial-1][i][0:int(model_input.N_BATCH_PER_TRIAL/model_input.N_SEARCH)] = results_i_cluster[0] \n", - " X_train_all_clusters.append(results_i_cluster[1])\n", - " X_test_all_clusters.append(results_i_cluster[2])\n", - " Y_train_all_clusters.append(results_i_cluster[3])\n", - " Y_test_all_clusters.append(results_i_cluster[4])\n", - " model_gps_all_clusters.append(results_i_cluster[5])" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "id": "ad1511ca", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[[[0. 0. 0. 0.]\n", - " [1. 2. 0. 0.]\n", - " [0. 0. 0. 0.]]\n", - "\n", - " [[0. 0. 0. 0.]\n", - " [0. 0. 0. 0.]\n", - " [0. 0. 0. 0.]]\n", - "\n", - " [[0. 0. 0. 0.]\n", - " [0. 0. 0. 0.]\n", - " [0. 0. 0. 0.]]]\n" - ] - } - ], - "source": [ - "l = np.zeros((3,3,4))\n", - "\n", - "l[0][1][0:2] = [1,2]\n", - "\n", - "print(l)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4a686cad", - "metadata": {}, - "outputs": [], - "source": [ - "# Plot the best value obtained vs number of iterations\n", - "plt.figure(figsize=(10, 6))\n", - "for i in range(model_input.NUM_CLUSTER):\n", - " plt.plot(best_observed_all_clusters[i], label=f'Cluster {i}')\n", - "plt.xlabel('Number of Iterations')\n", - "plt.ylabel('Best Value')\n", - "plt.legend()\n", - "plt.grid()\n", - "plt.show()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.2" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/src/code_inputs.py b/src/code_inputs.py index 55d70ff..a21fee0 100644 --- a/src/code_inputs.py +++ b/src/code_inputs.py @@ -9,9 +9,12 @@ TEST_SIZE = 0.95 # Output folder name OUT_FOLDER = RUN_FOLDER+'/bo_output/' -OUT_FOLDER = OUT_FOLDER+'test_with_cluster/' +# OUT_FOLDER = OUT_FOLDER+'results_with_three_clusters/' +OUT_FOLDER = OUT_FOLDER+'results_with_three_clusters/' VERBOSE = True DEEP_VERBOSE = False +# Initial search before epsilon greedy search train GPs in parallel ? +TRAIN_PARALLEL = False # Reading and data processing inputs ADD_TARGET_NOISE = False @@ -28,13 +31,15 @@ MAXIMIZATION = True NEW_VALUES_PREDICT_FROM_MODEL = False # Number of trials to perform BO from scratch -N_TRIALS = 1 +N_TRIALS = 10 # Number of experiments performed in each trial -N_BATCH_PER_TRIAL = 1 +N_BATCH_PER_TRIAL = 300 # Number of iterations after which the model is updated -N_UPDATE = 1 -# Number of times to perform epsilon greedy search. N_BATCH_PER_TRIAL/N_SEARCH must be an integer -N_SEARCH = 1 +N_UPDATE = 10 +# Number of times to perform search. +# If N_SEARCH is set to 1 then only the initial search is performed +# If N_SEARCH is set to > 1 then only the epsilon greedy search is performed +N_SEARCH = 6 # Epsilon greedy search parameter EPSILON = 0.1 # Select the GP surrogate to train