FBBO - Noisy function setup.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import tqdm.notebook\n",
    "import torch\n",
    "import botorch\n",
    "import gpytorch\n",
    "import numpy as np\n",
    "import pyDOE2\n",
    "\n",
    "import matplotlib\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "import fbbo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Estimate standard deviation of all test function outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "synthetic_problems = [\n",
    "    ('Branin', {}),\n",
    "    ('Eggholder', {}),\n",
    "    ('GoldsteinPrice', {}),\n",
    "    ('SixHumpCamel',{}),\n",
    "    ('Hartmann3', {}),\n",
    "    ('Ackley', {'d': 5}),\n",
    "    ('Hartmann6', {}),\n",
    "    ('Michalewicz', {'d': 10}),\n",
    "    ('Rosenbrock', {'d': 10}),\n",
    "    ('StyblinskiTang', {'d': 10}),\n",
    "    (\"Michalewicz\", {\"d\": 5}),\n",
    "    (\"StyblinskiTang\", {\"d\": 5}),\n",
    "    (\"Rosenbrock\", {\"d\": 7}),\n",
    "    (\"StyblinskiTang\", {\"d\": 7}),\n",
    "    (\"Ackley\", {\"d\": 10}),\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set the percentage of the range to use as the standard deviation of the noise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "STDEV_PCT = 0.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# storage for each problem and its noise stddev\n",
    "problem_noise_dict = {}\n",
    "# pnd = {\n",
    "#     'd': {\n",
    "#         2: {\n",
    "#             'run_no': stdv\n",
    "#         }\n",
    "#     }\n",
    "# }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e08169dc197641caab3ec21e7cc992f9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/9 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# pregenerate 1 million LHS samples\n",
    "lhs_samples = {}\n",
    "for d in tqdm.notebook.trange(2, 11):\n",
    "    lhs_samples[d] = pyDOE2.lhs(d, samples=100000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bc274c0aaebd4a9d8c4a0ba52cc4daf1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/15 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Branin {} 303.62421815471447\n",
      "Eggholder {} 2002.5421167854306\n",
      "GoldsteinPrice {} 1011973.8042217783\n",
      "SixHumpCamel {} 162.62380351990475\n",
      "Hartmann3 {} 3.8630560881723675\n",
      "Ackley {'d': 5} 22.22939890683161\n",
      "Hartmann6 {} 3.3223674027850363\n",
      "Michalewicz {'d': 10} 9.660149998169544\n",
      "Rosenbrock {'d': 10} 4355212.627842652\n",
      "StyblinskiTang {'d': 10} 900.8315043115236\n",
      "Michalewicz {'d': 5} 4.687658\n",
      "StyblinskiTang {'d': 5} 587.4235626225984\n",
      "Rosenbrock {'d': 7} 3762662.0778023107\n",
      "StyblinskiTang {'d': 7} 796.4516432191913\n",
      "Ackley {'d': 10} 22.116357576831536\n"
     ]
    }
   ],
   "source": [
    "problems = synthetic_problems\n",
    "\n",
    "for problem_name, problem_params in tqdm.notebook.tqdm(problems):\n",
    "    if problem_name not in problem_noise_dict:\n",
    "        problem_noise_dict[problem_name] = {}\n",
    "\n",
    "    if \"bbob\" in problem_name:\n",
    "        max_run_no = 15  # inclusve\n",
    "    else:\n",
    "        max_run_no = 1\n",
    "    for i in range(1, max_run_no + 1):\n",
    "        if max_run_no > 1:\n",
    "            problem_params[\"run_no\"] = i\n",
    "\n",
    "        f = fbbo.util.test_func_getter(problem_name, problem_params)\n",
    "\n",
    "        # extact info about the function\n",
    "        lb, ub, d, fmin = f.lb, f.ub, f.dim, f.yopt.item()\n",
    "\n",
    "        # rescale lhs samples to domain\n",
    "        X = lhs_samples[d] * (ub - lb) + lb\n",
    "\n",
    "        # evaluate f(X)\n",
    "        try:\n",
    "            fX = f(X)\n",
    "\n",
    "        except ValueError:\n",
    "            fX = [f(x) for x in X]\n",
    "\n",
    "        dX = np.max(fX) - fmin\n",
    "\n",
    "        print(problem_name, problem_params, dX)\n",
    "\n",
    "        if d not in problem_noise_dict[problem_name]:\n",
    "            problem_noise_dict[problem_name][d] = {}\n",
    "\n",
    "        if \"run_no\" in problem_params:\n",
    "            problem_noise_dict[problem_name][d][problem_params[\"run_no\"]] = {\n",
    "                \"stdev\": STDEV_PCT * dX\n",
    "            }\n",
    "        else:\n",
    "            problem_noise_dict[problem_name][d][\"stdev\"] = STDEV_PCT * dX"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Branin': {2: {'stdev': 60.7248436309429}},\n",
       " 'Eggholder': {2: {'stdev': 400.50842335708614}},\n",
       " 'GoldsteinPrice': {2: {'stdev': 202394.76084435568}},\n",
       " 'SixHumpCamel': {2: {'stdev': 32.52476070398095}},\n",
       " 'Hartmann3': {3: {'stdev': 0.7726112176344735}},\n",
       " 'Ackley': {5: {'stdev': 4.445879781366322}, 10: {'stdev': 4.423271515366308}},\n",
       " 'Hartmann6': {6: {'stdev': 0.6644734805570073}},\n",
       " 'Michalewicz': {10: {'stdev': 1.932029999633909}, 5: {'stdev': 0.9375316}},\n",
       " 'Rosenbrock': {10: {'stdev': 871042.5255685304},\n",
       "  7: {'stdev': 752532.4155604622}},\n",
       " 'StyblinskiTang': {10: {'stdev': 180.16630086230475},\n",
       "  5: {'stdev': 117.4847125245197},\n",
       "  7: {'stdev': 159.29032864383828}}}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "problem_noise_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savez(f'problem_noise_dict_{STDEV_PCT:g}.npz', pnd=problem_noise_dict)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}