diff --git a/.gitignore b/.gitignore index c9a692f..e1da78f 100755 --- a/.gitignore +++ b/.gitignore @@ -96,6 +96,7 @@ docs/build/ docs/modules.rst docs/atm.rst docs/atm.*.rst +docs/api # PyBuilder target/ @@ -120,6 +121,9 @@ celerybeat-schedule venv/ ENV/ +# vim temporary files +*.swp + # Spyder project settings .spyderproject .spyproject diff --git a/AUTHORS.rst b/AUTHORS.rst index 5d9b997..4efc1b2 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -9,8 +9,9 @@ Contributors * Thomas Swearingen * Kalyan Veeramachaneni * Laura Gustafson -* Carles Sala +* Carles Sala * Micah Smith +* Plamen Valentinov * Kiran Karra * swearin3 * Max Kanter diff --git a/HISTORY.md b/HISTORY.md index b159b1e..7e26a33 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,14 @@ # History +## 0.1.1 (2019-04-02) + +First Release on PyPi. + +### New Features + +* Upgrade to latest BTB. +* New Command Line Interface. + ## 0.1.0 (2018-05-04) -* First release on PyPI. +* First Release. diff --git a/Makefile b/Makefile index 73a420a..94a2957 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,3 @@ -.PHONY: clean clean-test clean-pyc clean-build clean-docs docs help .DEFAULT_GOAL := help define BROWSER_PYSCRIPT @@ -26,11 +25,13 @@ export PRINT_HELP_PYSCRIPT BROWSER := python -c "$$BROWSER_PYSCRIPT" +.PHONY: help help: @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) -clean: clean-build clean-pyc clean-coverage clean-test clean-docs clean-data ## remove all build, test, coverage, docs and Python artifacts +# CLEAN TARGETS +.PHONY: clean-build clean-build: ## remove build artifacts rm -fr build/ rm -fr dist/ @@ -38,79 +39,173 @@ clean-build: ## remove build artifacts find . -name '*.egg-info' -exec rm -fr {} + find . -name '*.egg' -exec rm -f {} + +.PHONY: clean-pyc clean-pyc: ## remove Python file artifacts find . -name '*.pyc' -exec rm -f {} + find . -name '*.pyo' -exec rm -f {} + find . -name '*~' -exec rm -f {} + find . -name '__pycache__' -exec rm -fr {} + +.PHONY: clean-docs +clean-docs: ## remove previously built docs + rm -rf docs/build + rm -f docs/atm.rst + rm -f docs/atm.*.rst + rm -f docs/modules.rst + $(MAKE) -C docs clean + +.PHONY: clean-coverage clean-coverage: ## remove coverage artifacts rm -f .coverage rm -f .coverage.* rm -fr htmlcov/ -clean-test: ## remove test and coverage artifacts +.PHONY: clean-test +clean-test: ## remove test artifacts rm -fr .tox/ rm -fr .pytest_cache -clean-data: ## remove generated data - rm -fr *.db - rm -fr models/ - rm -fr metrics/ - rm -fr logs/ +.PHONY: clean +clean: clean-build clean-pyc clean-test clean-coverage clean-docs ## remove all build, test, coverage, docs and Python artifacts -clean-docs: ## remove previously built docs - rm -rf docs/build - rm -f docs/atm.rst - rm -f docs/atm.*.rst - rm -f docs/modules.rst - $(MAKE) -C docs clean +# INSTALL TARGETS + +.PHONY: install +install: clean-build clean-pyc ## install the package to the active Python's site-packages + pip install . + +.PHONY: install-test +install-test: clean-build clean-pyc ## install the package and test dependencies + pip install .[test] + +.PHONY: install-develop +install-develop: clean-build clean-pyc ## install the package in editable mode and dependencies for development + pip install -e .[dev] + + +# LINT TARGETS + +.PHONY: lint lint: ## check style with flake8 and isort - flake8 atm # tests - isort -c --recursive atm # tests + flake8 atm tests + isort -c --recursive atm tests -fixlint: ## fix lint issues using autoflake, autopep8, and isort +.PHONY: fix-lint +fix-lint: ## fix lint issues using autoflake, autopep8, and isort find atm -name '*.py' | xargs autoflake --in-place --remove-all-unused-imports --remove-unused-variables autopep8 --in-place --recursive --aggressive atm isort --apply --atomic --recursive atm - # find tests -name '*.py' | xargs autoflake --in-place --remove-all-unused-imports --remove-unused-variables - # autopep8 --in-place --recursive --aggressive tests - # isort --apply --atomic --recursive tests + find tests -name '*.py' | xargs autoflake --in-place --remove-all-unused-imports --remove-unused-variables + autopep8 --in-place --recursive --aggressive tests + isort --apply --atomic --recursive tests + +# TEST TARGETS + +.PHONY: test test: ## run tests quickly with the default Python - pytest + python -m pytest tests +.PHONY: test-all test-all: ## run tests on every Python version with tox tox -coverage: clean-coverage ## check code coverage quickly with the default Python +.PHONY: coverage +coverage: ## check code coverage quickly with the default Python coverage run --source atm -m pytest coverage report -m coverage html $(BROWSER) htmlcov/index.html + +# DOCS TARGETS + +.PHONY: docs docs: clean-docs ## generate Sphinx HTML documentation, including API docs - sphinx-apidoc -o docs/ atm + sphinx-apidoc --module-first --separate -o docs/api/ atm $(MAKE) -C docs html -viewdocs: docs ## view docs in browser - $(BROWSER) docs/build/index.html +.PHONY: view-docs +view-docs: docs ## view docs in browser + $(BROWSER) docs/_build/html/index.html + +.PHONY: serve-docs +serve-docs: view-docs ## compile the docs watching for changes + watchmedo shell-command -W -R -D -p '*.rst;*.md' -c '$(MAKE) -C docs html' . -servedocs: docs ## compile the docs watching for changes - watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . +# RELEASE TARGETS + +.PHONY: dist dist: clean ## builds source and wheel package python setup.py sdist python setup.py bdist_wheel ls -l dist -release: dist ## package and upload a release - twine upload dist/* - -test-release: dist ## package and upload a release on TestPyPI +.PHONY: test-publish +test-publish: dist ## package and upload a release on TestPyPI twine upload --repository-url https://test.pypi.org/legacy/ dist/* -install: clean ## install the package to the active Python's site-packages - python setup.py install +.PHONY: publish +publish: dist ## package and upload a release + twine upload dist/* + +.PHONY: bumpversion-release +bumpversion-release: ## Merge master to stable and bumpversion release + git checkout stable + git merge --no-ff master -m"make release-tag: Merge branch 'master' into stable" + bumpversion release + git push --tags origin stable + +.PHONY: test-bumpversion-release +test-bumpversion-release: ## Merge master to stable and bumpversion release + git checkout stable + git merge --no-ff master -m"make release-tag: Merge branch 'master' into stable" + bumpversion release + +.PHONY: bumpversion-patch +bumpversion-patch: ## Merge stable to master and bumpversion patch + git checkout master + git merge stable + bumpversion --no-tag patch + git push + +.PHONY: test-bumpversion-patch +test-bumpversion-patch: ## Merge stable to master and bumpversion patch + git checkout master + git merge stable + bumpversion --no-tag patch + +.PHONY: bumpversion-minor +bumpversion-minor: ## Bump the version the next minor skipping the release + bumpversion --no-tag minor + +.PHONY: bumpversion-major +bumpversion-major: ## Bump the version the next major skipping the release + bumpversion --no-tag major + +CURRENT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null) +CHANGELOG_LINES := $(shell git diff HEAD..stable HISTORY.md 2>/dev/null | wc -l) + +.PHONY: check-release +check-release: ## Check if the release can be made +ifneq ($(CURRENT_BRANCH),master) + $(error Please make the release from master branch\n) +endif +ifeq ($(CHANGELOG_LINES),0) + $(error Please insert the release notes in HISTORY.md before releasing) +endif + +.PHONY: release +release: check-release bumpversion-release publish bumpversion-patch + +.PHONY: release-minor +release-minor: check-release bumpversion-minor release + +.PHONY: release-major +release-major: check-release bumpversion-major release + +.PHONY: test-release +test-release: check-release test-bumpversion-release test-publish test-bumpversion-patch diff --git a/README.md b/README.md index 06ea65a..18464c1 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,10 @@ +

+“ATM” +An open source project from Data to AI Lab at MIT. +

+ + + [![CircleCI][circleci-img]][circleci-url] [![Coverage status][codecov-img]][codecov-url] [![Documentation][rtd-img]][rtd-url] @@ -67,14 +74,8 @@ Unix-based systems. ATM is compatible with and has been tested on Python 2.7, 3.5, and 3.6. -1. **Clone the project** - - ``` - git clone https://github.com/hdi-project/ATM.git /path/to/atm - cd /path/to/atm - ``` -2. **Install a database** +1. **Install a database** You will need to install the libmysqlclient-dev package (for sqlalchemy) @@ -96,19 +97,26 @@ ATM is compatible with and has been tested on Python 2.7, 3.5, and 3.6. sudo apt install mysql-server mysql-client ``` -3. **Install python dependencies**. +2. **Install ATM** - This will also install [btb](https://github.com/hdi-project/btb), the core AutoML library in - development under the HDI project, as an egg which will track changes to the git repository. + To get started with **ATM**, we recommend using `pip`: - Here, usage of `virtualenv` is shown, but you can substitute `conda` or your preferred - environment manager as well. + ```bash + pip install atm + ``` + + Alternatively, you can clone the repository and install it from source by running + `make install`: + + ```bash + git clone git@github.com:HDI-Project/ATM.git + cd ATM + make install + ``` + + For development, you can use the `make install-develop` command instead in order to install all + the required dependencies for testing and linting. - ``` - virtualenv venv - . venv/bin/activate - python setup.py install - ``` ## Quick Usage @@ -140,12 +148,12 @@ The data has 15 features and the last column is the `class` label. 1. **Create a datarun** ``` - python scripts/enter_data.py + atm enter_data ``` This command will create a `datarun`. In ATM, a "datarun" is a single logical machine learning task. If you run the above command without any arguments, it will use the default settings - found in `atm/config.py` to create a new SQLite3 database at `./atm.db`, create a new + found in the code to create a new SQLite3 database at `./atm.db`, create a new `dataset` instance which refers to the data above, and create a `datarun` instance which points to that dataset. More about what is stored in this database and what is it used for can be found [here](https://cyphe.rs/static/atm.pdf). @@ -169,7 +177,7 @@ The data has 15 features and the last column is the `class` label. 2. **Start a worker** ``` - python scripts/worker.py + atm worker ``` This will start a process that builds classifiers, tests them, and saves them to the @@ -204,7 +212,7 @@ all workers will exit gracefully. ## Customizing ATM's configuration and using your own data -ATM's default configuration is fully controlled by `atm/config.py`. Our documentation will +ATM's default configuration is fully controlled by the intern code. Our documentation will cover the configuration in more detail, but this section provides a brief overview of how to specify the most important values. @@ -218,27 +226,50 @@ to the example shown above. The format is: * The first row is the header row, which contains names for each column of data * A single column (the *target* or *label*) is named `class` -Next, you'll need to use `enter_data.py` to create a `dataset` and `datarun` for your task. +Next, you'll need to use `atm enter_data` to create a `dataset` and `datarun` for your task. -The script will look for values for each configuration variable in the following places, in order: +The command line will look for values for each configuration variable in the following places, +in order: 1. Command line arguments 2. Configuration files -3. Defaults specified in `atm/config.py` +3. Defaults specified inside the code. That means there are two ways to pass configuration to the command. -1. **Using YAML configuration files** +1. **Using command line arguments** + + You can specify each argument individually on the command line. The names of the + variables are the same as those in the YAML files. SQL configuration variables must be + prepended by `sql-`, and AWS config variables must be prepended by `aws-`. + + Using command line arguments is convenient for quick experiments, or for cases where you + need to change just a couple of values from the default configuration. For example: + + ``` + atm enter_data --train-path ./data/my-custom-data.csv --selector bestkvel + ``` + + You can also use a mixture of config files and command line arguments; any command line + arguments you specify will override the values found in config files. + +2. **Using YAML configuration files** - Saving configuration as YAML files is an easy way to save complicated setups or share - them with team members. + You can also save the configuration as YAML files is an easy way to save complicated setups + or share them with team members. - You should start with the templates provided in `atm/config/templates` and modify them - to suit your own needs. + You should start with the templates provided by the `atm make_config` command: ``` - mkdir config - cp atm/config/templates/*.yaml config/ + atm make_config + ``` + + This will generate a folder called `config/templates` in your current working directory which + will contain 5 files, which you will need to copy over to the `config` folder and edit according + to your needs: + + ``` + cp config/templates/*.yaml config/ vim config/*.yaml ``` @@ -263,43 +294,22 @@ That means there are two ways to pass configuration to the command. `aws.yaml` should contain the settings for running ATM in the cloud. This is not necessary for local operation. - Once your YAML files have been updated, run the datarun creation script and pass it the paths + Once your YAML files have been updated, run the datarun creation command and pass it the paths to your new config files: ``` - python scripts/enter_data.py --sql-config config/sql.yaml \ - --aws-config config/aws.yaml \ - --run-config config/run.yaml + atm enter_data --sql-config config/sql.yaml \ + --aws-config config/aws.yaml \ + --run-config config/run.yaml ``` -2. **Using command line arguments** - - You can also specify each argument individually on the command line. The names of the - variables are the same as those in the YAML files. SQL configuration variables must be - prepended by `sql-`, and AWS config variables must be prepended by `aws-`. - - Using command line arguments is convenient for quick experiments, or for cases where you - need to change just a couple of values from the default configuration. For example: - - ``` - python scripts/enter_data.py --train-path ./data/my-custom-data.csv --selector bestkvel - ``` - - You can also use a mixture of config files and command line arguments; any command line - arguments you specify will override the values found in config files. - -Once you've created your custom datarun, start a worker, specifying your config files and the -datarun(s) you'd like to compute on. - -``` -python scripts/worker.py --sql-config config/sql.yaml \ - --aws-config config/aws.yaml \ - --dataruns 1 -``` - It's important that the SQL configuration used by the worker matches the configuration you -passed to `enter_data.py` -- otherwise, the worker will be looking in the wrong ModelHub +passed to `enter_data` -- otherwise, the worker will be looking in the wrong ModelHub database for its datarun! + ``` + atm worker --sql-config config/sql.yaml \ + --aws-config config/aws.yaml \ + ``` diff --git a/atm/__init__.py b/atm/__init__.py index 8fc841b..ed4f58a 100644 --- a/atm/__init__.py +++ b/atm/__init__.py @@ -1,8 +1,8 @@ """Auto Tune Models A multi-user, multi-data AutoML framework. """ -from __future__ import absolute_import -from __future__ import unicode_literals +from __future__ import absolute_import, unicode_literals + import logging import os @@ -10,9 +10,13 @@ # reference files relative to there. PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__)) +__author__ = """MIT Data To AI Lab""" +__email__ = 'dailabmit@gmail.com' +__version__ = '0.1.1-dev' + # this defines which modules will be imported by "from atm import *" -__all__ = ['config', 'constants', 'database', 'enter_data', 'method', 'metrics', - 'model', 'utilities', 'worker'] +__all__ = ['config', 'classifier', 'constants', 'database', 'enter_data', + 'method', 'metrics', 'models', 'utilities', 'worker'] # by default, nothing should be logged logger = logging.getLogger('atm') diff --git a/atm/model.py b/atm/classifier.py similarity index 96% rename from atm/model.py rename to atm/classifier.py index 92892a2..cc25af3 100644 --- a/atm/model.py +++ b/atm/classifier.py @@ -16,9 +16,8 @@ import pandas as pd from past.utils import old_div from sklearn import decomposition -from sklearn.gaussian_process.kernels import (RBF, ConstantKernel, - ExpSineSquared, Matern, - RationalQuadratic) +from sklearn.gaussian_process.kernels import ( + RBF, ConstantKernel, ExpSineSquared, Matern, RationalQuadratic) from sklearn.model_selection import train_test_split from sklearn.pipeline import Pipeline from sklearn.preprocessing import MinMaxScaler, StandardScaler @@ -146,8 +145,9 @@ def cross_validate(self, X, y): self.cv_judgment_metric = np.mean(df[self.judgment_metric]) self.cv_judgment_metric_stdev = np.std(df[self.judgment_metric]) - self.mu_sigma_judgment_metric = (self.cv_judgment_metric - - 2 * self.cv_judgment_metric_stdev) + cv_stdev = (2 * self.cv_judgment_metric_stdev) + self.mu_sigma_judgment_metric = self.cv_judgment_metric - cv_stdev + return cv_scores def test_final_model(self, X, y): @@ -247,7 +247,7 @@ def special_conversions(self, params): """ # create list parameters lists = defaultdict(list) - element_regex = re.compile('(.*)\[(\d)\]') + element_regex = re.compile(r'(.*)\[(\d)\]') for name, param in list(params.items()): # look for variables of the form "param_name[1]" match = element_regex.match(name) diff --git a/atm/cli.py b/atm/cli.py new file mode 100644 index 0000000..76e739b --- /dev/null +++ b/atm/cli.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- + +import argparse +import glob +import os +import shutil + +from atm.config import ( + add_arguments_aws_s3, add_arguments_datarun, add_arguments_logging, add_arguments_sql) +from atm.models import ATM + + +def _end_to_end_test(args): + """End to end test""" + + +def _work(args): + atm = ATM(**vars(args)) + atm.work( + datarun_ids=args.dataruns, + choose_randomly=args.choose_randomly, + save_files=args.save_files, + cloud_mode=args.cloud_mode, + total_time=args.time, + wait=False + ) + + +def _enter_data(args): + atm = ATM(**vars(args)) + atm.enter_data() + + +def _make_config(args): + config_templates = os.path.join('config', 'templates') + config_dir = os.path.join(os.path.dirname(__file__), config_templates) + target_dir = os.path.join(os.getcwd(), config_templates) + if not os.path.exists(target_dir): + os.makedirs(target_dir) + + for template in glob.glob(os.path.join(config_dir, '*.yaml')): + target_file = os.path.join(target_dir, os.path.basename(template)) + print('Generating file {}'.format(target_file)) + shutil.copy(template, target_file) + + +# load other functions from config.py +def _add_common_arguments(parser): + add_arguments_sql(parser) + add_arguments_aws_s3(parser) + add_arguments_logging(parser) + + +def _get_parser(): + parent = argparse.ArgumentParser(add_help=False) + + parser = argparse.ArgumentParser(description='ATM Command Line Interface') + + subparsers = parser.add_subparsers(title='action', help='Action to perform') + parser.set_defaults(action=None) + + # Enter Data Parser + enter_data = subparsers.add_parser('enter_data', parents=[parent]) + enter_data.set_defaults(action=_enter_data) + _add_common_arguments(enter_data) + add_arguments_datarun(enter_data) + enter_data.add_argument('--run-per-partition', default=False, action='store_true', + help='if set, generate a new datarun for each hyperpartition') + + # Worker + worker = subparsers.add_parser('worker', parents=[parent]) + worker.set_defaults(action=_work) + _add_common_arguments(worker) + worker.add_argument('--cloud-mode', action='store_true', default=False, + help='Whether to run this worker in cloud mode') + + worker.add_argument('--dataruns', help='Only train on dataruns with these ids', nargs='+') + worker.add_argument('--time', help='Number of seconds to run worker', type=int) + worker.add_argument('--choose-randomly', action='store_true', + help='Choose dataruns to work on randomly (default = sequential order)') + + worker.add_argument('--no-save', dest='save_files', default=True, + action='store_const', const=False, + help="don't save models and metrics at all") + + # Make Config + make_config = subparsers.add_parser('make_config', parents=[parent]) + make_config.set_defaults(action=_make_config) + + # End to end test + end_to_end = subparsers.add_parser('end_to_end', parents=[parent]) + end_to_end.set_defaults(action=_end_to_end_test) + end_to_end.add_argument('--processes', help='number of processes to run concurrently', + type=int, default=4) + + end_to_end.add_argument('--total-time', help='Total time for each worker to work in seconds.', + type=int, default=None) + + return parser + + +def main(): + parser = _get_parser() + args = parser.parse_args() + + if not args.action: + parser.print_help() + parser.exit() + + args.action(args) diff --git a/atm/compat.py b/atm/compat.py new file mode 100644 index 0000000..a29d1a4 --- /dev/null +++ b/atm/compat.py @@ -0,0 +1,10 @@ +import inspect + +from six import PY2 + + +def getargs(func): + if PY2: + return inspect.getargspec(func).args + else: + return inspect.getfullargspec(func).args diff --git a/atm/config.py b/atm/config.py index ce99cdc..7a39961 100644 --- a/atm/config.py +++ b/atm/config.py @@ -9,10 +9,10 @@ from builtins import map, object, str import yaml -from atm.constants import (BUDGET_TYPES, CUSTOM_CLASS_REGEX, DATA_TEST_PATH, - JSON_REGEX, LOG_LEVELS, METHODS, METRICS, - SCORE_TARGETS, SELECTORS, SQL_DIALECTS, TIME_FMT, - TUNERS) + +from atm.constants import ( + BUDGET_TYPES, CUSTOM_CLASS_REGEX, DATA_TEST_PATH, JSON_REGEX, LOG_LEVELS, METHODS, METRICS, + SCORE_TARGETS, SELECTORS, SQL_DIALECTS, TIME_FMT, TUNERS) from atm.utilities import ensure_directory @@ -386,7 +386,7 @@ def add_arguments_datarun(parser): help='Method or list of methods to use for ' 'classification. Each method can either be one of the ' 'pre-defined method codes listed below or a path to a ' - 'JSON file defining a custom method.' + + 'JSON file defining a custom method.' '\n\nOptions: [%s]' % ', '.join(str(s) for s in METHODS)) parser.add_argument('--priority', type=int, help='Priority of the datarun (higher = more important') @@ -396,8 +396,8 @@ def add_arguments_datarun(parser): help='Value of the budget, either in classifiers or minutes') parser.add_argument('--deadline', help='Deadline for datarun completion. If provided, this ' - 'overrides the configured walltime budget.\nFormat: ' + - TIME_FMT.replace('%', '%%')) + 'overrides the configured walltime budget.\nFormat: {}'.format( + TIME_FMT.replace('%', '%%'))) # Which field to use to judge performance, for the sake of AutoML # options: diff --git a/atm/constants.py b/atm/constants.py index a3f2df9..81cde22 100644 --- a/atm/constants.py +++ b/atm/constants.py @@ -4,9 +4,9 @@ import os from builtins import object -from btb.selection import (UCB1, BestKReward, BestKVelocity, - HierarchicalByAlgorithm, PureBestKVelocity, - RecentKReward, RecentKVelocity) +from btb.selection import ( + UCB1, BestKReward, BestKVelocity, HierarchicalByAlgorithm, PureBestKVelocity, RecentKReward, + RecentKVelocity) from btb.selection import Uniform as UniformSelector from btb.tuning import GP, GPEi, GPEiVelocity from btb.tuning import Uniform as UniformTuner @@ -35,8 +35,8 @@ DATA_DL_PATH = os.path.join(PROJECT_ROOT, 'data/downloads') METHOD_PATH = os.path.join(PROJECT_ROOT, 'methods') -CUSTOM_CLASS_REGEX = '(.*\.py):(\w+)$' -JSON_REGEX = '(.*\.json)$' +CUSTOM_CLASS_REGEX = r'(.*\.py):(\w+)$' +JSON_REGEX = r'(.*\.json)$' N_FOLDS_DEFAULT = 10 diff --git a/atm/database.py b/atm/database.py index 38fa938..745f24a 100644 --- a/atm/database.py +++ b/atm/database.py @@ -8,17 +8,17 @@ from operator import attrgetter import pandas as pd -from sqlalchemy import (Column, DateTime, Enum, ForeignKey, Integer, MetaData, - Numeric, String, Text, and_, create_engine, func, - inspect) +from sqlalchemy import ( + Column, DateTime, Enum, ForeignKey, Integer, MetaData, Numeric, String, Text, and_, + create_engine, func, inspect) from sqlalchemy.engine.url import URL from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import relationship, sessionmaker from sqlalchemy.orm.properties import ColumnProperty -from atm.constants import (BUDGET_TYPES, CLASSIFIER_STATUS, DATARUN_STATUS, - METRICS, PARTITION_STATUS, SCORE_TARGETS, - ClassifierStatus, PartitionStatus, RunStatus) +from atm.constants import ( + BUDGET_TYPES, CLASSIFIER_STATUS, DATARUN_STATUS, METRICS, PARTITION_STATUS, SCORE_TARGETS, + ClassifierStatus, PartitionStatus, RunStatus) from atm.utilities import base_64_to_object, object_to_base_64 # The maximum number of errors allowed in a single hyperpartition. If more than @@ -277,8 +277,7 @@ def mu_sigma_judgment_metric(self): # judgment metric if self.cv_judgment_metric is None: return None - return (self.cv_judgment_metric - 2 * - self.cv_judgment_metric_stdev) + return (self.cv_judgment_metric - 2 * self.cv_judgment_metric_stdev) def __repr__(self): params = ', '.join(['%s: %s' % i for i in @@ -329,10 +328,10 @@ def from_csv(self, path): # interpret strings as datetimes on its own. # yes, this is the easiest way to do it for c in inspect(model).attrs: - if type(c) != ColumnProperty: + if not isinstance(c, ColumnProperty): continue col = c.columns[0] - if type(col.type) == DateTime: + if isinstance(col.type, DateTime): df[c.key] = pd.to_datetime(df[c.key], infer_datetime_format=True) @@ -422,11 +421,9 @@ def get_hyperpartitions(self, dataset_id=None, datarun_id=None, method=None, if method is not None: query = query.filter(self.Hyperpartition.method == method) if ignore_gridding_done: - query = query.filter(self.Hyperpartition.status != - PartitionStatus.GRIDDING_DONE) + query = query.filter(self.Hyperpartition.status != PartitionStatus.GRIDDING_DONE) if ignore_errored: - query = query.filter(self.Hyperpartition.status != - PartitionStatus.ERRORED) + query = query.filter(self.Hyperpartition.status != PartitionStatus.ERRORED) return query.all() @@ -449,8 +446,7 @@ def get_classifiers(self, dataset_id=None, datarun_id=None, method=None, query = query.join(self.Hyperpartition)\ .filter(self.Hyperpartition.method == method) if hyperpartition_id is not None: - query = query.filter(self.Classifier.hyperpartition_id == - hyperpartition_id) + query = query.filter(self.Classifier.hyperpartition_id == hyperpartition_id) if status is not None: query = query.filter(self.Classifier.status == status) @@ -614,8 +610,9 @@ def mark_classifier_errored(self, classifier_id, error_message): classifier.error_message = error_message classifier.status = ClassifierStatus.ERRORED classifier.end_time = datetime.now() - if (self.get_number_of_hyperpartition_errors(classifier.hyperpartition_id) > - MAX_HYPERPARTITION_ERRORS): + + noh_errors = self.get_number_of_hyperpartition_errors(classifier.hyperpartition_id) + if noh_errors > MAX_HYPERPARTITION_ERRORS: self.mark_hyperpartition_errored(classifier.hyperpartition_id) @try_with_session(commit=True) diff --git a/atm/encoder.py b/atm/encoder.py index 60dbc4f..2e4b714 100644 --- a/atm/encoder.py +++ b/atm/encoder.py @@ -57,29 +57,32 @@ def fit(self, data): raise KeyError('Class column "%s" not found in dataset!' % self.class_column) - cat_cols = [] + self.categorical_columns = [] if self.feature_columns is None: - features = data.drop([self.class_column], axis=1) - self.feature_columns = features.columns + X = data.drop([self.class_column], axis=1) + self.feature_columns = X.columns else: - features = data[self.feature_columns] + X = data[self.feature_columns] # encode categorical columns, leave ordinal values alone - for column in features.columns: - if features[column].dtype == 'object': + for column in X.columns: + if X[column].dtype == 'object': # save the indices of categorical columns for one-hot encoding - cat_cols.append(features.columns.get_loc(column)) + self.categorical_columns.append(X.columns.get_loc(column)) # encode each feature as an integer in range(unique_vals) le = LabelEncoder() - features[column] = le.fit_transform(features[column]) + X[column] = le.fit_transform(X[column]) self.column_encoders[column] = le # One-hot encode the whole feature matrix. # Set sparse to False so that we can test for NaNs in the output - self.feature_encoder = OneHotEncoder(categorical_features=cat_cols, - sparse=False) - self.feature_encoder.fit(features) + if self.categorical_columns: + self.feature_encoder = OneHotEncoder( + categorical_features=self.categorical_columns, + sparse=False + ) + self.feature_encoder.fit(X) # Train an encoder for the label as well labels = np.array(data[[self.class_column]]) @@ -99,14 +102,19 @@ def transform(self, data): else: y = None - features = data[self.feature_columns] + X = data[self.feature_columns] - # encode each categorical feature as an integer - for column, encoder in list(self.column_encoders.items()): - features[column] = encoder.transform(features[column]) + # one-hot encode the categorical X + if self.categorical_columns: - # one-hot encode the categorical features - X = self.feature_encoder.transform(features) + # encode each categorical feature as an integer + for column, encoder in list(self.column_encoders.items()): + X[column] = encoder.transform(X[column]) + + X = self.feature_encoder.transform(X) + + else: + X = X.values return X, y diff --git a/atm/method.py b/atm/method.py index 297d11e..0ccf1f2 100644 --- a/atm/method.py +++ b/atm/method.py @@ -31,7 +31,7 @@ def is_constant(self): return len(self.range) == 1 def as_tunable(self): - return btb.HyperParameter(typ=self.type, rang=self.range) + return btb.HyperParameter(param_type=self.type, param_range=self.range) class Categorical(HyperParameter): @@ -63,7 +63,7 @@ def is_constant(self): return len(self.values) == 1 def as_tunable(self): - return btb.HyperParameter(typ=self.type, rang=self.values) + return btb.HyperParameter(param_type=self.type, param_range=self.values) class List(HyperParameter): @@ -92,6 +92,7 @@ class HyperPartition(object): """ Class which holds the hyperparameter settings that define a hyperpartition. """ + def __init__(self, categoricals, constants, tunables): """ categoricals: the values for this hyperpartition which have been fixed @@ -137,6 +138,7 @@ class Method(object): hyperparameter arguments it needs to run. Its main purpose is to generate hyperpartitions (possible combinations of categorical hyperparameters). """ + def __init__(self, method): """ method: method code or path to JSON file containing all the information @@ -167,7 +169,7 @@ def __init__(self, method): # CPT with a size hyperparameter and sets of element hyperparameters # conditioned on the size. for name, param in list(self.parameters.items()): - if type(param) == List: + if isinstance(param, List): elements, conditions = param.get_elements() for e in elements: self.parameters[e] = param.element diff --git a/atm/metrics.py b/atm/metrics.py index 68e418f..e06e0da 100644 --- a/atm/metrics.py +++ b/atm/metrics.py @@ -5,13 +5,12 @@ import numpy as np import pandas as pd from past.utils import old_div -from sklearn.metrics import (accuracy_score, average_precision_score, - cohen_kappa_score, f1_score, matthews_corrcoef, - precision_recall_curve, roc_auc_score, roc_curve) +from sklearn.metrics import ( + accuracy_score, average_precision_score, cohen_kappa_score, f1_score, matthews_corrcoef, + precision_recall_curve, roc_auc_score, roc_curve) from sklearn.model_selection import StratifiedKFold -from atm.constants import (METRICS_BINARY, METRICS_MULTICLASS, N_FOLDS_DEFAULT, - Metrics) +from atm.constants import METRICS_BINARY, METRICS_MULTICLASS, N_FOLDS_DEFAULT, Metrics def rank_n_accuracy(y_true, y_prob_mat, n=0.33): diff --git a/atm/models.py b/atm/models.py new file mode 100644 index 0000000..399919c --- /dev/null +++ b/atm/models.py @@ -0,0 +1,241 @@ +from __future__ import absolute_import, division, unicode_literals + +import logging +import os +import random +import time +from builtins import map, object +from datetime import datetime, timedelta +from operator import attrgetter + +from past.utils import old_div + +from atm.config import initialize_logging, load_config +from atm.constants import PROJECT_ROOT, TIME_FMT, PartitionStatus +from atm.database import Database +from atm.encoder import MetaData +from atm.method import Method +from atm.utilities import download_data, get_public_ip +from atm.worker import ClassifierError, Worker + +# load the library-wide logger +logger = logging.getLogger('atm') + + +class ATM(object): + """ + Thiss class is code API instance that allows you to use ATM in your python code. + """ + + LOOP_WAIT = 1 + + def __init__(self, **kwargs): + + if kwargs.get('log_config') is None: + kwargs['log_config'] = os.path.join(PROJECT_ROOT, + 'config/templates/log-script.yaml') + + self.sql_conf, self.run_conf, self.aws_conf, self.log_conf = load_config(**kwargs) + + self.db = Database(**vars(self.sql_conf)) + + initialize_logging(self.log_conf) + + def work(self, datarun_ids=None, save_files=False, choose_randomly=True, + cloud_mode=False, total_time=None, wait=True): + """ + Check the ModelHub database for unfinished dataruns, and spawn workers to + work on them as they are added. This process will continue to run until it + exceeds total_time or is broken with ctrl-C. + + datarun_ids (optional): list of IDs of dataruns to compute on. If None, + this will work on all unfinished dataruns in the database. + choose_randomly: if True, work on all highest-priority dataruns in random + order. If False, work on them in sequential order (by ID) + cloud_mode: if True, save processed datasets to AWS. If this option is set, + aws_config must be supplied. + total_time (optional): if set to an integer, this worker will only work for + total_time seconds. Otherwise, it will continue working until all + dataruns are complete (or indefinitely). + wait: if True, once all dataruns in the database are complete, keep spinning + and wait for new runs to be added. If False, exit once all dataruns are + complete. + """ + start_time = datetime.now() + public_ip = get_public_ip() + + # main loop + while True: + # get all pending and running dataruns, or all pending/running dataruns + # from the list we were given + dataruns = self.db.get_dataruns(include_ids=datarun_ids, ignore_complete=True) + if not dataruns: + if wait: + logger.warning('No dataruns found. Sleeping %d seconds and trying again.', + ATM.LOOP_WAIT) + time.sleep(ATM.LOOP_WAIT) + continue + + else: + logger.warning('No dataruns found. Exiting.') + break + + max_priority = max([datarun.priority for datarun in dataruns]) + priority_runs = [r for r in dataruns if r.priority == max_priority] + + # either choose a run randomly, or take the run with the lowest ID + if choose_randomly: + run = random.choice(priority_runs) + else: + run = sorted(dataruns, key=attrgetter('id'))[0] + + # say we've started working on this datarun, if we haven't already + self.db.mark_datarun_running(run.id) + + logger.info('Computing on datarun %d' % run.id) + # actual work happens here + worker = Worker(self.db, run, save_files=save_files, + cloud_mode=cloud_mode, aws_config=self.aws_conf, + log_config=self.log_conf, public_ip=public_ip) + try: + worker.run_classifier() + + except ClassifierError: + # the exception has already been handled; just wait a sec so we + # don't go out of control reporting errors + logger.warning('Something went wrong. Sleeping %d seconds.', ATM.LOOP_WAIT) + time.sleep(ATM.LOOP_WAIT) + + elapsed_time = (datetime.now() - start_time).total_seconds() + if total_time is not None and elapsed_time >= total_time: + logger.warning('Total run time for worker exceeded; exiting.') + break + + def create_dataset(self): + """ + Create a dataset and add it to the ModelHub database. + """ + # download data to the local filesystem to extract metadata + train_local, test_local = download_data(self.run_conf.train_path, + self.run_conf.test_path, + self.aws_conf) + + # create the name of the dataset from the path to the data + name = os.path.basename(train_local) + name = name.replace("_train.csv", "").replace(".csv", "") + + # process the data into the form ATM needs and save it to disk + meta = MetaData(self.run_conf.class_column, train_local, test_local) + + # enter dataset into database + dataset = self.db.create_dataset(name=name, + description=self.run_conf.data_description, + train_path=self.run_conf.train_path, + test_path=self.run_conf.test_path, + class_column=self.run_conf.class_column, + n_examples=meta.n_examples, + k_classes=meta.k_classes, + d_features=meta.d_features, + majority=meta.majority, + size_kb=old_div(meta.size, 1000)) + return dataset + + def create_datarun(self, dataset): + """ + Given a config, creates a set of dataruns for the config and enters them into + the database. Returns the ID of the created datarun. + + dataset: Dataset SQLAlchemy ORM object + """ + # describe the datarun by its tuner and selector + run_description = '__'.join([self.run_conf.tuner, self.run_conf.selector]) + + # set the deadline, if applicable + deadline = self.run_conf.deadline + if deadline: + deadline = datetime.strptime(deadline, TIME_FMT) + # this overrides the otherwise configured budget_type + # TODO: why not walltime and classifiers budget simultaneously? + self.run_conf.budget_type = 'walltime' + elif self.run_conf.budget_type == 'walltime': + deadline = datetime.now() + timedelta(minutes=self.run_conf.budget) + + target = self.run_conf.score_target + '_judgment_metric' + datarun = self.db.create_datarun(dataset_id=dataset.id, + description=run_description, + tuner=self.run_conf.tuner, + selector=self.run_conf.selector, + gridding=self.run_conf.gridding, + priority=self.run_conf.priority, + budget_type=self.run_conf.budget_type, + budget=self.run_conf.budget, + deadline=deadline, + metric=self.run_conf.metric, + score_target=target, + k_window=self.run_conf.k_window, + r_minimum=self.run_conf.r_minimum) + return datarun + + def enter_data(self, run_per_partition=False): + """ + Generate a datarun, including a dataset if necessary. + + Returns: ID of the generated datarun + """ + # connect to the database + + # if the user has provided a dataset id, use that. Otherwise, create a new + # dataset based on the arguments we were passed. + if self.run_conf.dataset_id is None: + dataset = self.create_dataset() + self.run_conf.dataset_id = dataset.id + else: + dataset = self.db.get_dataset(self.run_conf.dataset_id) + + method_parts = {} + for m in self.run_conf.methods: + # enumerate all combinations of categorical variables for this method + method = Method(m) + method_parts[m] = method.get_hyperpartitions() + logger.info('method %s has %d hyperpartitions' % + (m, len(method_parts[m]))) + + # create hyperpartitions and datarun(s) + run_ids = [] + if not run_per_partition: + logger.debug('saving datarun...') + datarun = self.create_datarun(dataset) + + logger.debug('saving hyperpartions...') + for method, parts in list(method_parts.items()): + for part in parts: + # if necessary, create a new datarun for each hyperpartition. + # This setting is useful for debugging. + if run_per_partition: + datarun = self.create_datarun(dataset) + run_ids.append(datarun.id) + + # create a new hyperpartition in the database + self.db.create_hyperpartition(datarun_id=datarun.id, + method=method, + tunables=part.tunables, + constants=part.constants, + categoricals=part.categoricals, + status=PartitionStatus.INCOMPLETE) + + logger.info('Data entry complete. Summary:') + logger.info('\tDataset ID: %d', dataset.id) + logger.info('\tTraining data: %s', dataset.train_path) + logger.info('\tTest data: %s', (dataset.test_path or 'None')) + + if run_per_partition: + logger.info('\tDatarun IDs: %s', ', '.join(map(str, run_ids))) + + else: + logger.info('\tDatarun ID: %d', datarun.id) + + logger.info('\tHyperpartition selection strategy: %s', datarun.selector) + logger.info('\tParameter tuning strategy: %s', datarun.tuner) + logger.info('\tBudget: %d (%s)', datarun.budget, datarun.budget_type) + + return run_ids or datarun.id diff --git a/atm/tests/__init__.py b/atm/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/atm/tests/integration_tests/__init__.py b/atm/tests/integration_tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/atm/tests/unit_tests/__init__.py b/atm/tests/unit_tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/atm/utilities.py b/atm/utilities.py index af09b2d..ba3f7d7 100644 --- a/atm/utilities.py +++ b/atm/utilities.py @@ -12,8 +12,8 @@ import numpy as np import requests from boto.s3.connection import Key, S3Connection -from btb import ParamTypes +from atm.compat import getargs from atm.constants import DATA_DL_PATH, HTTP_PREFIX, S3_PREFIX, FileType # global variable storing this machine's public IP address @@ -92,43 +92,46 @@ def obj_has_method(obj, method): # Converting hyperparameters to and from BTB-compatible formats -def vector_to_params(vector, tunables, categoricals, constants): +def update_params(params, categoricals, constants): """ - Converts a numpy vector to a dictionary mapping keys to named parameters. + Update params with categoricals and constants for the fitting proces. - vector: single example to convert + params: params proposed by the tuner Examples of the format for SVM sigmoid hyperpartition: - tunables = [('C', HyperParameter(type='float_exp', range=(1e-05, 1e5))), - ('degree', HyperParameter(type='int', range=(2, 4))), - ('gamma', HyperParameter(type='float_exp', range=(1e-05, 1e5)))] - categoricals = (('kernel', 'poly'), ('probability', True), ('_scale', True)) constants = [('cache_size', 15000)] """ - params = {} - - # add the tunables - for i, elt in enumerate(vector): - key, struct = tunables[i] - if struct.type in [ParamTypes.INT, ParamTypes.INT_EXP]: - params[key] = int(elt) - elif struct.type in [ParamTypes.FLOAT, ParamTypes.FLOAT_EXP]: - params[key] = float(elt) - else: - raise ValueError('Unknown data type: {}'.format(struct.type)) - - # add the fixed categorical settings and fixed constant values for key, value in categoricals + constants: params[key] = value return params +def get_instance(class_, **kwargs): + """Instantiate an instance of the given class with unused kwargs + + Args: + class_ (type): class to instantiate + **kwargs: keyword arguments to specific selector class + + Returns: + instance of specific class with the args that accepts. + """ + init_args = getargs(class_.__init__) + relevant_kwargs = { + k: kwargs[k] + for k in kwargs + if k in init_args + } + + return class_(**relevant_kwargs) + + def params_to_vectors(params, tunables): """ Converts a list of parameter vectors (with metadata) into a numpy array @@ -158,23 +161,12 @@ def params_to_vectors(params, tunables): for i, p in enumerate(params): for j, k in enumerate(keys): vectors[i, j] = p[k] + return vectors # Serializing and deserializing data on disk -def _make_save_path_old(dir, classifier, suffix): - """ - Generate the base save path for a classifier's model and metrics files, - based on the classifier's dataset name and hyperparameters. - """ - run_hash = hash_string(classifier.datarun.dataset.name) - params_hash = hash_dict(classifier.hyperparameter_values) - filename = "%s-%s-%s.%s" % (run_hash, params_hash, - classifier.datarun.description, suffix) - return os.path.join(dir, filename) - - def make_save_path(dir, classifier, suffix): """ Generate the base save path for a classifier's model and metrics files, @@ -323,14 +315,12 @@ def download_data(train_path, test_path=None, aws_config=None): if train_type == FileType.HTTP: assert (download_file_http(train_path) == local_train_path) elif train_type == FileType.S3: - assert (download_file_s3(train_path, aws_config=aws_config) == - local_train_path) + assert (download_file_s3(train_path, aws_config=aws_config) == local_train_path) if local_test_path and not os.path.isfile(local_test_path): if test_type == FileType.HTTP: assert (download_file_http(test_path) == local_test_path) elif test_type == FileType.S3: - assert (download_file_s3(test_path, aws_config=aws_config) == - local_test_path) + assert (download_file_s3(test_path, aws_config=aws_config) == local_test_path) return local_train_path, local_test_path diff --git a/atm/worker.py b/atm/worker.py index d3159ac..bd44003 100644 --- a/atm/worker.py +++ b/atm/worker.py @@ -5,26 +5,22 @@ import imp import logging import os -import random import re -import time import traceback import warnings from builtins import object, str from collections import defaultdict -from operator import attrgetter import numpy as np from boto.s3.connection import Key as S3Key from boto.s3.connection import S3Connection +from atm.classifier import Model from atm.config import LogConfig from atm.constants import CUSTOM_CLASS_REGEX, SELECTORS_MAP, TUNERS_MAP from atm.database import ClassifierStatus, db_session -from atm.model import Model -from atm.utilities import (download_data, ensure_directory, get_public_ip, - params_to_vectors, save_metrics, save_model, - vector_to_params) +from atm.utilities import ( + download_data, ensure_directory, get_instance, save_metrics, save_model, update_params) # shhh warnings.filterwarnings('ignore') @@ -32,9 +28,6 @@ # for garrays os.environ['GNUMPY_IMPLICIT_CONVERSION'] = 'allow' -# how long to sleep between loops while waiting for new dataruns to be added -LOOP_WAIT = 1 - # load the library-wide logger logger = logging.getLogger('atm') @@ -90,6 +83,7 @@ def load_selector(self): self.datarun.selector).groups() mod = imp.load_source('btb.selection.custom', path) Selector = getattr(mod, classname) + logger.info('Selector: %s' % Selector) # generate the arguments we need to initialize the selector @@ -97,12 +91,14 @@ def load_selector(self): hp_by_method = defaultdict(list) for hp in hyperpartitions: hp_by_method[hp.method].append(hp.id) + hyperpartition_ids = [hp.id for hp in hyperpartitions] # Selector classes support passing in redundant arguments - self.selector = Selector(choices=hyperpartition_ids, - k=self.datarun.k_window, - by_algorithm=dict(hp_by_method)) + self.selector = get_instance(Selector, + choices=hyperpartition_ids, + k=self.datarun.k_window, + by_algorithm=dict(hp_by_method)) def load_tuner(self): """ @@ -116,10 +112,10 @@ def load_tuner(self): if self.datarun.tuner in TUNERS_MAP: self.Tuner = TUNERS_MAP[self.datarun.tuner] else: - path, classname = re.match(CUSTOM_CLASS_REGEX, - self.datarun.tuner).groups() + path, classname = re.match(CUSTOM_CLASS_REGEX, self.datarun.tuner).groups() mod = imp.load_source('btb.tuning.custom', path) self.Tuner = getattr(mod, classname) + logger.info('Tuner: %s' % self.Tuner) def select_hyperpartition(self): @@ -163,43 +159,39 @@ def tune_hyperparameters(self, hyperpartition): if not len(tunables): logger.warning('No tunables for hyperpartition %d' % hyperpartition.id) self.db.mark_hyperpartition_gridding_done(hyperpartition.id) - return vector_to_params(vector=[], - tunables=tunables, - categoricals=hyperpartition.categoricals, - constants=hyperpartition.constants) + return update_params(params=[], + tunables=tunables, + categoricals=hyperpartition.categoricals, + constants=hyperpartition.constants) # Get previously-used parameters: every classifier should either be # completed or have thrown an error all_clfs = self.db.get_classifiers(hyperpartition_id=hyperpartition.id) - classifiers = [c for c in all_clfs - if c.status == ClassifierStatus.COMPLETE] + classifiers = [c for c in all_clfs if c.status == ClassifierStatus.COMPLETE] - # Extract parameters and scores as numpy arrays from classifiers - X = params_to_vectors([c.hyperparameter_values for c in classifiers], - tunables) - y = np.array([float(getattr(c, self.datarun.score_target)) - for c in classifiers]) + X = [c.hyperparameter_values for c in classifiers] + y = np.array([float(getattr(c, self.datarun.score_target)) for c in classifiers]) # Initialize the tuner and propose a new set of parameters # this has to be initialized with information from the hyperpartition, so we # need to do it fresh for each classifier (not in load_tuner) - tuner = self.Tuner(tunables=tunables, - gridding=self.datarun.gridding, - r_minimum=self.datarun.r_minimum) - tuner.fit(X, y) - vector = tuner.propose() - - if vector is None and self.datarun.gridding: + tuner = get_instance(self.Tuner, + tunables=tunables, + gridding=self.datarun.gridding, + r_minimum=self.datarun.r_minimum) + if len(X) > 0: + tuner.add(X, y) + + params = tuner.propose() + if params is None and self.datarun.gridding: logger.info('Gridding done for hyperpartition %d' % hyperpartition.id) self.db.mark_hyperpartition_gridding_done(hyperpartition.id) return None - # Convert the numpy array of parameters to a form that can be - # interpreted by ATM, then return. - return vector_to_params(vector=vector, - tunables=tunables, - categoricals=hyperpartition.categoricals, - constants=hyperpartition.constants) + # Append categorical and constants to the params. + return update_params(params=params, + categoricals=hyperpartition.categoricals, + constants=hyperpartition.constants) def test_classifier(self, method, params): """ @@ -211,11 +203,13 @@ def test_classifier(self, method, params): judgment_metric=self.datarun.metric, class_column=self.dataset.class_column, verbose_metrics=self.verbose_metrics) + train_path, test_path = download_data(self.dataset.train_path, self.dataset.test_path, self.aws_config) - metrics = model.train_test(train_path=train_path, - test_path=test_path) + + metrics = model.train_test(train_path=train_path, test_path=test_path) + target = self.datarun.score_target def metric_string(model): @@ -233,11 +227,11 @@ def metric_string(model): score_target=target) if old_best is not None: if getattr(model, target) > getattr(old_best, target): - logger.info('New best score! Previous best (classifier %s): %s' - % (old_best.id, metric_string(old_best))) + logger.info('New best score! Previous best (classifier %s): %s', + old_best.id, metric_string(old_best)) else: - logger.info('Best so far (classifier %s): %s' % (old_best.id, - metric_string(old_best))) + logger.info('Best so far (classifier %s): %s', + old_best.id, metric_string(old_best)) return model, metrics @@ -265,11 +259,11 @@ def save_classifier(self, classifier_id, model, metrics): if self.cloud_mode: try: self.save_classifier_cloud(model_path, metric_path) + except Exception: msg = traceback.format_exc() logger.error('Error in save_classifier_cloud()') - self.db.mark_classifier_errored(classifier_id, - error_message=msg) + self.db.mark_classifier_errored(classifier_id, error_message=msg) else: model_path = None metric_path = None @@ -376,6 +370,7 @@ def run_classifier(self, hyperpartition_id=None): # use tuner to choose a set of parameters for the hyperpartition params = self.tune_hyperparameters(hyperpartition) + except Exception: logger.error('Error choosing hyperparameters: datarun=%s' % str(self.datarun)) logger.error(traceback.format_exc()) @@ -389,6 +384,7 @@ def run_classifier(self, hyperpartition_id=None): param_info = 'Chose parameters for method "%s":' % hyperpartition.method for k in sorted(params.keys()): param_info += '\n\t%s = %s' % (k, params[k]) + logger.info(param_info) logger.debug('Creating classifier...') @@ -402,83 +398,10 @@ def run_classifier(self, hyperpartition_id=None): model, metrics = self.test_classifier(hyperpartition.method, params) logger.debug('Saving classifier...') self.save_classifier(classifier.id, model, metrics) + except Exception: msg = traceback.format_exc() logger.error('Error testing classifier: datarun=%s' % str(self.datarun)) logger.error(msg) self.db.mark_classifier_errored(classifier.id, error_message=msg) raise ClassifierError() - - -def work(db, datarun_ids=None, save_files=False, choose_randomly=True, - cloud_mode=False, aws_config=None, log_config=None, total_time=None, - wait=True): - """ - Check the ModelHub database for unfinished dataruns, and spawn workers to - work on them as they are added. This process will continue to run until it - exceeds total_time or is broken with ctrl-C. - - db: Database instance with which we can make queries to ModelHub - datarun_ids (optional): list of IDs of dataruns to compute on. If None, - this will work on all unfinished dataruns in the database. - choose_randomly: if True, work on all highest-priority dataruns in random - order. If False, work on them in sequential order (by ID) - cloud_mode: if True, save processed datasets to AWS. If this option is set, - aws_config must be supplied. - aws_config (optional): if cloud_mode is set, this must be an AWSConfig - object with connection details for an S3 bucket. - total_time (optional): if set to an integer, this worker will only work for - total_time seconds. Otherwise, it will continue working until all - dataruns are complete (or indefinitely). - wait: if True, once all dataruns in the database are complete, keep spinning - and wait for new runs to be added. If False, exit once all dataruns are - complete. - """ - start_time = datetime.datetime.now() - public_ip = get_public_ip() - - # main loop - while True: - # get all pending and running dataruns, or all pending/running dataruns - # from the list we were given - dataruns = db.get_dataruns(include_ids=datarun_ids, - ignore_complete=True) - if not dataruns: - if wait: - logger.warning('No dataruns found. Sleeping %d seconds and trying again.' - % LOOP_WAIT) - time.sleep(LOOP_WAIT) - continue - else: - logger.warning('No dataruns found. Exiting.') - break - - max_priority = max([r.priority for r in dataruns]) - priority_runs = [r for r in dataruns if r.priority == max_priority] - - # either choose a run randomly, or take the run with the lowest ID - if choose_randomly: - run = random.choice(priority_runs) - else: - run = sorted(dataruns, key=attrgetter('id'))[0] - - # say we've started working on this datarun, if we haven't already - db.mark_datarun_running(run.id) - - logger.info('Computing on datarun %d' % run.id) - # actual work happens here - worker = Worker(db, run, save_files=save_files, - cloud_mode=cloud_mode, aws_config=aws_config, - log_config=log_config, public_ip=public_ip) - try: - worker.run_classifier() - except ClassifierError: - # the exception has already been handled; just wait a sec so we - # don't go out of control reporting errors - logger.warning('Something went wrong. Sleeping %d seconds.' % LOOP_WAIT) - time.sleep(LOOP_WAIT) - - elapsed_time = (datetime.datetime.now() - start_time).total_seconds() - if total_time is not None and elapsed_time >= total_time: - logger.warning('Total run time for worker exceeded; exiting.') - break diff --git a/docs/source/add_method.rst b/docs/source/add_method.rst index 2013ce5..937e2d5 100644 --- a/docs/source/add_method.rst +++ b/docs/source/add_method.rst @@ -13,7 +13,7 @@ From 10,000 feet, a "method" in ATM comprises the following: 3. A *conditional parameter tree* that defines how hyperparameters depend on one another; and - + 4. A JSON file in ``atm/methods/`` that describes all of the above. 1. Valid method classes @@ -44,14 +44,14 @@ All configuration for a classification method must be described in a json file w - "name" is a short string (or "code") which ATM uses to refer to the method. - "class" is an import path to the class which Python can interpret. -- "hyperparameters" is a list of hyperparameters which ATM will attempt to tune. - +- "hyperparameters" is a list of hyperparameters which ATM will attempt to tune. + Defining hyperparameters ^^^^^^^^^^^^^^^^^^^^^^^^ -Most parameter definitions have two fields: "type" and either "range" or "values". +Most parameter definitions have two fields: "type" and either "range" or "values". The "type" is one of ["float", "float_exp", "float_cat", "int", "int_exp", "int_cat", "string", "bool"]. Types ending in "_cat" are categorical -types, and those ending in "_exp" are exponential types. +types, and those ending in "_exp" are exponential types. - If the type is ordinal or continuous (e.g. "int" or "float"), "range" defines the upper and lower bound on possible values for the parameter. @@ -131,11 +131,11 @@ If ``kernel`` is set to "matern", it means ``nu`` must also be set. If it's set The example above defines a conditional parameter tree that looks something like this:: - kernel----------------------- - | \ \ + kernel----------------------- + | \ \ matern rational_quadratic exp_sine_squared - | | | | | - nu length_scale alpha length_scale periodicity + | | | | | + nu length_scale alpha length_scale periodicity 3. (Optional) Adding a new method to the ATM library @@ -156,4 +156,3 @@ Test out your method with ``python scripts/test_method.py --method good to go. Commit your changes to a separate branch, then open up a pull request in the main repository. Explain why your method is a useful addition to ATM, and we'll merge it in if we agree! - diff --git a/docs/source/add_to_btb.rst b/docs/source/add_to_btb.rst index 3697c73..14aa0a4 100644 --- a/docs/source/add_to_btb.rst +++ b/docs/source/add_to_btb.rst @@ -1,4 +1,4 @@ -Adding a BTB Selector or Tuner +Adding a BTB Selector or Tuner ============================== BTB is the metamodeling library and framework at the core of ATM. It defines two @@ -24,7 +24,7 @@ create a new datarun with the 'selector' or 'tuner' set to .. Creating a hyperpartition Selector ---------------------------------- - A parameter selector can be created by creating a class which inherits the ``btb.Selector`` class. The class must have a ``select`` method which returns the chose parameters. + A parameter selector can be created by creating a class which inherits the ``btb.Selector`` class. The class must have a ``select`` method which returns the chose parameters. .. Changing the acquisition function @@ -35,4 +35,3 @@ create a new datarun with the 'selector' or 'tuner' set to .. Creating a hyperparameter Tuner ------------------------------- A parameter selector can be created by creating a class which inherits the ``btb.Tuner`` class. The class must have a ``select()`` method which returns the chose parameters. An example which uses the UCB1 algorithm to choose the hyperpartition is shown below. - diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index fe182b8..dd70400 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -10,17 +10,15 @@ Github and look for issues tagged with "`help wanted `_" or "`good first issue `_." An easy first pull request might flesh out the documentation for a confusing feature or just fix a typo. You can also file an issue to report a -bug, suggest a feature, or ask a question. +bug, suggest a feature, or ask a question. If you're looking to make a more in-depth contribution, check out our guides on `adding a classification method `_ and `adding a BTB Tuner or Selector `_. Requirements ------------ -If you'd like to contribute code or documentation, you should install the extra -requirements for testing, style checking, and building documents with:: - - pip install -r requirements-dev.txt +If you'd like to contribute code or documentation, to have installed the project in +development mode. Style ----- @@ -38,9 +36,9 @@ Tests We currently have a limited (for now!) suite of unit tests that ensure at least most of ATM is working correctly. You can run the tests locally with ``pytest`` (which will use your local python environment) or ``tox`` (which will create a -new one from scratch); All tests should pass for every commit on master -- this +new one from scratch); All tests should pass for every commit on master -- this means you'll have to update the code in ``atm/tests/unit_tests`` if you modify -the way anything works. In addition, you should create new tests for any new +the way anything works. In addition, you should create new tests for any new features or functionalities you add. See the `pytest documentation `_ and the existing tests for more information. diff --git a/docs/source/database.rst b/docs/source/database.rst index 3cbb67c..1571bc8 100644 --- a/docs/source/database.rst +++ b/docs/source/database.rst @@ -17,7 +17,7 @@ A Dataset represents a single set of data which can be used to train and test models by ATM. The table stores information about the location of the data as well as metadata to help with analysis. -- ``dataset_id`` (Int): Unique identifier for the dataset. +- ``dataset_id`` (Int): Unique identifier for the dataset. - ``name`` (String): Identifier string for a classification technique. - ``description`` (String): Human-readable description of the dataset. - not described in the paper @@ -27,12 +27,12 @@ well as metadata to help with analysis. The metadata fields below are not described in the paper. -- ``n_examples`` (Int): Number of samples (rows) in the dataset. -- ``k_classes`` (Int): Number of classes in the dataset. +- ``n_examples`` (Int): Number of samples (rows) in the dataset. +- ``k_classes`` (Int): Number of classes in the dataset. - ``d_features`` (Int): Number of features in the dataset. - ``majority`` (Number): Ratio of the number of samples in the largest class to - the number of samples in all other classes. -- ``size_kb`` (Int): Approximate size of the dataset in KB. + the number of samples in all other classes. +- ``size_kb`` (Int): Approximate size of the dataset in KB. Dataruns @@ -87,7 +87,7 @@ ATM configuration: multiclass problems - not in the paper - ``score_target`` (Enum): One of ["cv", "test", "mu_sigma"]. Determines how the - final comparative metric (the *judgment metric*) is calculated. + final comparative metric (the *judgment metric*) is calculated. - "cv" (cross-validation): the judgment metric is the average of a 5-fold cross-validation test. - "test": the judgment metric is computed on the test data. diff --git a/docs/source/introduction.rst b/docs/source/introduction.rst index b6bc61a..78db50b 100644 --- a/docs/source/introduction.rst +++ b/docs/source/introduction.rst @@ -14,7 +14,7 @@ Background `AutoML `_ systems attempt to automate part or all of the machine learning pipeline, from data cleaning to feature extraction to model selection and tuning. ATM focuses on the last part of the machine-learning -pipeline: model selection and hyperparameter tuning. +pipeline: model selection and hyperparameter tuning. Machine learning algorithms typically have a number of parameters (called *hyperparameters*) that must be chosen in order to define their behavior. ATM @@ -34,7 +34,7 @@ hyperparameters (using another HDI Project library, `BTB best model within a limited amount of time or by training a limited amount of total models. -ATM can be used locally or on a cloud-computing cluster with AWS. +ATM can be used locally or on a cloud-computing cluster with AWS. Currently, ATM only works with classification problems, but the project is under active development. If you like the project and would like to help out, check out our guide to `contributing `_! diff --git a/docs/source/quickstart.rst b/docs/source/quickstart.rst index 10dcb68..63a7964 100644 --- a/docs/source/quickstart.rst +++ b/docs/source/quickstart.rst @@ -5,7 +5,7 @@ This page is a quick tutorial to help you get ATM up and running for the first time. We'll use a featurized dataset for a binary classification problem, already saved in ``atm/data/test/pollution_1.csv``. This is one of the datasets available on `openml.org `_. More information about the -data can be found `here `_. +data can be found `here `_. Our goal is predict mortality using the metrics associated with the air pollution. Below we show a snapshot of the csv file. The dataset has 15 @@ -34,14 +34,14 @@ Before we can train any classifiers, we need to create a datarun. In ATM, a datarun is a single logical machine learning task. The ``enter_data.py`` script will set up everything you need.:: -$ python scripts/enter_data.py +(atm-env) $ atm enter_data The first time you run it, the above command will create a ModelHub database, a dataset, and a datarun. If you run it without any arguments, it will load configuration from the default values defined in ``atm/config.py``. By default, it will create a new SQLite3 database at ./atm.db, create a new dataset instance which refers to the data at ``atm/data/test/pollution_1.csv``, and create a -datarun instance which points to that dataset. +datarun instance which points to that dataset. The command should produce output that looks something like this::: @@ -75,7 +75,7 @@ An ATM *worker* is a process that connects to a ModelHub, asks it what dataruns need to be worked on, and trains and tests classifiers until all the work is done. To run one, use the following command:: -$ python scripts/worker.py +(atm-env) $ atm worker.py This will start a process that builds classifiers, tests them, and saves them to the ./models/ directory. As it runs, it should print output indicating which @@ -102,7 +102,7 @@ of hyperparameters to find the absolute best model for your problem. You can break out of the worker with Ctrl+C and restart it with the same command; it will pick up right where it left off. You can also run the command simultaneously in different terminals to parallelize the work -- all workers -will refer to the same ModelHub database. +will refer to the same ModelHub database. Occassionally, a worker will encounter an error in the process of building and testing a classifier. Don't worry: when this happens, the worker will print diff --git a/docs/source/setup.rst b/docs/source/setup.rst index 61b2800..017fb5b 100644 --- a/docs/source/setup.rst +++ b/docs/source/setup.rst @@ -4,18 +4,53 @@ This page will guide you though downloading and installing ATM. 0. Requirements --------------- -Currently, ATM is only compatible with Python 2.7 and \*NIX systems, and `git -`_ is required to download and update the software. -1. Clone the project ------------------------ -From the terminal, run:: +Currently, ATM is only compatible with Python 2.7, 3.5 and 3.6 and \*NIX systems. - $ git clone https://github.com/hdi-project/atm.git ./atm +We also recommend using `virtualenv `_, which +you can install as follows.:: + $ sudo apt-get install python-pip + $ sudo pip install virtualenv + +For development, also `git `_ is required in order to download and +update the software. + +1. Install ATM +-------------- + +Install using pip +~~~~~~~~~~~~~~~~~ + +The recommended way to install ATM is using `pip `_ inside +a dedicated virtualenv:: + + $ virtualenv atm-env + $ . atm-env/bin/activate + (atm-env) $ pip install atm + +Install from source +~~~~~~~~~~~~~~~~~~~ + +Alternatively, and for development, you can clone the repository and install it from +source by running ``make install``:: + + $ git clone https://github.com/hdi-project/atm.git + $ cd atm + $ virtualenv atm-env + $ . atm-env/bin/activate + (atm-env) $ make install + +For development, replace the last command with ``make install-develop`` command in order to +also install all the required dependencies for testing and linting. + +.. note:: You will need to execute the command ``. atm-env/bin/activate`` to activate the + virtualenv again every time you want to start working on ATM. You will know that your + virtualenv has been activated if you can see the **(atm-env)** prefix on your prompt. + If you do not, activate it again! 2. Install a database ------------------------ +--------------------- ATM requires a SQL-like database to store information about datasets, dataruns, and classifiers. It's currently compatible with the SQLite3 and MySQL dialects. @@ -35,33 +70,8 @@ library in order for SQLAlchemy to work correctly:: $ sudo apt-get install libmysqlclient-dev -3. Install Python dependencies ------------------------------- - -We recommend using `pip `_ and `virtualenv -`_ to make this process easier.:: - - $ sudo apt-get install python-pip - $ sudo pip install virtualenv - -Next, create the virtual environment and enter into it:: - - $ virtualenv atm-env - $ . atm-env/bin/activate - (atm-env) $ - -The required packages are: - -.. literalinclude:: ../../requirements.txt - -Install them with pip:: - - (atm-env) $ pip install -r requirements.txt - -Or, if you want to use ATM as a library in other applications, you can install -it as a package. This will install the requirements as well:: - - (atm-env) $ pip install -e . --process-dependency-links +3. Start using ATM! +------------------- You're all set. Head over to the `quick-start `_ section to create and execute your first job with ATM. diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index e1de6d6..6d6b982 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -29,7 +29,7 @@ Here's a handy Python script to create a CSV header line for data that doesn't h separator = "," zip([name for i in range(n_features)], range(1, n_features + 1, 1)) header_row_string = separator.join( - [x + str(y) for (x, y) in + [x + str(y) for (x, y) in ]) return separator.join([class_label_name, header_row_string]) @@ -41,13 +41,14 @@ Once your data in the proper format, you can upload it to the ModelHub for proce Configuration File ^^^^^^^^^^^^^^^^^^ + To run ATM, you must create a configuration file. + A configuration file template is included in ``config/atm.cnf.template`` (and shown below). Since the configuration file contains passwords, it's best to rename it to ``atm.cnf`` so that it will be ignored by git. This is especially true if you plan to make changes to ATM and upload them to the repository. The git repository is setup to ignore all files in the ``config`` folder except ``atm.cnf.template``. - The name of the file must also be a environmental variable called ``ATM_CONFIG_FILE``. For example if the configuration file is called ``atm.cnf`` in the ``config`` directory of the root atm directory, then an environmental variable would created with the command:: @@ -61,7 +62,7 @@ A datarun consists of all the parameters for a single experiment run, including The datarun ID in the database also ties together the `hyperpartitions` (frozen sets) which delineate how ATM can explore different subtypes of classifiers to maximize their performance. Once the configuration file is filled out, we can enter it in ModelHub with:: - (atm-env) $ python enter_data.py + (atm-env) $ atm enter_data Workers ------- @@ -73,7 +74,7 @@ On a Local Machine In local mode, this is simple:: - (atm-env) $ python worker.py + (atm-env) $ atm worker This command can b executed several times to create many workers that operate independently in parallel. How many to run depends of your judgment of your computer's capabilities. diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 22cb6ae..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,23 +0,0 @@ -# general -bumpversion>=0.5.3 -pip>=9.0.1 -watchdog>=0.8.3 - -# build docs -Sphinx>=1.6.5 -sphinx-rtd-theme>=0.2.4 -sphinxcontrib-websupport>=1.0.1 - -# style check -flake8>=3.4.1 -isort>=4.2.15 - -# automatically fix style issues -autoflake>=1.1 -autopep8>=1.3.5 - -# distribute on PyPI -twine>=1.10.0 -wheel>=0.30.0 - --r requirements-test.txt diff --git a/requirements-test.txt b/requirements-test.txt deleted file mode 100644 index 4f7b5ab..0000000 --- a/requirements-test.txt +++ /dev/null @@ -1,8 +0,0 @@ -codecov>=2.0.9 -coverage>=4.5.1 -mock>=2.0.0 -pytest-cov>=2.5.1 -pytest-runner>=3.0 -pytest-xdist>=1.20.1 -pytest>=3.4.2 -tox>=2.9.1 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 543d653..0000000 --- a/requirements.txt +++ /dev/null @@ -1,13 +0,0 @@ -baytune==0.1.2 # This one needs to be exact -boto>=2.48.0 -joblib>=0.11 -future>=0.16.0 -mysqlclient>=1.2 -numpy>=1.13.1 -pandas>=0.22.0 -pyyaml>=3.12 -requests>=2.18.4 -scikit-learn>=0.18.2,<0.20 -scipy>=0.19.1 -sklearn-pandas>=1.5.0 -sqlalchemy>=1.1.14 diff --git a/scripts/__init__.py b/scripts/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/scripts/end_to_end_test.py b/scripts/end_to_end_test.py deleted file mode 100644 index fbc313b..0000000 --- a/scripts/end_to_end_test.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/python2.7 -from __future__ import print_function -import argparse -import os -import yaml -from collections import defaultdict -from os.path import join - -from atm.config import * -from atm.database import Database -from atm.enter_data import enter_data -from atm.utilities import download_file_s3 -from atm.worker import work - -from utilities import * - - -CONF_DIR = os.path.join(PROJECT_ROOT, 'config/test/') -DATA_DIR = os.path.join(PROJECT_ROOT, 'data/test/') -RUN_CONFIG = join(CONF_DIR, 'run-all.yaml') -SQL_CONFIG = join(CONF_DIR, 'sql-sqlite.yaml') - -DATASETS_MAX_MIN = [ - 'wholesale-customers_1.csv', - 'car_1.csv', - 'wall-robot-navigation_1.csv', - 'wall-robot-navigation_2.csv', - 'wall-robot-navigation_3.csv', - 'analcatdata_authorship_1.csv', - 'cardiotocography_1.csv', - 'wine_1.csv', - 'seismic-bumps_1.csv', - 'balance-scale_1.csv', -] -DATASETS_MAX_FIRST = [ - 'wine_1.csv', - 'balance-scale_1.csv', - 'seeds_1.csv', - 'collins_1.csv', - 'cpu_1.csv', - 'vowel_1.csv', - 'car_2.csv', - 'hill-valley_2.csv', - 'rabe_97_1.csv', - 'monks-problems-2_1.csv', -] -DATASETS_SIMPLE = [ - 'pollution_1.csv', # binary test data - 'iris.data.csv', # ternary test data -] - -DATASETS = DATASETS_SIMPLE - - -parser = argparse.ArgumentParser(description=''' -Run a single end-to-end test with 10 sample datasets. -The script will create a datarun for each dataset, then run a worker until the -jobs are finished. -''') -parser.add_argument('--processes', help='number of processes to run concurrently', - type=int, default=4) - -args = parser.parse_args() -sql_config, run_config, _, _ = load_config(sql_path=SQL_CONFIG, - run_path=RUN_CONFIG) - -db = Database(**vars(sql_config)) - -print('creating dataruns...') -datarun_ids = [] -for ds in DATASETS: - run_config.train_path = join(DATA_DIR, ds) - datarun_ids.append(enter_data(sql_config=sql_config, - run_config=run_config)) - -work_parallel(db=db, datarun_ids=datarun_ids, n_procs=args.processes) - -print('workers finished.') - -for rid in datarun_ids: - print_summary(db, rid) diff --git a/scripts/enter_data.py b/scripts/enter_data.py deleted file mode 100755 index 5c936bd..0000000 --- a/scripts/enter_data.py +++ /dev/null @@ -1,51 +0,0 @@ -from __future__ import print_function - -import argparse -import os -import warnings - -from atm import PROJECT_ROOT -from atm.config import (add_arguments_aws_s3, add_arguments_sql, - add_arguments_datarun, add_arguments_logging, - load_config, initialize_logging) -from atm.enter_data import enter_data - -warnings.filterwarnings("ignore") - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description=""" -Creates a dataset (if necessary) and a datarun and adds them to the ModelHub. -All required arguments have default values. Running this script with no -arguments will create a new dataset with the file in data/pollution_1.csv and a -new datarun with the default arguments listed below. - -You can pass yaml configuration files (--sql-config, --aws-config, --run-config) -instead of passing individual arguments. Any arguments in the config files will -override arguments passed on the command line. See the examples in the config/ -folder for more information. """) - # Add argparse arguments for aws, sql, and datarun config - add_arguments_aws_s3(parser) - add_arguments_sql(parser) - add_arguments_datarun(parser) - add_arguments_logging(parser) - parser.add_argument('--run-per-partition', default=False, action='store_true', - help='if set, generate a new datarun for each hyperpartition') - - args = parser.parse_args() - - # default logging config is different if initialized from the command line - if args.log_config is None: - args.log_config = os.path.join(PROJECT_ROOT, - 'config/templates/log-script.yaml') - - # create config objects from the config files and/or command line args - sql_conf, run_conf, aws_conf, log_conf = load_config(sql_path=args.sql_config, - run_path=args.run_config, - aws_path=args.aws_config, - log_path=args.log_config, - **vars(args)) - initialize_logging(log_conf) - - # create and save the dataset and datarun - enter_data(sql_conf, run_conf, aws_conf, args.run_per_partition) diff --git a/scripts/evaluate_btb.py b/scripts/evaluate_btb.py deleted file mode 100644 index 0fd39be..0000000 --- a/scripts/evaluate_btb.py +++ /dev/null @@ -1,100 +0,0 @@ -from __future__ import print_function -import argparse -import os -import random -from os.path import join - -from atm import PROJECT_ROOT -from atm.config import * -from atm.database import Database -from atm.enter_data import enter_data - -from utilities import * - - -CONF_DIR = os.path.join(PROJECT_ROOT, 'config/test/') -RUN_CONFIG = join(CONF_DIR, 'run-default.yaml') -SQL_CONFIG = join(CONF_DIR, 'sql-sqlite.yaml') - -DATASETS_MAX_FIRST = [ - 'collins_1.csv', - 'cpu_1.csv', - 'vowel_1.csv', - 'car_2.csv', - 'hill-valley_2.csv', - 'rabe_97_1.csv', - 'monks-problems-2_1.csv', - # these datasets do not have baseline numbers - #'wine_1.csv', - #'balance-scale_1.csv', - #'seeds_1.csv', -] - - -def btb_test(dataruns=None, datasets=None, processes=1, graph=False, **kwargs): - """ - Run a test datarun using the chosen tuner and selector, and compare it to - the baseline performance. - - Tuner and selector will be specified in **kwargs, along with the rest of the - standard datarun arguments. - """ - sql_conf, run_conf, _, _ = load_config(sql_path=SQL_CONFIG, - run_path=RUN_CONFIG, - **kwargs) - - db = Database(**vars(sql_conf)) - datarun_ids = dataruns or [] - datarun_ids_per_dataset = [[each] for each in dataruns] if dataruns else [] - datasets = datasets or DATASETS_MAX_FIRST - - # if necessary, generate datasets and dataruns - if not datarun_ids: - for ds in datasets: - run_conf.train_path = DATA_URL + ds - run_conf.dataset_id = None - print('Creating 10 dataruns for', run_conf.train_path) - run_ids = [enter_data(sql_conf, run_conf) for i in range(10)] - datarun_ids_per_dataset.append(run_ids) - datarun_ids.extend(run_ids) - - # work on the dataruns til they're done - print('Working on %d dataruns' % len(datarun_ids)) - work_parallel(db=db, datarun_ids=datarun_ids, n_procs=processes) - print('Finished!') - - results = {} - - # compute and maybe graph the results for each dataset - for rids in datarun_ids_per_dataset: - res = report_auc_vs_baseline(db, rids, graph=graph) - results[tuple(rids)] = {'test': res[0], 'baseline': res[1]} - - return results - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description=''' - Test the performance of an AutoML method and compare it to the baseline - performance curve. - ''') - parser.add_argument('--processes', help='number of processes to run concurrently', - type=int, default=1) - parser.add_argument('--graph', action='store_true', default=False, - help='if this flag is inculded, graph the best-so-far ' - 'results of each datarun against the baseline.') - parser.add_argument('--dataruns', nargs='+', type=int, - help='(optional) IDs of previously-created dataruns to ' - 'graph. If this option is included, no new dataruns ' - 'will be created, but any of the specified dataruns ' - 'will be finished if they are not already.') - parser.add_argument('--datasets', nargs='+', - help='(optional) file names of training data to use. ' - 'Each should be a csv file present in the downloaded/ ' - 'folder of the HDI project S3 bucket ' - '(https://s3.amazonaws.com/mit-dai-delphi-datastore/downloaded/).' - 'The default is to use the files in DATASETS_MAX_FIRST.') - add_arguments_datarun(parser) - args = parser.parse_args() - - btb_test(**vars(args)) diff --git a/scripts/method_test.py b/scripts/method_test.py deleted file mode 100644 index c4b505c..0000000 --- a/scripts/method_test.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/python2.7 -from __future__ import print_function -import argparse -import os -import yaml -from collections import defaultdict -from os.path import join - -from atm.config import * -from atm.database import Database -from atm.enter_data import enter_data -from atm.utilities import download_file_s3 -from atm.worker import work - -from utilities import * - - -CONF_DIR = os.path.join(PROJECT_ROOT, 'config/test/') -DATA_DIR = os.path.join(PROJECT_ROOT, 'data/test/') -RUN_CONFIG = join(CONF_DIR, 'run-default.yaml') -SQL_CONFIG = join(CONF_DIR, 'sql-sqlite.yaml') -DATASETS = [ - 'iris.data.csv', - 'pollution_1.csv', -] - - -parser = argparse.ArgumentParser(description=''' -Run a single end-to-end test with 10 sample datasets. -The script will create a datarun for each dataset, then run a worker until the -jobs are finished. -''') -parser.add_argument('--processes', help='number of processes to run concurrently', - type=int, default=1) -parser.add_argument('--method', help='code for method to test') -parser.add_argument('--method-path', help='path to JSON config for method to test') - -args = parser.parse_args() -sql_config, run_config, aws_config, _ = load_config(sql_path=SQL_CONFIG, - run_path=RUN_CONFIG) -db = Database(**vars(sql_config)) - -print('creating dataruns...') -datarun_ids = [] -for ds in DATASETS: - run_config.train_path = join(DATA_DIR, ds) - if args.method: - run_config.methods = [args.method] - else: - run_config.methods = METHODS - datarun_ids.extend(enter_data(sql_config, run_config, aws_config, - run_per_partition=True)) - -print('computing on dataruns', datarun_ids) -work_parallel(db=db, datarun_ids=datarun_ids, aws_config=aws_config, - n_procs=args.processes) - -print('workers finished.') - -for rid in datarun_ids: - print_hp_summary(db, rid) diff --git a/scripts/utilities.py b/scripts/utilities.py deleted file mode 100644 index 4a245bd..0000000 --- a/scripts/utilities.py +++ /dev/null @@ -1,209 +0,0 @@ -from __future__ import print_function -import argparse -import numpy as np -import os - -from collections import defaultdict -from multiprocessing import Process -from sklearn.metrics import auc - -from atm.config import * -from atm.worker import work -from atm.database import db_session -from atm.utilities import download_file_http - -try: - import matplotlib.pyplot as plt -except ImportError: - plt = None - -BASELINE_PATH = os.path.join(PROJECT_ROOT, 'test/baselines/best_so_far_multi_trial/') -DATA_URL = 'https://s3.amazonaws.com/mit-dai-delphi-datastore/downloaded/' -BASELINE_URL = 'https://s3.amazonaws.com/mit-dai-delphi-datastore/best_so_far_multi_trial/' - - -def get_best_so_far(db, datarun_id): - """ - Get a series representing best-so-far performance for datarun_id. - """ - # generate a list of the "best so far" score after each classifier was - # computed (in chronological order) - classifiers = db.get_classifiers(datarun_id=datarun_id) - y = [] - for l in classifiers: - best_so_far = max(y + [l.cv_judgment_metric]) - y.append(best_so_far) - return y - - -def graph_series(length, title, **series): - """ - Graph series of performance metrics against one another. - - length: all series will be truncated to this length - title: what to title the graph - **series: mapping of labels to series of performance data - """ - if plt is None: - raise ImportError("Unable to import matplotlib") - - lines = [] - for label, data in series.items(): - # copy up to `length` of the values in `series` into y. - y = data[:length] - x = range(len(y)) - - # plot y against x - line, = plt.plot(x, y, '-', label=label) - lines.append(line) - - plt.xlabel('classifiers') - plt.ylabel('performance') - plt.title(title) - plt.legend(handles=lines) - plt.show() - -def report_auc_vs_baseline(db, rids, graph=False): - - if len(rids) == 0: - return - rid = rids[0] - with db_session(db): - run = db.get_datarun(rid) - ds = run.dataset - test = np.array([[float(y) for y in get_best_so_far(db, rid)] for rid in rids]) - test = test.T - mean_test = np.mean(test, axis =1).tolist() - - ds_file = os.path.basename(ds.train_path) - bl_path = download_file_http(BASELINE_URL + ds_file, - local_folder=BASELINE_PATH) - with open(bl_path) as f: - baseline = np.array([[float(each) for each in l.strip().split('\t')] for l in f]) - mean_baseline = np.mean(baseline, axis =1).tolist() - - min_len = min(baseline.shape[0], test.shape[0]) - x = range(min_len) - - test_aucs = np.array([auc(x, test[:min_len,row]) for row in range(test.shape[1])]) - bl_aucs = np.array([auc(x, baseline[:min_len,row]) for row in range(baseline.shape[1])]) - # get avg, std, min of AUC over trials - mean_auc_test = np.mean(test_aucs) - mean_auc_bl = np.mean(bl_aucs) - std_auc_test = np.std(test_aucs) - std_auc_bl = np.std(bl_aucs) - min_auc_test = np.min(test_aucs) - min_auc_bl = np.min(bl_aucs) - mean_auc_diff = mean_auc_test - mean_auc_bl - print('Dataset %s (dataruns %s)' % (ds_file, rids)) - print ('Comparing %d trials to baseline generated by %d trials'%(len(rids), baseline.shape[1])) - print('MEAN AUC: test = %.3f, baseline = %.3f (%.3f)' % (mean_auc_test, mean_auc_bl, mean_auc_diff)) - print('STD AUC: test = %.3f, baseline = %.3f' % (std_auc_test, std_auc_bl)) - print('MIN AUC: test = %.3f, baseline = %.3f' % (min_auc_test, min_auc_bl)) - - if graph: - graph_series(100, ds_file, baseline=mean_baseline, test=mean_test) - - return mean_auc_test, mean_auc_bl - - -def print_summary(db, rid): - run = db.get_datarun(rid) - ds = db.get_dataset(run.dataset_id) - print() - print('Dataset %s' % ds) - print('Datarun %s' % run) - - classifiers = db.get_classifiers(datarun_id=rid) - errs = db.get_classifiers(datarun_id=rid, status=ClassifierStatus.ERRORED) - complete = db.get_classifiers(datarun_id=rid, - status=ClassifierStatus.COMPLETE) - print('Classifiers: %d total; %d errors, %d complete' % - (len(classifiers), len(errs), len(complete))) - - best = db.get_best_classifier(score_target=run.score_target, - datarun_id=run.id) - if best is not None: - score = best.cv_judgment_metric - err = 2 * best.cv_judgment_metric_stdev - print('Best result overall: classifier %d, %s = %.3f +- %.3f' %\ - (best.id, run.metric, score, err)) - - -def print_method_summary(db, rid): - # maps methods to sets of hyperpartitions, and hyperpartitions to lists of - # classifiers - alg_map = {a: defaultdict(list) for a in db.get_methods(datarun_id=rid)} - - run = db.get_datarun(rid) - classifiers = db.get_classifiers(datarun_id=rid) - for l in classifiers: - hp = db.get_hyperpartition(l.hyperpartition_id) - alg_map[hp.method][hp.id].append(l) - - for alg, hp_map in alg_map.items(): - print() - print('method %s:' % alg) - - classifiers = sum(hp_map.values(), []) - errored = len([l for l in classifiers if l.status == - ClassifierStatus.ERRORED]) - complete = len([l for l in classifiers if l.status == - ClassifierStatus.COMPLETE]) - print('\t%d errored, %d complete' % (errored, complete)) - - best = db.get_best_classifier(score_target=run.score_target, - datarun_id=rid, method=alg) - if best is not None: - score = best.cv_judgment_metric - err = 2 * best.cv_judgment_metric_stdev - print('\tBest: classifier %s, %s = %.3f +- %.3f' % (best, run.metric, - score, err)) - -def print_hp_summary(db, rid): - run = db.get_datarun(rid) - classifiers = db.get_classifiers(datarun_id=rid) - - part_map = defaultdict(list) - for c in classifiers: - hp = c.hyperpartition_id - part_map[hp].append(c) - - for hp, classifiers in part_map.items(): - print() - print('hyperpartition', hp) - print(db.get_hyperpartition(hp)) - - errored = len([c for c in classifiers if c.status == - ClassifierStatus.ERRORED]) - complete = len([c for c in classifiers if c.status == - ClassifierStatus.COMPLETE]) - print('\t%d errored, %d complete' % (errored, complete)) - - best = db.get_best_classifier(score_target=run.score_target, - datarun_id=rid, hyperpartition_id=hp) - if best is not None: - score = best.cv_judgment_metric - err = 2 * best.cv_judgment_metric_stdev - print('\tBest: classifier %s, %s = %.3f +- %.3f' % (best, run.metric, - score, err)) - -def work_parallel(db, datarun_ids=None, aws_config=None, n_procs=4): - print('starting workers...') - kwargs = dict(db=db, datarun_ids=datarun_ids, save_files=False, - choose_randomly=True, cloud_mode=False, - aws_config=aws_config, wait=False) - - if n_procs > 1: - # spawn a set of worker processes to work on the dataruns - procs = [] - for i in range(n_procs): - p = Process(target=work, kwargs=kwargs) - p.start() - procs.append(p) - - # wait for them to finish - for p in procs: - p.join() - else: - work(**kwargs) diff --git a/scripts/worker.py b/scripts/worker.py deleted file mode 100755 index 3fdc0af..0000000 --- a/scripts/worker.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/python2.7 -from __future__ import print_function - -import argparse -import datetime -import os -import warnings - -from atm import PROJECT_ROOT -from atm.config import (add_arguments_aws_s3, add_arguments_logging, - add_arguments_sql, load_config, initialize_logging) -from atm.database import Database -from atm.worker import Worker, work - -warnings.filterwarnings('ignore') - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Add more classifiers to database') - add_arguments_sql(parser) - add_arguments_aws_s3(parser) - add_arguments_logging(parser) - - # add worker-specific arguments - parser.add_argument('--cloud-mode', action='store_true', default=False, - help='Whether to run this worker in cloud mode') - parser.add_argument('--dataruns', help='Only train on dataruns with these ids', - nargs='+') - parser.add_argument('--time', help='Number of seconds to run worker', type=int) - parser.add_argument('--choose-randomly', action='store_true', - help='Choose dataruns to work on randomly (default = sequential order)') - parser.add_argument('--no-save', dest='save_files', default=True, - action='store_const', const=False, - help="don't save models and metrics at all") - - # parse arguments and load configuration - args = parser.parse_args() - - # default logging config is different if initialized from the command line - if args.log_config is None: - args.log_config = os.path.join(PROJECT_ROOT, - 'config/templates/log-script.yaml') - - sql_config, _, aws_config, log_config = load_config(**vars(args)) - initialize_logging(log_config) - - # let's go - work(db=Database(**vars(sql_config)), - datarun_ids=args.dataruns, - choose_randomly=args.choose_randomly, - save_files=args.save_files, - cloud_mode=args.cloud_mode, - aws_config=aws_config, - log_config=log_config, - total_time=args.time, - wait=False) diff --git a/setup.cfg b/setup.cfg index 79e6832..456fa01 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,17 @@ [bumpversion] -current_version = 0.1.0 +current_version = 0.1.1-dev commit = True tag = True +parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+))? +serialize = + {major}.{minor}.{patch}-{release} + {major}.{minor}.{patch} + +[bumpversion:part:release] +optional_value = release +values = + dev + release [bumpversion:file:setup.py] search = version='{current_version}' @@ -23,18 +33,17 @@ ignore = # Keep empty to prevent default ignores include_trailing_comment = True known_first_party = atm known_third_party = sqlalchemy -line_length=79 +line_length=99 lines_between_types = 0 -multi_line_output = 0 +multi_line_output = 4 use_parentheses = True +not_skip = __init__.py [metadata] description-file = README.md [aliases] -# Define setup.py command aliases here test = pytest [tool:pytest] collect_ignore = ['setup.py'] -python_files = atm/tests/* diff --git a/setup.py b/setup.py index 17797f6..222ece0 100644 --- a/setup.py +++ b/setup.py @@ -9,8 +9,8 @@ with open('HISTORY.md') as history_file: history = history_file.read() -requirements = [ - 'baytune==0.1.2', # This one needs to be exact +install_requires = [ + 'baytune==0.2.5', 'boto>=2.48.0', 'future>=0.16.0', 'joblib>=0.11', @@ -18,7 +18,8 @@ 'numpy>=1.13.1', 'pandas>=0.22.0', 'pyyaml>=3.12', - 'scikit-learn>=0.18.2,<0.20', + 'requests>=2.18.4', + 'scikit-learn>=0.18.2', 'scipy>=0.19.1', 'sklearn-pandas>=1.5.0', 'sqlalchemy>=1.1.14', @@ -28,7 +29,7 @@ 'pytest-runner' ] -test_requirements = [ +tests_require = [ 'mock>=2.0.0', 'pytest-cov>=2.5.1', 'pytest-runner>=3.0', @@ -36,6 +37,34 @@ 'pytest>=3.2.3', ] +development_requires = [ + # general + 'bumpversion>=0.5.3', + 'pip>=9.0.1', + 'watchdog>=0.8.3', + + # docs + 'm2r>=0.2.0', + 'Sphinx>=1.7.1', + 'sphinx_rtd_theme>=0.2.4', + + # style check + 'flake8>=3.7.7', + 'isort>=4.3.4', + + # fix style issues + 'autoflake>=1.1', + 'autopep8>=1.4.3', + + # distribute on PyPI + 'twine>=1.10.0', + 'wheel>=0.30.0', + + # Advanced testing + 'coverage>=4.5.1', + 'tox>=2.9.1', +] + setup( author="MIT Data To AI Lab", author_email='dailabmit@gmail.com', @@ -52,19 +81,27 @@ 'Topic :: Scientific/Engineering :: Artificial Intelligence', ], description="Auto Tune Models", - install_requires=requirements, + entry_points={ + 'console_scripts': [ + 'atm=atm.cli:main' + ] + }, + extras_require={ + 'dev': development_requires + tests_require, + 'tests': tests_require, + }, + include_package_data=True, + install_requires=install_requires, license="MIT license", long_description=readme + '\n\n' + history, long_description_content_type='text/markdown', - include_package_data=True, keywords='machine learning hyperparameters tuning classification', name='atm', packages=find_packages(include=['atm', 'atm.*']), - python_requires='>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*', setup_requires=setup_requires, - test_suite='atm/tests', - tests_require=test_requirements, + test_suite='tests', + tests_require=tests_require, url='https://github.com/HDI-project/ATM', - version='0.1.0', + version='0.1.1-dev', zip_safe=False, ) diff --git a/atm/tests/unit_tests/test_enter_data.py b/tests/test_enter_data.py similarity index 100% rename from atm/tests/unit_tests/test_enter_data.py rename to tests/test_enter_data.py diff --git a/atm/tests/unit_tests/test_method.py b/tests/test_method.py similarity index 90% rename from atm/tests/unit_tests/test_method.py rename to tests/test_method.py index 701165f..ac2888b 100644 --- a/atm/tests/unit_tests/test_method.py +++ b/tests/test_method.py @@ -31,5 +31,4 @@ def test_enumerate(): assert len(hps) == 12 assert all('a' in list(zip(*hp.categoricals))[0] for hp in hps) assert all(('f', 0.5) in hp.constants for hp in hps) - assert len([hp for hp in hps if hp.tunables and - 'b' in list(zip(*hp.tunables))[0]]) == 1 + assert len([hp for hp in hps if hp.tunables and 'b' in list(zip(*hp.tunables))[0]]) == 1 diff --git a/atm/tests/unit_tests/test_utilities.py b/tests/test_utilities.py similarity index 59% rename from atm/tests/unit_tests/test_utilities.py rename to tests/test_utilities.py index 78a5aac..2d4ce47 100644 --- a/atm/tests/unit_tests/test_utilities.py +++ b/tests/test_utilities.py @@ -1,9 +1,23 @@ import socket import pytest +from btb.selection.selector import Selector from mock import patch from atm import utilities +from atm.constants import SELECTORS_MAP + + +def test_make_selector(): + kwargs = { + 'choices': [1, 2, 3], + 'k': 3, + 'by_algorithm': {'svm': [1, 2], 'rf': [3, 4]} + } + + for selector_class in SELECTORS_MAP.values(): + selector = utilities.get_instance(selector_class, **kwargs) + assert isinstance(selector, Selector) @patch('atm.utilities.requests') @@ -34,11 +48,10 @@ def test_public_ip_success(): pytest.fail("Invalid IP address") -@patch('atm.utilities.requests') -def test_public_ip_fail(requests_mock): +@patch('atm.utilities.requests.get', side_effect=Exception) +def test_public_ip_fail(mock_get): # Set-up utilities.public_ip = None - requests_mock.get.side_effect = Exception # Force fail # run ip = utilities.get_public_ip() @@ -46,4 +59,4 @@ def test_public_ip_fail(requests_mock): # asserts assert ip == utilities.public_ip assert ip == 'localhost' - requests_mock.get.assert_called_once_with(utilities.PUBLIC_IP_URL) + mock_get.assert_called_once_with(utilities.PUBLIC_IP_URL) diff --git a/atm/tests/unit_tests/test_worker.py b/tests/test_worker.py similarity index 90% rename from atm/tests/unit_tests/test_worker.py rename to tests/test_worker.py index 7ee189f..2bc5e30 100644 --- a/atm/tests/unit_tests/test_worker.py +++ b/tests/test_worker.py @@ -4,16 +4,18 @@ import numpy as np import pytest -from btb.selection import BestKVelocity, Selector -from btb.tuning import GP, Tuner +from btb.selection import BestKVelocity +from btb.selection.selector import Selector +from btb.tuning import GP +from btb.tuning.tuner import BaseTuner from mock import ANY, Mock, patch from atm import PROJECT_ROOT +from atm.classifier import Model from atm.config import LogConfig, RunConfig, SQLConfig from atm.constants import METRICS_BINARY, TIME_FMT from atm.database import Database, db_session from atm.enter_data import enter_data -from atm.model import Model from atm.utilities import download_data, load_metrics, load_model from atm.worker import ClassifierError, Worker @@ -115,19 +117,19 @@ def get_new_worker(**kwargs): def test_load_selector_and_tuner(db, dataset): worker = get_new_worker(selector='bestkvel', k_window=7, tuner='gp') - assert type(worker.selector) == BestKVelocity + assert isinstance(worker.selector, BestKVelocity) assert len(worker.selector.choices) == 8 assert worker.selector.k == 7 assert worker.Tuner == GP def test_load_custom_selector_and_tuner(db, dataset): - tuner_path = os.path.join(PROJECT_ROOT, 'tests/utilities/mytuner.py') - selector_path = os.path.join(PROJECT_ROOT, 'tests/utilities/myselector.py') + tuner_path = os.path.join(PROJECT_ROOT, '../tests/utilities/mytuner.py') + selector_path = os.path.join(PROJECT_ROOT, '../tests/utilities/myselector.py') worker = get_new_worker(selector=selector_path + ':MySelector', tuner=tuner_path + ':MyTuner') assert isinstance(worker.selector, Selector) - assert issubclass(worker.Tuner, Tuner) + assert issubclass(worker.Tuner, BaseTuner) def test_select_hyperpartition(worker): @@ -154,16 +156,15 @@ def test_tune_hyperparameters(worker, hyperpartition): mock_tuner = Mock() worker.Tuner = Mock(return_value=mock_tuner) - with patch('atm.worker.vector_to_params') as vtp_mock: + with patch('atm.worker.update_params') as update_params_mock: worker.tune_hyperparameters(hyperpartition) - vtp_mock.assert_called() - - approximate_tunables = [(k, ObjWithAttrs(range=v.range)) - for k, v in hyperpartition.tunables] - worker.Tuner.assert_called_with(tunables=approximate_tunables, - gridding=worker.datarun.gridding, - r_minimum=worker.datarun.r_minimum) - mock_tuner.fit.assert_called() + + update_params_mock.assert_called_once_with( + params=mock_tuner.propose.return_value, + categoricals=hyperpartition.categoricals, + constants=hyperpartition.constants + ) + mock_tuner.propose.assert_called() @@ -174,7 +175,7 @@ def test_test_classifier(db, dataset): model, metrics = worker.test_classifier(method='dt', params=DT_PARAMS) judge_mets = [m[metric] for m in metrics['cv']] - assert type(model) == Model + assert isinstance(model, Model) assert model.judgment_metric == metric assert model.cv_judgment_metric == np.mean(judge_mets) assert model.cv_judgment_metric_stdev == np.std(judge_mets) @@ -197,7 +198,7 @@ def test_save_classifier(db, datarun, model, metrics): clf = db.get_classifier(classifier.id) loaded = load_model(clf, MODEL_DIR) - assert type(loaded) == Model + assert isinstance(loaded, Model) assert loaded.method == model.method assert loaded.random_state == model.random_state diff --git a/atm/tests/utilities/myselector.py b/tests/utilities/myselector.py similarity index 82% rename from atm/tests/utilities/myselector.py rename to tests/utilities/myselector.py index 87a1b12..3a372ba 100644 --- a/atm/tests/utilities/myselector.py +++ b/tests/utilities/myselector.py @@ -1,6 +1,6 @@ import random -from btb.selection import Selector +from btb.selection.selector import Selector class MySelector(Selector): diff --git a/atm/tests/utilities/mytuner.py b/tests/utilities/mytuner.py similarity index 78% rename from atm/tests/utilities/mytuner.py rename to tests/utilities/mytuner.py index e1b0b97..7e047d3 100644 --- a/atm/tests/utilities/mytuner.py +++ b/tests/utilities/mytuner.py @@ -1,10 +1,11 @@ -from btb.tuning import Tuner +from btb.tuning.tuner import BaseTuner -class MyTuner(Tuner): +class MyTuner(BaseTuner): """ Very bare_bones tuner that returns a random set of parameters each time. """ + def propose(self): """ Generate and return a random set of parameters. diff --git a/tox.ini b/tox.ini index 7541b1f..dca9da6 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,10 @@ [tox] -envlist = py27, py35, py36, docs +envlist = py27, py35, py36, docs, lint [travis] python = - 3.6: py36, docs + 3.6: py36, docs, lint 3.5: py35 2.7: py27 @@ -13,24 +13,18 @@ python = setenv = PYTHONPATH = {toxinidir} deps = - -r{toxinidir}/requirements-dev.txt + .[dev] commands = - pip install -U pip - pytest --basetemp={envtmpdir} + /usr/bin/env python setup.py test -# [testenv:flake8] -# deps = -# flake8 -# isort -# commands = -# /usr/bin/env make lint +[testenv:lint] +skipsdist = true +commands = + /usr/bin/env make lint [testenv:docs] -setenv = - PYTHONPATH = {toxinidir} -deps = - -r{toxinidir}/requirements-dev.txt +skipsdist = true commands = /usr/bin/env make docs