From b69f7bec46e11fa7e8aa32326b59ef0e96376707 Mon Sep 17 00:00:00 2001 From: James Kent Date: Wed, 14 Aug 2024 16:44:43 -0500 Subject: [PATCH 01/10] test commit --- .github/workflows/benchmark.yml | 33 ++++++ asv.conf.json | 202 ++++++++++++++++++++++++++++++++ benchmarks/__init__.py | 0 benchmarks/bench_cbma.py | 24 ++++ benchmarks/benchmarks.py | 31 +++++ 5 files changed, 290 insertions(+) create mode 100644 .github/workflows/benchmark.yml create mode 100644 asv.conf.json create mode 100644 benchmarks/__init__.py create mode 100644 benchmarks/bench_cbma.py create mode 100644 benchmarks/benchmarks.py diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 000000000..b7529797f --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,33 @@ +name: Benchmark + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + benchmark: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install asv + pip install -r requirements.txt + + - name: Run benchmarks + run: | + asv run NEW --show-stderr --python=same + asv compare -s 1.2 NEW diff --git a/asv.conf.json b/asv.conf.json new file mode 100644 index 000000000..0fa0ba300 --- /dev/null +++ b/asv.conf.json @@ -0,0 +1,202 @@ +{ + // The version of the config file format. Do not change, unless + // you know what you are doing. + "version": 1, + + // The name of the project being benchmarked + "project": "project", + + // The project's homepage + "project_url": "http://project-homepage.org/", + + // The URL or local path of the source code repository for the + // project being benchmarked + "repo": ".", + + // The Python project's subdirectory in your repo. If missing or + // the empty string, the project is assumed to be located at the root + // of the repository. + // "repo_subdir": "", + + // Customizable commands for building the project. + // See asv.conf.json documentation. + // To build the package using pyproject.toml (PEP518), uncomment the following lines + // "build_command": [ + // "python -m pip install ." + // ], + // "build": [ + // "pip install -e ." + // ], + // "build_command": [ + // "python -m pip install build", + // "python -m build", + // "python -mpip wheel -w {build_cache_dir} {build_dir}" + // ], + // To build the package using setuptools and a setup.py file, uncomment the following lines + // "build_command": [ + // "python setup.py build", + // "python -mpip wheel -w {build_cache_dir} {build_dir}" + // ], + + // Customizable commands for installing and uninstalling the project. + // See asv.conf.json documentation. + // "install_command": ["in-dir={env_dir} python -mpip install {wheel_file}"], + // "uninstall_command": ["return-code=any python -mpip uninstall -y {project}"], + // "install_command": [ + // "python -m pip install -e .[all]" + // ], + // List of branches to benchmark. If not provided, defaults to "main" + // (for git) or "default" (for mercurial). + // "branches": ["main"], // for git + // "branches": ["default"], // for mercurial + + // The DVCS being used. If not set, it will be automatically + // determined from "repo" by looking at the protocol in the URL + // (if remote), or by looking for special directories, such as + // ".git" (if local). + // "dvcs": "git", + + // The tool to use to create environments. May be "conda", + // "virtualenv", "mamba" (above 3.8) + // or other value depending on the plugins in use. + // If missing or the empty string, the tool will be automatically + // determined by looking for tools on the PATH environment + // variable. + "environment_type": "virtualenv", + + // timeout in seconds for installing any dependencies in environment + // defaults to 10 min + //"install_timeout": 600, + + // the base URL to show a commit for the project. + // "show_commit_url": "http://github.com/owner/project/commit/", + + // The Pythons you'd like to test against. If not provided, defaults + // to the current version of Python used to run `asv`. + // "pythons": ["3.8", "3.12"], + + // The list of conda channel names to be searched for benchmark + // dependency packages in the specified order + // "conda_channels": ["conda-forge", "defaults"], + + // A conda environment file that is used for environment creation. + // "conda_environment_file": "environment.yml", + + // The matrix of dependencies to test. Each key of the "req" + // requirements dictionary is the name of a package (in PyPI) and + // the values are version numbers. An empty list or empty string + // indicates to just test against the default (latest) + // version. null indicates that the package is to not be + // installed. If the package to be tested is only available from + // PyPi, and the 'environment_type' is conda, then you can preface + // the package name by 'pip+', and the package will be installed + // via pip (with all the conda available packages installed first, + // followed by the pip installed packages). + // + // The ``@env`` and ``@env_nobuild`` keys contain the matrix of + // environment variables to pass to build and benchmark commands. + // An environment will be created for every combination of the + // cartesian product of the "@env" variables in this matrix. + // Variables in "@env_nobuild" will be passed to every environment + // during the benchmark phase, but will not trigger creation of + // new environments. A value of ``null`` means that the variable + // will not be set for the current combination. + // + // "matrix": { + // "req": { + // "numpy": ["1.6", "1.7"], + // "six": ["", null], // test with and without six installed + // "pip+emcee": [""] // emcee is only available for install with pip. + // }, + // "env": {"ENV_VAR_1": ["val1", "val2"]}, + // "env_nobuild": {"ENV_VAR_2": ["val3", null]}, + // }, + + // Combinations of libraries/python versions can be excluded/included + // from the set to test. Each entry is a dictionary containing additional + // key-value pairs to include/exclude. + // + // An exclude entry excludes entries where all values match. The + // values are regexps that should match the whole string. + // + // An include entry adds an environment. Only the packages listed + // are installed. The 'python' key is required. The exclude rules + // do not apply to includes. + // + // In addition to package names, the following keys are available: + // + // - python + // Python version, as in the *pythons* variable above. + // - environment_type + // Environment type, as above. + // - sys_platform + // Platform, as in sys.platform. Possible values for the common + // cases: 'linux2', 'win32', 'cygwin', 'darwin'. + // - req + // Required packages + // - env + // Environment variables + // - env_nobuild + // Non-build environment variables + // + // "exclude": [ + // {"python": "3.2", "sys_platform": "win32"}, // skip py3.2 on windows + // {"environment_type": "conda", "req": {"six": null}}, // don't run without six on conda + // {"env": {"ENV_VAR_1": "val2"}}, // skip val2 for ENV_VAR_1 + // ], + // + // "include": [ + // // additional env for python3.12 + // {"python": "3.12", "req": {"numpy": "1.26"}, "env_nobuild": {"FOO": "123"}}, + // // additional env if run on windows+conda + // {"platform": "win32", "environment_type": "conda", "python": "3.12", "req": {"libpython": ""}}, + // ], + + // The directory (relative to the current directory) that benchmarks are + // stored in. If not provided, defaults to "benchmarks" + // "benchmark_dir": "benchmarks", + + // The directory (relative to the current directory) to cache the Python + // environments in. If not provided, defaults to "env" + "env_dir": ".asv/env", + + // The directory (relative to the current directory) that raw benchmark + // results are stored in. If not provided, defaults to "results". + "results_dir": ".asv/results", + + // The directory (relative to the current directory) that the html tree + // should be written to. If not provided, defaults to "html". + "html_dir": ".asv/html", + + // The number of characters to retain in the commit hashes. + // "hash_length": 8, + + // `asv` will cache results of the recent builds in each + // environment, making them faster to install next time. This is + // the number of builds to keep, per environment. + // "build_cache_size": 2, + + // The commits after which the regression search in `asv publish` + // should start looking for regressions. Dictionary whose keys are + // regexps matching to benchmark names, and values corresponding to + // the commit (exclusive) after which to start looking for + // regressions. The default is to start from the first commit + // with results. If the commit is `null`, regression detection is + // skipped for the matching benchmark. + // + // "regressions_first_commits": { + // "some_benchmark": "352cdf", // Consider regressions only after this commit + // "another_benchmark": null, // Skip regression detection altogether + // }, + + // The thresholds for relative change in results, after which `asv + // publish` starts reporting regressions. Dictionary of the same + // form as in ``regressions_first_commits``, with values + // indicating the thresholds. If multiple entries match, the + // maximum is taken. If no entry matches, the default is 5%. + // + // "regressions_thresholds": { + // "some_benchmark": 0.01, // Threshold of 1% + // "another_benchmark": 0.5, // Threshold of 50% + // }, +} diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/benchmarks/bench_cbma.py b/benchmarks/bench_cbma.py new file mode 100644 index 000000000..52fb27be4 --- /dev/null +++ b/benchmarks/bench_cbma.py @@ -0,0 +1,24 @@ +import nimare +from nimare.meta.cbma import ALE, MKDADensity, KDA, MKDAChi2 +from nimare.tests.utils import get_test_data_path + + +class TimeCBMA: + def setup(self): + self.dataset = nimare.dataset.Dataset(get_test_data_path()) + + def time_ale(self): + meta = ALE() + meta.fit(self.dataset) + + def time_mkdadensity(self): + meta = MKDADensity() + meta.fit(self.dataset) + + def time_kda(self): + meta = KDA() + meta.fit(self.dataset) + + def time_mkdachi2(self): + meta = MKDAChi2() + meta.fit(self.dataset) diff --git a/benchmarks/benchmarks.py b/benchmarks/benchmarks.py new file mode 100644 index 000000000..6248a9342 --- /dev/null +++ b/benchmarks/benchmarks.py @@ -0,0 +1,31 @@ +# Write the benchmarking functions here. +# See "Writing benchmarks" in the asv docs for more information. + + +class TimeSuite: + """ + An example benchmark that times the performance of various kinds + of iterating over dictionaries in Python. + """ + def setup(self): + self.d = {} + for x in range(500): + self.d[x] = None + + def time_keys(self): + for key in self.d.keys(): + pass + + def time_values(self): + for value in self.d.values(): + pass + + def time_range(self): + d = self.d + for key in range(500): + d[key] + + +class MemSuite: + def mem_list(self): + return [0] * 256 From 537010b8bfc8df4c5e3e77512950607e871397be Mon Sep 17 00:00:00 2001 From: James Kent Date: Wed, 14 Aug 2024 16:47:23 -0500 Subject: [PATCH 02/10] replace hyphen with underscore --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index ba4d56415..cfde66da1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,7 +6,7 @@ author_email = tsalo006@fiu.edu maintainer = Taylor Salo maintainer_email = tsalo006@fiu.edu description = NiMARE: Neuroimaging Meta-Analysis Research Environment -description-file = README.md +description_file = README.md long_description = NiMARE ====== From 3e6b4791c3c8d34b6bdf488d2ae15b3a8706a7ff Mon Sep 17 00:00:00 2001 From: James Kent Date: Wed, 14 Aug 2024 17:02:52 -0500 Subject: [PATCH 03/10] update benchmark --- .github/workflows/benchmark.yml | 28 +++++++++------------------- benchmarks/bench_cbma.py | 7 ++++--- 2 files changed, 13 insertions(+), 22 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index b7529797f..7c1df4354 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -10,24 +10,14 @@ on: jobs: benchmark: + name: benchmark runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install asv - pip install -r requirements.txt - - - name: Run benchmarks - run: | - asv run NEW --show-stderr --python=same - asv compare -s 1.2 NEW + - uses: actions/setup-python@v2 + with: + python-version: "3.10" + - uses: actions/checkout@v3 + - run: git fetch origin $GITHUB_BASE_REF:base $GITHUB_REF:pr + - run: pip install asv virtualenv + - run: asv machine --yes + - run: asv continuous base pr --strict -e diff --git a/benchmarks/bench_cbma.py b/benchmarks/bench_cbma.py index 52fb27be4..dd6cf83b4 100644 --- a/benchmarks/bench_cbma.py +++ b/benchmarks/bench_cbma.py @@ -1,3 +1,5 @@ +import os + import nimare from nimare.meta.cbma import ALE, MKDADensity, KDA, MKDAChi2 from nimare.tests.utils import get_test_data_path @@ -5,8 +7,7 @@ class TimeCBMA: def setup(self): - self.dataset = nimare.dataset.Dataset(get_test_data_path()) - + self.dataset = nimare.dataset.Dataset(os.path.join(get_test_data_path(), "test_pain_dataset.json")) def time_ale(self): meta = ALE() meta.fit(self.dataset) @@ -21,4 +22,4 @@ def time_kda(self): def time_mkdachi2(self): meta = MKDAChi2() - meta.fit(self.dataset) + meta.fit(self.dataset, self.dataset) From 438dea422faa4278f649440fdd45a3d8a73d7157 Mon Sep 17 00:00:00 2001 From: James Kent Date: Wed, 14 Aug 2024 17:08:06 -0500 Subject: [PATCH 04/10] remove strict --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 7c1df4354..57abf0995 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -20,4 +20,4 @@ jobs: - run: git fetch origin $GITHUB_BASE_REF:base $GITHUB_REF:pr - run: pip install asv virtualenv - run: asv machine --yes - - run: asv continuous base pr --strict -e + - run: asv continuous base pr -e From feec62e8afafd0f6d5d0ce8111426e43b0d70314 Mon Sep 17 00:00:00 2001 From: James Kent Date: Wed, 14 Aug 2024 17:10:45 -0500 Subject: [PATCH 05/10] add python same and show std-err --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 57abf0995..c5c166f42 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -20,4 +20,4 @@ jobs: - run: git fetch origin $GITHUB_BASE_REF:base $GITHUB_REF:pr - run: pip install asv virtualenv - run: asv machine --yes - - run: asv continuous base pr -e + - run: asv continuous base pr -e --python=same --show-stderr From d0349de1a15f13e1d475244385d8c11ea095f14f Mon Sep 17 00:00:00 2001 From: James Kent Date: Wed, 14 Aug 2024 17:14:28 -0500 Subject: [PATCH 06/10] remove same --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c5c166f42..657677156 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -20,4 +20,4 @@ jobs: - run: git fetch origin $GITHUB_BASE_REF:base $GITHUB_REF:pr - run: pip install asv virtualenv - run: asv machine --yes - - run: asv continuous base pr -e --python=same --show-stderr + - run: asv continuous base pr -e --show-stderr From 117c7b9d367a0aa2c76134b40f0b1362b34db611 Mon Sep 17 00:00:00 2001 From: James Kent Date: Wed, 14 Aug 2024 17:14:41 -0500 Subject: [PATCH 07/10] add verbose --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 657677156..b0a2391ba 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -20,4 +20,4 @@ jobs: - run: git fetch origin $GITHUB_BASE_REF:base $GITHUB_REF:pr - run: pip install asv virtualenv - run: asv machine --yes - - run: asv continuous base pr -e --show-stderr + - run: asv continuous base pr -e --show-stderr --verbose From 6f1af113865cbf41ffca5b579caa68f49fdb547e Mon Sep 17 00:00:00 2001 From: James Kent Date: Wed, 14 Aug 2024 17:55:05 -0500 Subject: [PATCH 08/10] install pytest --- asv.conf.json | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/asv.conf.json b/asv.conf.json index 0fa0ba300..7fdd97a54 100644 --- a/asv.conf.json +++ b/asv.conf.json @@ -30,6 +30,7 @@ // "build_command": [ // "python -m pip install build", // "python -m build", + // "python -m pip install -e .[all]", // "python -mpip wheel -w {build_cache_dir} {build_dir}" // ], // To build the package using setuptools and a setup.py file, uncomment the following lines @@ -37,7 +38,13 @@ // "python setup.py build", // "python -mpip wheel -w {build_cache_dir} {build_dir}" // ], - + "matrix": { + "env": { + "dependencies": [ + "pytest" + ] + } + }, // Customizable commands for installing and uninstalling the project. // See asv.conf.json documentation. // "install_command": ["in-dir={env_dir} python -mpip install {wheel_file}"], From e9ae22da579053ca7b6187cbfd5fae2dc31e2316 Mon Sep 17 00:00:00 2001 From: James Kent Date: Wed, 14 Aug 2024 17:55:39 -0500 Subject: [PATCH 09/10] pin asv to 0.6.2 --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index b0a2391ba..ec2cecf5f 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -18,6 +18,6 @@ jobs: python-version: "3.10" - uses: actions/checkout@v3 - run: git fetch origin $GITHUB_BASE_REF:base $GITHUB_REF:pr - - run: pip install asv virtualenv + - run: pip install asv==0.6.2 virtualenv - run: asv machine --yes - run: asv continuous base pr -e --show-stderr --verbose From eab040bd78fe004eac92784f7256a70f0be3267d Mon Sep 17 00:00:00 2001 From: James Kent Date: Wed, 14 Aug 2024 17:59:26 -0500 Subject: [PATCH 10/10] add existing environment --- asv.conf.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/asv.conf.json b/asv.conf.json index 7fdd97a54..4e7978b21 100644 --- a/asv.conf.json +++ b/asv.conf.json @@ -69,7 +69,7 @@ // If missing or the empty string, the tool will be automatically // determined by looking for tools on the PATH environment // variable. - "environment_type": "virtualenv", + "environment_type": "existing", // timeout in seconds for installing any dependencies in environment // defaults to 10 min