From 0b28083a779e8c82e126ba785904512a59c18a72 Mon Sep 17 00:00:00 2001 From: Jordan Cook Date: Mon, 26 Jul 2021 11:56:04 -0500 Subject: [PATCH 1/2] Update all API request functions to optionally take a Limiter object and a dry_run option --- HISTORY.md | 6 +- docs/conf.py | 2 +- docs/user_guide.md | 76 ++++++++++++++++++++------ poetry.lock | 11 +++- pyinaturalist/__init__.py | 4 +- pyinaturalist/api_requests.py | 53 ++++++++---------- pyinaturalist/docs/forge_utils.py | 25 +++++++-- pyinaturalist/v0/observation_fields.py | 2 +- pyinaturalist/v0/observations.py | 26 ++++----- pyinaturalist/v1/identifications.py | 2 +- pyinaturalist/v1/observations.py | 10 ++-- pyinaturalist/v1/places.py | 4 +- pyinaturalist/v1/posts.py | 2 +- pyinaturalist/v1/projects.py | 2 +- pyinaturalist/v1/search.py | 2 +- pyinaturalist/v1/taxa.py | 4 +- pyinaturalist/v1/users.py | 2 +- pyproject.toml | 10 ++-- test/test_api_requests.py | 7 +++ 19 files changed, 154 insertions(+), 96 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index cda5f1b0..5731f2a5 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,6 +1,10 @@ # History -## 0.14.1 (2021-07-TBD) +## 0.15.0 (2021-TBD) +* Allow all API request functions to accept a `limiter` argument to override rate-limiting settings +* Allow all API request functions to accept a `dry_run` argument to dry-run an individual request + +## 0.14.1 (2021-07-21) * Added new function for **Posts** endpoint: `get_posts()` * Fix broken `response_format` parameter in `v0.get_observations()` diff --git a/docs/conf.py b/docs/conf.py index c98d9c94..429fac88 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -33,7 +33,7 @@ # Add project path so we can import our package sys.path.insert(0, '..') from pyinaturalist import __version__ -from pyinaturalist.constants import DOCS_DIR, PROJECT_DIR, EXAMPLES_DIR, SAMPLE_DATA_DIR +from pyinaturalist.constants import DOCS_DIR, EXAMPLES_DIR, PROJECT_DIR, SAMPLE_DATA_DIR from pyinaturalist.docs.model_docs import document_models # Relevant doc directories used in extension settings diff --git a/docs/user_guide.md b/docs/user_guide.md index d270aec6..145499d7 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -337,42 +337,82 @@ Credentials storage with keyring + KeePassXC While developing and testing, it can be useful to temporarily mock out HTTP requests, especially requests that add, modify, or delete real data. Pyinaturalist has some settings to make this easier. -### Dry-run all requests -To enable dry-run mode, set the `DRY_RUN_ENABLED` variable. When set, requests will not be sent -but will be logged instead: - +### Dry-run individual requests +All API request functions take an optional `dry_run` argument. When set to `True`, requests will not +be sent but will be logged instead: ```python >>> import logging ->>> import pyinaturalist ->>> ->>> # Enable at least INFO-level logging +>>> from pyinaturalist import get_taxa +>>> # Enable at least INFO-level logging to see the request info >>> logging.basicConfig(level='INFO') >>> ->>> pyinaturalist.DRY_RUN_ENABLED = True ->>> get_taxa(q='warbler', locale=1) -{'results': \[\], 'total_results': 0} +>>> get_taxa(q='warbler', locale=1, dry_run=True) +{'results': [], 'total_results': 0} INFO:pyinaturalist.api_requests:Request: GET, https://api.inaturalist.org/v1/taxa, params={'q': 'warbler', 'locale': 1}, headers={'Accept': 'application/json', 'User-Agent': 'Pyinaturalist/0.9.1'} ``` -You can also set this as an environment variable (case-insensitive): +### Dry-run all requests +To enable dry-run mode for all requests, set the `DRY_RUN_ENABLED` environment variable: +:::{tab} Python +```python +>>> import os +>>> os.environ['DRY_RUN_ENABLED'] = 'true' +``` +::: +:::{tab} Unix (MacOS / Linux) ```bash -$ export DRY_RUN_ENABLED=true -$ python my_script.py +export DRY_RUN_ENABLED=true +``` +::: +:::{tab} Windows CMD +```bat +set DRY_RUN_ENABLED="true" ``` +::: +:::{tab} PowerShell +```powershell +$Env:DRY_RUN_ENABLED="true" +``` +::: + +You can also set this as a global variable: + +:::{warning} +This usage is deprecated and will be removed in a future release. +```python +>>> import pyinaturalist +>>> pyinaturalist.DRY_RUN_ENABLED = True +``` +::: ### Dry-run only write requests If you would like to send real `GET` requests but mock out any requests that modify data -(`POST`, `PUT`, `DELETE`, etc.), you can use the `DRY_RUN_WRITE_ONLY` variable -instead: +(`POST`, `PUT`, and `DELETE`), you can use the `DRY_RUN_WRITE_ONLY` variable instead: + +:::{tab} Python ```python ->>> pyinaturalist.DRY_RUN_WRITE_ONLY = True ->>> # Also works as an environment variable >>> import os ->>> os.environ\["DRY_RUN_WRITE_ONLY"\] = 'True' +>>> os.environ['DRY_RUN_WRITE_ONLY'] = 'true' ``` +::: +:::{tab} Unix (MacOS / Linux) +```bash +export DRY_RUN_WRITE_ONLY=true +``` +::: +:::{tab} Windows CMD +```bat +set DRY_RUN_WRITE_ONLY="true" +``` +::: +:::{tab} PowerShell +```powershell +$Env:DRY_RUN_WRITE_ONLY="true" +``` +::: ## User Agent While not mandatory, it's good practice to include a [user-agent](https://en.wikipedia.org/wiki/User_agent) in diff --git a/poetry.lock b/poetry.lock index eac10b56..29aa69fa 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1259,7 +1259,7 @@ python-versions = ">=3.6" [[package]] name = "sphinx" -version = "4.1.1" +version = "4.1.2" description = "Python documentation generator" category = "main" optional = false @@ -1684,6 +1684,11 @@ cffi = [ {file = "cffi-1.14.6-cp27-cp27m-win_amd64.whl", hash = "sha256:7bcac9a2b4fdbed2c16fa5681356d7121ecabf041f18d97ed5b8e0dd38a80224"}, {file = "cffi-1.14.6-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:ed38b924ce794e505647f7c331b22a693bee1538fdf46b0222c4717b42f744e7"}, {file = "cffi-1.14.6-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:e22dcb48709fc51a7b58a927391b23ab37eb3737a98ac4338e2448bef8559b33"}, + {file = "cffi-1.14.6-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:aedb15f0a5a5949ecb129a82b72b19df97bbbca024081ed2ef88bd5c0a610534"}, + {file = "cffi-1.14.6-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:48916e459c54c4a70e52745639f1db524542140433599e13911b2f329834276a"}, + {file = "cffi-1.14.6-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:f627688813d0a4140153ff532537fbe4afea5a3dffce1f9deb7f91f848a832b5"}, + {file = "cffi-1.14.6-cp35-cp35m-win32.whl", hash = "sha256:f0010c6f9d1a4011e429109fda55a225921e3206e7f62a0c22a35344bfd13cca"}, + {file = "cffi-1.14.6-cp35-cp35m-win_amd64.whl", hash = "sha256:57e555a9feb4a8460415f1aac331a2dc833b1115284f7ded7278b54afc5bd218"}, {file = "cffi-1.14.6-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e8c6a99be100371dbb046880e7a282152aa5d6127ae01783e37662ef73850d8f"}, {file = "cffi-1.14.6-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:19ca0dbdeda3b2615421d54bef8985f72af6e0c47082a8d26122adac81a95872"}, {file = "cffi-1.14.6-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:d950695ae4381ecd856bcaf2b1e866720e4ab9a1498cba61c602e56630ca7195"}, @@ -2315,8 +2320,8 @@ soupsieve = [ {file = "soupsieve-2.2.1.tar.gz", hash = "sha256:052774848f448cf19c7e959adf5566904d525f33a3f8b6ba6f6f8f26ec7de0cc"}, ] sphinx = [ - {file = "Sphinx-4.1.1-py3-none-any.whl", hash = "sha256:3d513088236eef51e5b0adb78b0492eb22cc3b8ccdb0b36dd021173b365d4454"}, - {file = "Sphinx-4.1.1.tar.gz", hash = "sha256:23c846a1841af998cb736218539bb86d16f5eb95f5760b1966abcd2d584e62b8"}, + {file = "Sphinx-4.1.2-py3-none-any.whl", hash = "sha256:46d52c6cee13fec44744b8c01ed692c18a640f6910a725cbb938bc36e8d64544"}, + {file = "Sphinx-4.1.2.tar.gz", hash = "sha256:3092d929cd807926d846018f2ace47ba2f3b671b309c7a89cd3306e80c826b13"}, ] sphinx-autobuild = [ {file = "sphinx-autobuild-2021.3.14.tar.gz", hash = "sha256:de1ca3b66e271d2b5b5140c35034c89e47f263f2cd5db302c9217065f7443f05"}, diff --git a/pyinaturalist/__init__.py b/pyinaturalist/__init__.py index 91ae2e73..dd11b18b 100644 --- a/pyinaturalist/__init__.py +++ b/pyinaturalist/__init__.py @@ -1,6 +1,6 @@ # flake8: noqa: F401, F403 -__version__ = '0.14.1' -DEFAULT_USER_AGENT = f'pyinaturalist/{__version__}' +__version__ = '0.15.0' +DEFAULT_USER_AGENT: str = f'pyinaturalist/{__version__}' user_agent = DEFAULT_USER_AGENT # Ignore ImportErrors if this is imported outside a virtualenv diff --git a/pyinaturalist/api_requests.py b/pyinaturalist/api_requests.py index 5ee45280..397eda4e 100644 --- a/pyinaturalist/api_requests.py +++ b/pyinaturalist/api_requests.py @@ -1,4 +1,5 @@ """Some common functions for HTTP requests used by all API modules""" +# TODO: Default retry and backoff settings import threading from contextlib import contextmanager from logging import getLogger @@ -6,6 +7,7 @@ from typing import Dict from unittest.mock import Mock +from pyrate_limiter import Duration, Limiter, RequestRate from requests import Response, Session import pyinaturalist @@ -21,6 +23,13 @@ from pyinaturalist.docs import copy_signature from pyinaturalist.request_params import prepare_request +# Default rate-limiting settings +RATE_LIMITER = Limiter( + RequestRate(REQUESTS_PER_SECOND, Duration.SECOND), + RequestRate(REQUESTS_PER_MINUTE, Duration.MINUTE), + RequestRate(REQUESTS_PER_DAY, Duration.DAY), +) + # Mock response content to return in dry-run mode MOCK_RESPONSE = Mock(spec=Response) MOCK_RESPONSE.json.return_value = {'results': [], 'total_results': 0, 'access_token': ''} @@ -33,29 +42,32 @@ def request( method: str, url: str, access_token: str = None, - user_agent: str = None, - ids: MultiInt = None, + dry_run: bool = False, headers: Dict = None, + ids: MultiInt = None, json: Dict = None, + limiter: Limiter = None, session: Session = None, raise_for_status: bool = True, timeout: float = 5, + user_agent: str = None, **params: RequestParams, ) -> Response: - """Wrapper around :py:func:`requests.request` that supports dry-run mode and rate-limiting, - and adds appropriate headers. + """Wrapper around :py:func:`requests.request` with additional options specific to iNat API requests Args: method: HTTP method url: Request URL access_token: access_token: the access token, as returned by :func:`get_access_token()` - user_agent: A custom user-agent string to provide to the iNaturalist API + dry_run: Just log the request instead of sending a real request ids: One or more integer IDs used as REST resource(s) to request headers: Request headers json: JSON request body - session: Existing Session object to use instead of creating a new one + limiter: Custom rate limits to apply to this request + session: An existing Session object to use instead of creating a new one timeout: Time (in seconds) to wait for a response from the server; if exceeded, a :py:exc:`requests.exceptions.Timeout` will be raised. + user_agent: A custom user-agent string to provide to the iNaturalist API params: All other keyword arguments are interpreted as request parameters Returns: @@ -70,10 +82,11 @@ def request( headers, json, ) + limiter = limiter or RATE_LIMITER session = session or get_session() # Run either real request or mock request depending on settings - if is_dry_run_enabled(method): + if dry_run or is_dry_run_enabled(method): logger.debug('Dry-run mode enabled; mocking request') log_request(method, url, params=params, headers=headers) return MOCK_RESPONSE @@ -89,25 +102,25 @@ def request( @copy_signature(request, exclude='method') def delete(url: str, **kwargs) -> Response: - """Wrapper around :py:func:`requests.delete` that supports dry-run mode and rate-limiting""" + """Wrapper around :py:func:`requests.delete` with additional options specific to iNat API requests""" return request('DELETE', url, **kwargs) @copy_signature(request, exclude='method') def get(url: str, **kwargs) -> Response: - """Wrapper around :py:func:`requests.get` that supports dry-run mode and rate-limiting""" + """Wrapper around :py:func:`requests.get` with additional options specific to iNat API requests""" return request('GET', url, **kwargs) @copy_signature(request, exclude='method') def post(url: str, **kwargs) -> Response: - """Wrapper around :py:func:`requests.post` that supports dry-run mode and rate-limiting""" + """Wrapper around :py:func:`requests.post` with additional options specific to iNat API requests""" return request('POST', url, **kwargs) @copy_signature(request, exclude='method') def put(url: str, **kwargs) -> Response: - """Wrapper around :py:func:`requests.put` that supports dry-run mode and rate-limiting""" + """Wrapper around :py:func:`requests.put` with additional options specific to iNat API requests""" return request('PUT', url, **kwargs) @@ -124,21 +137,6 @@ def ratelimit(bucket=pyinaturalist.user_agent): yield -def get_limiter(): - """Get a rate limiter object, if pyrate-limiter is installed""" - try: - from pyrate_limiter import Duration, Limiter, RequestRate - - requst_rates = [ - RequestRate(REQUESTS_PER_SECOND, Duration.SECOND), - RequestRate(REQUESTS_PER_MINUTE, Duration.MINUTE), - RequestRate(REQUESTS_PER_DAY, Duration.DAY), - ] - return Limiter(*requst_rates) - except ImportError: - return None - - def get_session() -> Session: """Get a Session object that will be reused across requests to take advantage of connection pooling. This is especially relevant for large paginated requests. If used in a multi-threaded @@ -173,6 +171,3 @@ def log_request(*args, **kwargs): """Log all relevant information about an HTTP request""" kwargs_strs = [f'{k}={v}' for k, v in kwargs.items()] logger.info('Request: {}'.format(', '.join(list(args) + kwargs_strs))) - - -RATE_LIMITER = get_limiter() diff --git a/pyinaturalist/docs/forge_utils.py b/pyinaturalist/docs/forge_utils.py index cb73f6ec..e3a2dc3c 100644 --- a/pyinaturalist/docs/forge_utils.py +++ b/pyinaturalist/docs/forge_utils.py @@ -7,6 +7,7 @@ from logging import getLogger from typing import Callable, List, get_type_hints +from pyrate_limiter import Limiter from requests import Session from pyinaturalist.constants import TemplateFunction @@ -14,7 +15,7 @@ logger = getLogger(__name__) -def copy_doc_signature(template_functions: List[TemplateFunction]) -> Callable: +def copy_doc_signature(*template_functions: TemplateFunction, add_common_args: bool = True) -> Callable: """Document a function with docstrings, function signatures, and type annotations from one or more template functions. @@ -50,8 +51,10 @@ def copy_doc_signature(template_functions: List[TemplateFunction]) -> Callable: Args: template_functions: Template functions containing docstrings and params to apply to the wrapped function + add_common_args: Add additional keyword arguments common to most functions """ - template_functions += [_user_agent, _session] + if add_common_args: + template_functions = list(template_functions) + [_dry_run, _limiter, _session, _user_agent] def wrapper(func): # Modify annotations and docstring @@ -159,13 +162,25 @@ def _get_combined_revision(target_function: Callable, template_functions: List[T # Param templates that are added to every function signature by default -def _user_agent(user_agent: str = None): +def _dry_run(dry_run: bool = False): """ - user_agent: A custom user-agent string to provide to the iNaturalist API + dry_run: Just log the request instead of sending a real request + """ + + +def _limiter(limiter: Limiter = None): + """ + limiter: Custom rate limits to apply to this request """ def _session(session: Session = None): """ - session: Allows managing your own `Session object `_ + session: An existing `Session object `_ to use instead of creating a new one + """ + + +def _user_agent(user_agent: str = None): + """ + user_agent: A custom user-agent string to provide to the iNaturalist API """ diff --git a/pyinaturalist/v0/observation_fields.py b/pyinaturalist/v0/observation_fields.py index ead5e47a..adf087d9 100644 --- a/pyinaturalist/v0/observation_fields.py +++ b/pyinaturalist/v0/observation_fields.py @@ -8,7 +8,7 @@ from pyinaturalist.pagination import add_paginate_all -@document_request_params([docs._search_query, docs._pagination]) +@document_request_params(docs._search_query, docs._pagination) @add_paginate_all(method='page') def get_observation_fields(**params) -> JsonResponse: """Search observation fields. Observation fields are basically typed data fields that diff --git a/pyinaturalist/v0/observations.py b/pyinaturalist/v0/observations.py index 19466479..000dc728 100644 --- a/pyinaturalist/v0/observations.py +++ b/pyinaturalist/v0/observations.py @@ -25,12 +25,10 @@ @document_request_params( - [ - docs._observation_common, - docs._observation_rest_only, - docs._bounding_box, - docs._pagination, - ] + docs._observation_common, + docs._observation_rest_only, + docs._bounding_box, + docs._pagination, ) @add_paginate_all(method='page') def get_observations(**params) -> Union[List, str]: @@ -83,8 +81,6 @@ def get_observations(**params) -> Union[List, str]: Return type will be ``dict`` for the ``json`` response format, and ``str`` for all others. """ response_format = params.pop('response_format', 'json') - if response_format == 'geojson': - raise ValueError('For geojson format, use pyinaturalist.v1.get_geojson_observations') if response_format not in OBSERVATION_FORMATS: raise ValueError('Invalid response format') validate_multiple_choice_param(params, 'order_by', REST_OBS_ORDER_BY_PROPERTIES) @@ -99,7 +95,7 @@ def get_observations(**params) -> Union[List, str]: return response.text -@document_request_params([docs._access_token, docs._create_observation]) +@document_request_params(docs._access_token, docs._create_observation) def create_observation(access_token: str, **params) -> ListResponse: """Create a new observation. @@ -155,12 +151,10 @@ def create_observation(access_token: str, **params) -> ListResponse: @document_request_params( - [ - docs._observation_id, - docs._access_token, - docs._create_observation, - docs._update_observation, - ] + docs._observation_id, + docs._access_token, + docs._create_observation, + docs._update_observation, ) def update_observation(observation_id: int, access_token: str, **params) -> ListResponse: """ @@ -332,7 +326,7 @@ def upload_sounds(observation_id: int, sounds: MultiFile, access_token: str, **p return [response.json() for response in responses] -@document_request_params([docs._observation_id, docs._access_token]) +@document_request_params(docs._observation_id, docs._access_token) def delete_observation(observation_id: int, access_token: str = None, **params): """ Delete an observation. diff --git a/pyinaturalist/v1/identifications.py b/pyinaturalist/v1/identifications.py index e3249f19..1ca90321 100644 --- a/pyinaturalist/v1/identifications.py +++ b/pyinaturalist/v1/identifications.py @@ -33,7 +33,7 @@ def get_identifications_by_id(identification_id: MultiInt, **params) -> JsonResp return identifications -@document_request_params([docs._identification_params, docs._pagination, docs._only_id]) +@document_request_params(docs._identification_params, docs._pagination, docs._only_id) @add_paginate_all(method='page') def get_identifications(**params) -> JsonResponse: """Search identifications. diff --git a/pyinaturalist/v1/observations.py b/pyinaturalist/v1/observations.py index a88e9a8c..cf8d9bd0 100644 --- a/pyinaturalist/v1/observations.py +++ b/pyinaturalist/v1/observations.py @@ -50,7 +50,7 @@ def get_observation(observation_id: int, **params) -> JsonResponse: raise ObservationNotFound() -@document_request_params([*docs._get_observations, docs._observation_histogram]) +@document_request_params(*docs._get_observations, docs._observation_histogram) def get_observation_histogram(**params) -> HistogramResponse: """Search observations and return histogram data for the given time interval @@ -104,7 +104,7 @@ def get_observation_histogram(**params) -> HistogramResponse: return convert_histogram(response.json()) -@document_request_params([*docs._get_observations, docs._pagination, docs._only_id]) +@document_request_params(*docs._get_observations, docs._pagination, docs._only_id) @add_paginate_all(method='id') def get_observations(**params) -> JsonResponse: """Search observations. @@ -149,7 +149,7 @@ def get_observations(**params) -> JsonResponse: return observations -@document_request_params([*docs._get_observations, docs._pagination]) +@document_request_params(*docs._get_observations, docs._pagination) @add_paginate_all(method='page') def get_observation_species_counts(**params) -> JsonResponse: """Get all species (or other 'leaf taxa') associated with observations matching the search @@ -178,7 +178,7 @@ def get_observation_species_counts(**params) -> JsonResponse: return response.json() -@document_request_params([*docs._get_observations, docs._pagination]) +@document_request_params(*docs._get_observations, docs._pagination) def get_observation_observers(**params) -> JsonResponse: """Get observers of observations matching the search criteria and the count of observations and distinct taxa of rank species they have observed. @@ -212,7 +212,7 @@ def get_observation_observers(**params) -> JsonResponse: return response.json() -@document_request_params([*docs._get_observations, docs._pagination]) +@document_request_params(*docs._get_observations, docs._pagination) def get_observation_identifiers(**params) -> JsonResponse: """Get identifiers of observations matching the search criteria and the count of observations they have identified. By default, results are sorted by ID count in descending. diff --git a/pyinaturalist/v1/places.py b/pyinaturalist/v1/places.py index 13e52a38..054e1bfe 100644 --- a/pyinaturalist/v1/places.py +++ b/pyinaturalist/v1/places.py @@ -37,7 +37,7 @@ def get_places_by_id(place_id: MultiInt, **params) -> JsonResponse: return places -@document_request_params([docs._bounding_box, docs._name]) +@document_request_params(docs._bounding_box, docs._name) def get_places_nearby(**params) -> JsonResponse: """ Given an bounding box, and an optional name query, return places nearby @@ -75,7 +75,7 @@ def get_places_nearby(**params) -> JsonResponse: return convert_all_place_coordinates(response.json()) -@document_request_params([docs._search_query, docs._pagination]) +@document_request_params(docs._search_query, docs._pagination) @add_paginate_all(method='autocomplete') def get_places_autocomplete(q: str = None, **params) -> JsonResponse: """Given a query string, get places with names starting with the search term diff --git a/pyinaturalist/v1/posts.py b/pyinaturalist/v1/posts.py index 5eac0014..cacc2102 100644 --- a/pyinaturalist/v1/posts.py +++ b/pyinaturalist/v1/posts.py @@ -5,7 +5,7 @@ from pyinaturalist.v1 import get_v1 -@document_request_params([docs._get_posts]) +@document_request_params(docs._get_posts) def get_posts(**params) -> ListResponse: """Search posts. diff --git a/pyinaturalist/v1/projects.py b/pyinaturalist/v1/projects.py index ed3b4c1a..c602cac8 100644 --- a/pyinaturalist/v1/projects.py +++ b/pyinaturalist/v1/projects.py @@ -7,7 +7,7 @@ from pyinaturalist.v1 import get_v1 -@document_request_params([docs._projects_params, docs._pagination]) +@document_request_params(docs._projects_params, docs._pagination) @add_paginate_all(method='page') def get_projects(**params) -> JsonResponse: """Given zero to many of following parameters, get projects matching the search criteria. diff --git a/pyinaturalist/v1/search.py b/pyinaturalist/v1/search.py index 25dd862e..1a2a1c5f 100644 --- a/pyinaturalist/v1/search.py +++ b/pyinaturalist/v1/search.py @@ -5,7 +5,7 @@ from pyinaturalist.v1 import get_v1 -@document_request_params([docs._search_params, docs._pagination]) +@document_request_params(docs._search_params, docs._pagination) def search(q: str, **params) -> JsonResponse: """A unified search endpoint for places, projects, taxa, and/or users diff --git a/pyinaturalist/v1/taxa.py b/pyinaturalist/v1/taxa.py index 1f102db7..e7d069f2 100644 --- a/pyinaturalist/v1/taxa.py +++ b/pyinaturalist/v1/taxa.py @@ -7,7 +7,7 @@ from pyinaturalist.v1 import get_v1 -@document_request_params([docs._taxon_params, docs._taxon_id_params, docs._pagination]) +@document_request_params(docs._taxon_params, docs._taxon_id_params, docs._pagination) @add_paginate_all(method='page') def get_taxa(**params) -> JsonResponse: """Given zero to many of following parameters, get taxa matching the search criteria. @@ -73,7 +73,7 @@ def get_taxa_by_id(taxon_id: MultiInt, **params) -> JsonResponse: return taxa -@document_request_params([docs._taxon_params]) +@document_request_params(docs._taxon_params) def get_taxa_autocomplete(**params) -> JsonResponse: """Given a query string, return taxa with names starting with the search term diff --git a/pyinaturalist/v1/users.py b/pyinaturalist/v1/users.py index 67ce82ee..64cd14a1 100644 --- a/pyinaturalist/v1/users.py +++ b/pyinaturalist/v1/users.py @@ -38,7 +38,7 @@ def get_user_by_id(user_id: int, **params) -> JsonResponse: return convert_generic_timestamps(results[0]) -@document_request_params([docs._search_query, docs._project_id, docs._pagination]) +@document_request_params(docs._search_query, docs._project_id, docs._pagination) def get_users_autocomplete(q: str, **params) -> JsonResponse: """Given a query string, return users with names or logins starting with the search term diff --git a/pyproject.toml b/pyproject.toml index 10e9299b..17b8ac77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pyinaturalist" -version = "0.14.1" +version = "0.15.0" description = "iNaturalist API client for python" authors = ["Nicolas Noé ", "Jordan Cook "] license = "MIT" @@ -28,13 +28,11 @@ include = [ [tool.poetry.dependencies] python = "^3.6" attrs = ">=21.2.0" -python-dateutil = ">=2.0" -requests = ">=2.20" - -# These could potentially be made optional, but for now they're useful enough to include by default keyring = ">=22.3" -python-forge = ">=18.6.0" pyrate-limiter = ">=2.3.3" +python-dateutil = ">=2.0" +python-forge = ">=18.6.0" +requests = ">=2.20" rich = ">=10.0" # Documentation dependencies needed for Readthedocs builds diff --git a/test/test_api_requests.py b/test/test_api_requests.py index 2428d32b..83339aa3 100644 --- a/test/test_api_requests.py +++ b/test/test_api_requests.py @@ -127,6 +127,13 @@ def test_request_dry_run( assert mock_request.call_count == 0 +@patch('pyinaturalist.api_requests.Session.request') +def test_request_dry_run_kwarg(mock_request): + response = request('GET', 'http://url', dry_run=True) + assert response == MOCK_RESPONSE + assert mock_request.call_count == 0 + + # In addition to the test cases above, ensure that the request/response isn't altered with dry-run disabled def test_request_dry_run_disabled(requests_mock): real_response = {'results': ['response object']} From fe460bed84fd1869dde6b80367f3a34fa5f4ab9f Mon Sep 17 00:00:00 2001 From: Jordan Cook Date: Mon, 26 Jul 2021 16:35:57 -0500 Subject: [PATCH 2/2] Add info about rate-limiting to user guide --- docs/user_guide.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/user_guide.md b/docs/user_guide.md index 145499d7..4f1e6365 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -333,6 +333,21 @@ macOS and Linux systems. See this guide for setup info: Credentials storage with keyring + KeePassXC ``` +## Rate Limiting +By default, pyinaturalist applies rate limiting to all requests so they stay within the rates specified by +iNaturalist's [API Recommended Practices](https://www.inaturalist.org/pages/api+recommended+practices). +If you want to customize these rate limits, all API request functions take an optional `limiter` argument, +which takes a [`pyrate_limiter.Limiter`](https://github.com/vutran1710/PyrateLimiter) object. + +For example, to increase the rate to 75 requests per minute: +```python +>>> from pyinaturalist import get_taxa +>>> from pyrate_limiter import Duration, Limiter, RequestRate +>>> +>>> limiter = Limiter(RequestRate(75, Duration.MINUTE)) +>>> get_taxa(q='warbler', locale=1, limiter=limiter) +``` + ## Dry-run mode While developing and testing, it can be useful to temporarily mock out HTTP requests, especially requests that add, modify, or delete real data. Pyinaturalist has some settings to make this easier.