From 46282daa2d734f81e879f250613bfdaf5e0ad879 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Wed, 11 Oct 2023 21:16:27 -0700 Subject: [PATCH 01/13] Use `pydantic` > 1 Try un-pinning `pydantic` and allowing it to be pinned by other packages as needed. --- devtools/conda-envs/alchemiscale-client.yml | 2 +- devtools/conda-envs/alchemiscale-compute.yml | 2 +- devtools/conda-envs/alchemiscale-server.yml | 2 +- devtools/conda-envs/test.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/devtools/conda-envs/alchemiscale-client.yml b/devtools/conda-envs/alchemiscale-client.yml index c3abc819..fd8fa25a 100644 --- a/devtools/conda-envs/alchemiscale-client.yml +++ b/devtools/conda-envs/alchemiscale-client.yml @@ -13,7 +13,7 @@ dependencies: - requests - click - httpx - - pydantic<2.0 + - pydantic >1 ## user client printing - rich diff --git a/devtools/conda-envs/alchemiscale-compute.yml b/devtools/conda-envs/alchemiscale-compute.yml index d24f4e1c..38667b3a 100644 --- a/devtools/conda-envs/alchemiscale-compute.yml +++ b/devtools/conda-envs/alchemiscale-compute.yml @@ -13,7 +13,7 @@ dependencies: - requests - click - httpx - - pydantic<2.0 + - pydantic >1 # perses dependencies - openeye-toolkits diff --git a/devtools/conda-envs/alchemiscale-server.yml b/devtools/conda-envs/alchemiscale-server.yml index b0bd1534..9ae5bb02 100644 --- a/devtools/conda-envs/alchemiscale-server.yml +++ b/devtools/conda-envs/alchemiscale-server.yml @@ -12,7 +12,7 @@ dependencies: - openfe=0.13.0 - requests - click - - pydantic<2.0 + - pydantic >1 ## state store - py2neo diff --git a/devtools/conda-envs/test.yml b/devtools/conda-envs/test.yml index 8d09243e..0b1b3a9f 100644 --- a/devtools/conda-envs/test.yml +++ b/devtools/conda-envs/test.yml @@ -10,7 +10,7 @@ dependencies: - numpy<1.24 # https://github.com/numba/numba/issues/8615#issuecomment-1360792615 - networkx - rdkit - - pydantic<2.0 + - pydantic >1 - openff-toolkit - openff-units >=0.2.0 - openff-models >=0.0.4 From 83c97f5042934a39354abcf070123c5c98067b43 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Wed, 18 Oct 2023 21:24:05 -0700 Subject: [PATCH 02/13] Updates to deprecated pydantic decorators --- alchemiscale/models.py | 12 ++++++------ alchemiscale/security/models.py | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/alchemiscale/models.py b/alchemiscale/models.py index ebbe1dbe..ea3da70d 100644 --- a/alchemiscale/models.py +++ b/alchemiscale/models.py @@ -4,7 +4,7 @@ """ from typing import Optional, Union -from pydantic import BaseModel, Field, validator, root_validator +from pydantic import BaseModel, Field, field_validator, model_validator from gufe.tokenization import GufeKey from re import fullmatch @@ -60,19 +60,19 @@ def _validate_component(v, component): return v - @validator("org") + @field_validator("org") def valid_org(cls, v): return cls._validate_component(v, "org") - @validator("campaign") + @field_validator("campaign") def valid_campaign(cls, v): return cls._validate_component(v, "campaign") - @validator("project") + @field_validator("project") def valid_project(cls, v): return cls._validate_component(v, "project") - @root_validator + @model_validator def check_scope_hierarchy(cls, values): if not _hierarchy_valid(values): raise InvalidScopeError( @@ -129,7 +129,7 @@ class ScopedKey(BaseModel): class Config: frozen = True - @validator("gufe_key") + @field_validator("gufe_key") def cast_gufe_key(cls, v): return GufeKey(v) diff --git a/alchemiscale/security/models.py b/alchemiscale/security/models.py index 4f8322d1..f5ab910a 100644 --- a/alchemiscale/security/models.py +++ b/alchemiscale/security/models.py @@ -7,7 +7,7 @@ from datetime import datetime, timedelta from typing import List, Union, Optional -from pydantic import BaseModel, validator +from pydantic import BaseModel, field_validator from ..models import Scope @@ -32,7 +32,7 @@ class ScopedIdentity(BaseModel): disabled: bool = False scopes: List[str] = [] - @validator("scopes", pre=True, each_item=True) + @field_validator("scopes", pre=True, each_item=True) def cast_scopes_to_str(cls, scope): """Ensure that each scope object is correctly cast to its str representation""" if isinstance(scope, Scope): From 9b7670cae9a7f5c2364eb2161cb4ba2e167d36b7 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Thu, 19 Oct 2023 16:46:16 -0700 Subject: [PATCH 03/13] Think this fixes model_validator usage --- alchemiscale/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/alchemiscale/models.py b/alchemiscale/models.py index ea3da70d..fb1870b2 100644 --- a/alchemiscale/models.py +++ b/alchemiscale/models.py @@ -72,7 +72,8 @@ def valid_campaign(cls, v): def valid_project(cls, v): return cls._validate_component(v, "project") - @model_validator + @model_validator(mode='before') + @classmethod def check_scope_hierarchy(cls, values): if not _hierarchy_valid(values): raise InvalidScopeError( From f60979c2bf3980db3567f2432d7b018f10e3af27 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Thu, 19 Oct 2023 17:13:49 -0700 Subject: [PATCH 04/13] More pydantic 2 updates --- alchemiscale/models.py | 13 ++++++++----- alchemiscale/settings.py | 8 ++++---- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/alchemiscale/models.py b/alchemiscale/models.py index fb1870b2..81d384b1 100644 --- a/alchemiscale/models.py +++ b/alchemiscale/models.py @@ -4,7 +4,7 @@ """ from typing import Optional, Union -from pydantic import BaseModel, Field, field_validator, model_validator +from pydantic import BaseModel, Field, field_validator, model_validator, ConfigDict from gufe.tokenization import GufeKey from re import fullmatch @@ -33,8 +33,9 @@ def __eq__(self, other): return str(self) == str(other) - class Config: - frozen = True + model_config: ConfigDict( + frozen = True, + ) @staticmethod def _validate_component(v, component): @@ -127,8 +128,10 @@ class ScopedKey(BaseModel): campaign: str project: str - class Config: - frozen = True + model_config: ConfigDict( + frozen = True, + arbitrary_types_allowed = True + ) @field_validator("gufe_key") def cast_gufe_key(cls, v): diff --git a/alchemiscale/settings.py b/alchemiscale/settings.py index c670a0ba..132767b3 100644 --- a/alchemiscale/settings.py +++ b/alchemiscale/settings.py @@ -7,13 +7,13 @@ from functools import lru_cache from typing import Optional -from pydantic import BaseSettings +from pydantic import BaseSettings, ConfigDict class FrozenSettings(BaseSettings): - class Config: - frozen = True - + model_config: ConfigDict( + frozen = True, + ) class Neo4jStoreSettings(FrozenSettings): """Automatically populates settings from environment variables where they From 6eb5af9d1a694cedd395382afc26255b8a7b6b92 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Thu, 19 Oct 2023 17:16:55 -0700 Subject: [PATCH 05/13] Black! --- alchemiscale/models.py | 11 ++++------- alchemiscale/settings.py | 5 +++-- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/alchemiscale/models.py b/alchemiscale/models.py index 81d384b1..3e256c92 100644 --- a/alchemiscale/models.py +++ b/alchemiscale/models.py @@ -34,8 +34,8 @@ def __eq__(self, other): return str(self) == str(other) model_config: ConfigDict( - frozen = True, - ) + frozen=True, + ) @staticmethod def _validate_component(v, component): @@ -73,7 +73,7 @@ def valid_campaign(cls, v): def valid_project(cls, v): return cls._validate_component(v, "project") - @model_validator(mode='before') + @model_validator(mode="before") @classmethod def check_scope_hierarchy(cls, values): if not _hierarchy_valid(values): @@ -128,10 +128,7 @@ class ScopedKey(BaseModel): campaign: str project: str - model_config: ConfigDict( - frozen = True, - arbitrary_types_allowed = True - ) + model_config: ConfigDict(frozen=True, arbitrary_types_allowed=True) @field_validator("gufe_key") def cast_gufe_key(cls, v): diff --git a/alchemiscale/settings.py b/alchemiscale/settings.py index 132767b3..296fe9ea 100644 --- a/alchemiscale/settings.py +++ b/alchemiscale/settings.py @@ -12,8 +12,9 @@ class FrozenSettings(BaseSettings): model_config: ConfigDict( - frozen = True, - ) + frozen=True, + ) + class Neo4jStoreSettings(FrozenSettings): """Automatically populates settings from environment variables where they From c725fb3ec81c838e4a7ea2ecebec9cffdb111fb7 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Thu, 19 Oct 2023 21:29:39 -0700 Subject: [PATCH 06/13] More pydantic updates... --- alchemiscale/models.py | 4 ++-- alchemiscale/settings.py | 2 +- alchemiscale/storage/models.py | 6 +++++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/alchemiscale/models.py b/alchemiscale/models.py index 3e256c92..871c4d2e 100644 --- a/alchemiscale/models.py +++ b/alchemiscale/models.py @@ -33,7 +33,7 @@ def __eq__(self, other): return str(self) == str(other) - model_config: ConfigDict( + model_config = ConfigDict( frozen=True, ) @@ -128,7 +128,7 @@ class ScopedKey(BaseModel): campaign: str project: str - model_config: ConfigDict(frozen=True, arbitrary_types_allowed=True) + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) @field_validator("gufe_key") def cast_gufe_key(cls, v): diff --git a/alchemiscale/settings.py b/alchemiscale/settings.py index 296fe9ea..325e0011 100644 --- a/alchemiscale/settings.py +++ b/alchemiscale/settings.py @@ -11,7 +11,7 @@ class FrozenSettings(BaseSettings): - model_config: ConfigDict( + model_config = ConfigDict( frozen=True, ) diff --git a/alchemiscale/storage/models.py b/alchemiscale/storage/models.py index 40c7af1f..fdbba768 100644 --- a/alchemiscale/storage/models.py +++ b/alchemiscale/storage/models.py @@ -12,7 +12,7 @@ import hashlib -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, ConfigDict from gufe.tokenization import GufeTokenizable, GufeKey from ..models import ScopedKey, Scope @@ -29,6 +29,8 @@ class ComputeServiceRegistration(BaseModel): registered: datetime heartbeat: datetime + model_config = ConfigDict(arbitrary_types_allowed=True) + def __repr__(self): # pragma: no cover return f"" @@ -59,6 +61,8 @@ class TaskProvenance(BaseModel): datetime_start: datetime datetime_end: datetime + model_config = ConfigDict(arbitrary_types_allowed=True) + # this should include versions of various libraries From a7801b891a301e108dbdd54bfd8df3f1f424a973 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Thu, 19 Oct 2023 22:06:03 -0700 Subject: [PATCH 07/13] Think I finally have tests working again --- alchemiscale/models.py | 1 - alchemiscale/security/models.py | 24 +++++++++++--------- alchemiscale/settings.py | 4 ++-- devtools/conda-envs/alchemiscale-client.yml | 1 + devtools/conda-envs/alchemiscale-compute.yml | 1 + devtools/conda-envs/alchemiscale-server.yml | 1 + devtools/conda-envs/test.yml | 1 + 7 files changed, 19 insertions(+), 14 deletions(-) diff --git a/alchemiscale/models.py b/alchemiscale/models.py index 871c4d2e..6619f824 100644 --- a/alchemiscale/models.py +++ b/alchemiscale/models.py @@ -74,7 +74,6 @@ def valid_project(cls, v): return cls._validate_component(v, "project") @model_validator(mode="before") - @classmethod def check_scope_hierarchy(cls, values): if not _hierarchy_valid(values): raise InvalidScopeError( diff --git a/alchemiscale/security/models.py b/alchemiscale/security/models.py index f5ab910a..6b2fbdc4 100644 --- a/alchemiscale/security/models.py +++ b/alchemiscale/security/models.py @@ -32,20 +32,22 @@ class ScopedIdentity(BaseModel): disabled: bool = False scopes: List[str] = [] - @field_validator("scopes", pre=True, each_item=True) - def cast_scopes_to_str(cls, scope): + @field_validator("scopes") + def cast_scopes_to_str(cls, scopes): """Ensure that each scope object is correctly cast to its str representation""" - if isinstance(scope, Scope): - scope = str(scope) - elif isinstance(scope, str): - try: - Scope.from_str(scope) - except: + scopes_ = [] + for scope in scopes: + if isinstance(scope, Scope): + scopes_.append(str(scope)) + elif isinstance(scope, str): + try: + scopes_.append(Scope.from_str(scope)) + except: + raise ValueError(f"Invalid scope `{scope}` set for `{cls}`") + else: raise ValueError(f"Invalid scope `{scope}` set for `{cls}`") - else: - raise ValueError(f"Invalid scope `{scope}` set for `{cls}`") - return scope + return scopes_ class UserIdentity(ScopedIdentity): diff --git a/alchemiscale/settings.py b/alchemiscale/settings.py index 325e0011..f0974d19 100644 --- a/alchemiscale/settings.py +++ b/alchemiscale/settings.py @@ -7,11 +7,11 @@ from functools import lru_cache from typing import Optional -from pydantic import BaseSettings, ConfigDict +from pydantic_settings import BaseSettings, SettingsConfigDict class FrozenSettings(BaseSettings): - model_config = ConfigDict( + model_config = SettingsConfigDict( frozen=True, ) diff --git a/devtools/conda-envs/alchemiscale-client.yml b/devtools/conda-envs/alchemiscale-client.yml index 799fb643..8decd8e0 100644 --- a/devtools/conda-envs/alchemiscale-client.yml +++ b/devtools/conda-envs/alchemiscale-client.yml @@ -15,6 +15,7 @@ dependencies: - click - httpx - pydantic >1 + - pydantic-settings ## user client printing - rich diff --git a/devtools/conda-envs/alchemiscale-compute.yml b/devtools/conda-envs/alchemiscale-compute.yml index 974af429..0b245d1e 100644 --- a/devtools/conda-envs/alchemiscale-compute.yml +++ b/devtools/conda-envs/alchemiscale-compute.yml @@ -15,6 +15,7 @@ dependencies: - click - httpx - pydantic >1 + - pydantic-settings # perses dependencies - openeye-toolkits diff --git a/devtools/conda-envs/alchemiscale-server.yml b/devtools/conda-envs/alchemiscale-server.yml index c6992805..ab03c7f5 100644 --- a/devtools/conda-envs/alchemiscale-server.yml +++ b/devtools/conda-envs/alchemiscale-server.yml @@ -14,6 +14,7 @@ dependencies: - requests - click - pydantic >1 + - pydantic-settings ## state store - py2neo diff --git a/devtools/conda-envs/test.yml b/devtools/conda-envs/test.yml index 5fe19797..163e230a 100644 --- a/devtools/conda-envs/test.yml +++ b/devtools/conda-envs/test.yml @@ -11,6 +11,7 @@ dependencies: - networkx - rdkit - pydantic >1 + - pydantic-settings - openff-toolkit - openff-units >=0.2.0 - openff-models >=0.0.4 From 435cc448692e830d73f9deb98c91303f786f77e3 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Thu, 19 Oct 2023 22:11:20 -0700 Subject: [PATCH 08/13] Fix broken docs --- docs/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/conf.py b/docs/conf.py index 5a7c3348..a5268ba9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -46,6 +46,7 @@ "passlib", "py2neo", "pydantic", + "pydantic_settings", "starlette", "yaml", ] From 8bac219ad6cc5a68e75ed947ba0f5d9a3d5f4c0b Mon Sep 17 00:00:00 2001 From: David Dotson Date: Wed, 8 Nov 2023 22:27:26 -0700 Subject: [PATCH 09/13] More pydantic fixes; added docstring to authenticate function --- alchemiscale/compute/api.py | 2 +- alchemiscale/models.py | 2 +- alchemiscale/security/auth.py | 25 ++++++++++++++++++++++--- alchemiscale/security/models.py | 4 ++-- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/alchemiscale/compute/api.py b/alchemiscale/compute/api.py index 8679c3a1..50cb1189 100644 --- a/alchemiscale/compute/api.py +++ b/alchemiscale/compute/api.py @@ -107,7 +107,7 @@ def register_computeservice( ): now = datetime.utcnow() csreg = ComputeServiceRegistration( - identifier=compute_service_id, registered=now, heartbeat=now + identifier=ComputeServiceID(compute_service_id), registered=now, heartbeat=now ) compute_service_id_ = n4js.register_computeservice(csreg) diff --git a/alchemiscale/models.py b/alchemiscale/models.py index 6619f824..1427b1a9 100644 --- a/alchemiscale/models.py +++ b/alchemiscale/models.py @@ -122,7 +122,7 @@ class ScopedKey(BaseModel): """ - gufe_key: GufeKey + gufe_key: Union[GufeKey, str] org: str campaign: str project: str diff --git a/alchemiscale/security/auth.py b/alchemiscale/security/auth.py index 16626529..a4dae9c1 100644 --- a/alchemiscale/security/auth.py +++ b/alchemiscale/security/auth.py @@ -25,12 +25,31 @@ def generate_secret_key(): return secrets.token_hex(32) -def authenticate(db, cls, identifier: str, key: str) -> CredentialedEntity: +def authenticate(db, cls, identifier: str, key: str) -> Optional[CredentialedEntity]: + """Authenticate the given identity+key against the db instance. + + Parameters + ---------- + db + State store instance featuring a `get_credentialed_entity` method. + cls + The `CredentialedEntity` subclass the identity corresponds to. + identity + String identifier for the the identity. + key + Secret key string for the identity. + + Returns + ------- + If successfully authenticated, returns the `CredentialedEntity` subclass instance. + If not, returns `None`. + + """ entity: CredentialedEntity = db.get_credentialed_entity(identifier, cls) if entity is None: - return False + return None if not pwd_context.verify(key, entity.hashed_key): - return False + return None return entity diff --git a/alchemiscale/security/models.py b/alchemiscale/security/models.py index 6b2fbdc4..85b0feaf 100644 --- a/alchemiscale/security/models.py +++ b/alchemiscale/security/models.py @@ -30,7 +30,7 @@ class CredentialedEntity(BaseModel): class ScopedIdentity(BaseModel): identifier: str disabled: bool = False - scopes: List[str] = [] + scopes: List[Union[Scope, str]] = [] @field_validator("scopes") def cast_scopes_to_str(cls, scopes): @@ -41,7 +41,7 @@ def cast_scopes_to_str(cls, scopes): scopes_.append(str(scope)) elif isinstance(scope, str): try: - scopes_.append(Scope.from_str(scope)) + scopes_.append(str(Scope.from_str(scope))) except: raise ValueError(f"Invalid scope `{scope}` set for `{cls}`") else: From e3d11e6bd27ed293c9fb80ce2a4b29fe0d5e21be Mon Sep 17 00:00:00 2001 From: David Dotson Date: Wed, 8 Nov 2023 22:36:14 -0700 Subject: [PATCH 10/13] Force newer `openff-models` --- devtools/conda-envs/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devtools/conda-envs/test.yml b/devtools/conda-envs/test.yml index e7698593..c6558659 100644 --- a/devtools/conda-envs/test.yml +++ b/devtools/conda-envs/test.yml @@ -14,7 +14,7 @@ dependencies: - pydantic-settings - openff-toolkit - openff-units >=0.2.0 - - openff-models >=0.0.4 + - openff-models >=0.1.1 - openeye-toolkits - typing-extensions From 23f8b759a41bfc7f03d4662d88c2a76c34db0f82 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Wed, 8 Nov 2023 22:39:59 -0700 Subject: [PATCH 11/13] Update CI to use latest best practices for micromamba Getting weird solves; trying to address this. --- .github/workflows/ci-integration.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-integration.yml b/.github/workflows/ci-integration.yml index e9c4cacb..e5598fbb 100644 --- a/.github/workflows/ci-integration.yml +++ b/.github/workflows/ci-integration.yml @@ -35,14 +35,10 @@ jobs: steps: - uses: actions/checkout@v2 - - uses: conda-incubator/setup-miniconda@v2 + - name: Install environment + uses: mamba-org/setup-micromamba@v1 with: - auto-update-conda: true - use-mamba: true - python-version: ${{ matrix.python-version }} - miniforge-variant: Mambaforge - environment-file: devtools/conda-envs/test.yml - activate-environment: alchemiscale-test + environment-file: devtools/conda-envs/test.yml - name: Decrypt OpenEye license if: ${{ !github.event.pull_request.head.repo.fork }} From fb32353f5e3c09878a2efb31a1797a5c704ec1d4 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Fri, 17 Nov 2023 12:02:31 -0700 Subject: [PATCH 12/13] Fix CI --- .github/workflows/ci-integration.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-integration.yml b/.github/workflows/ci-integration.yml index e5598fbb..6aa47c09 100644 --- a/.github/workflows/ci-integration.yml +++ b/.github/workflows/ci-integration.yml @@ -39,6 +39,9 @@ jobs: uses: mamba-org/setup-micromamba@v1 with: environment-file: devtools/conda-envs/test.yml + create-args: >- + python=${{ matrix.python-version }} + cache-environment: true - name: Decrypt OpenEye license if: ${{ !github.event.pull_request.head.repo.fork }} @@ -53,8 +56,8 @@ jobs: - name: "Environment Information" run: | - mamba info -a - mamba list + conda info -a + conda list - name: "Run tests" run: | From 0a4ccd8a796e18ef63840e74d9eb58c6a5b1cd07 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Mon, 20 Nov 2023 22:54:20 -0700 Subject: [PATCH 13/13] Add pydantic-settings to test env --- devtools/conda-envs/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/devtools/conda-envs/test.yml b/devtools/conda-envs/test.yml index e145b187..b49de8df 100644 --- a/devtools/conda-envs/test.yml +++ b/devtools/conda-envs/test.yml @@ -10,6 +10,7 @@ dependencies: - openfe>=0.14.0 - openmmforcefields>=0.12.0 - pydantic >1 + - pydantic-settings ## state store - neo4j-python-driver