From 3d5daf41ede26b2b9cd4a9952ba7777125de59a1 Mon Sep 17 00:00:00 2001 From: Heidi Tong Date: Fri, 14 Jun 2024 10:45:26 -0700 Subject: [PATCH 01/22] v0.9.3 --- CHANGES.md | 4 ++++ src/fmetools/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 79fc4bb..fa331b3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,9 @@ # fmetools changes +## 0.9.3 + +* Update docstrings for FMEEnhancedTransformer and FMEBaseTransformer. + ## 0.9.2 * In `fmetools.scripted_selection.ContainerContentResponse`, diff --git a/src/fmetools/__init__.py b/src/fmetools/__init__.py index 995e98d..050f487 100644 --- a/src/fmetools/__init__.py +++ b/src/fmetools/__init__.py @@ -1,6 +1,6 @@ # coding: utf-8 -__version__ = "0.9.2" +__version__ = "0.9.3" import gettext import os From a0f1cb5f2867d426940b5aec93149c299666470f Mon Sep 17 00:00:00 2001 From: Carson Lam Date: Wed, 10 Jul 2024 13:52:21 -0700 Subject: [PATCH 02/22] Update library metadata to require Python 3.8+. --- setup.cfg | 86 +++++++++++++++++++++++++++---------------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/setup.cfg b/setup.cfg index 33c713d..31ba663 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,43 +1,43 @@ -[metadata] -name = fmetools -version = attr: fmetools.__version__ -author = Safe Software Inc. -description = Tools for extending Safe Software's FME using Python. -long_description = file: README.md, CHANGES.md -long_description_content_type = text/markdown -keywords = FME fmeobjects -url = https://github.com/safesoftware/fmetools -project_urls = - Documentation = https://docs.safe.com/fme/html/fmetools/ -license = BSD -license_file = LICENSE -classifiers = - Programming Language :: Python :: 3 - Programming Language :: Python :: 3.7 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.10 - Programming Language :: Python :: 3.11 - License :: OSI Approved :: BSD License - Intended Audience :: Developers - -[options] -package_dir = - =src -packages = find: -python_requires = >=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.* -include_package_data = True - -[options.packages.find] -where=src - -[options.extras_require] -dev = - ruff>=0.1.5 - pytest - hypothesis - sphinx>=6.0.0 - sphinx-rtd-theme>=1.2.2 - -[bdist_wheel] -universal=1 +[metadata] +name = fmetools +version = attr: fmetools.__version__ +author = Safe Software Inc. +description = Tools for extending Safe Software's FME using Python. +long_description = file: README.md, CHANGES.md +long_description_content_type = text/markdown +keywords = FME fmeobjects +url = https://github.com/safesoftware/fmetools +project_urls = + Documentation = https://docs.safe.com/fme/html/fmetools/ +license = BSD +license_file = LICENSE +classifiers = + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 + License :: OSI Approved :: BSD License + Intended Audience :: Developers + +[options] +package_dir = + =src +packages = find: +python_requires = >=3.8.0 +include_package_data = True + +[options.packages.find] +where=src + +[options.extras_require] +dev = + ruff>=0.1.5 + pytest + hypothesis + sphinx>=6.0.0 + sphinx-rtd-theme>=1.2.2 + +[bdist_wheel] +universal=1 From 88c66f781ef85565352a50bf8408f347ab014d43 Mon Sep 17 00:00:00 2001 From: Carson Lam Date: Wed, 10 Jul 2024 13:56:36 -0700 Subject: [PATCH 03/22] Fix F401, F632. --- src/fmetools/parsers.py | 5 +++-- tests/test_fmehttp.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/fmetools/parsers.py b/src/fmetools/parsers.py index e3f0b02..fcc27c5 100644 --- a/src/fmetools/parsers.py +++ b/src/fmetools/parsers.py @@ -3,12 +3,13 @@ This module contains parsers to support FME reader/writer development. It is not intended for general use. """ + from collections import OrderedDict, namedtuple import fme import six -from fmeobjects import FMEFeature, FMESession -from pluginbuilder import FMEMappingFile +from fmeobjects import FMEFeature, FMESession # noqa F401 +from pluginbuilder import FMEMappingFile # noqa F401 from six import string_types from .utils import string_to_bool diff --git a/tests/test_fmehttp.py b/tests/test_fmehttp.py index 632478e..a9e4457 100644 --- a/tests/test_fmehttp.py +++ b/tests/test_fmehttp.py @@ -146,7 +146,7 @@ def test_generalproxyhandler_use_system_proxy_flag(value): mockSession = MockFMESession(["use-system-proxy", value]) handler = FMEGeneralProxyHandler() handler.configure(mockSession) - assert handler.use_pac == (value is "yes") + assert handler.use_pac == (value == "yes") @pytest.mark.parametrize( From 51e5d7e04ad7b659d1fdee127b59fd9d75d7468b Mon Sep 17 00:00:00 2001 From: Carson Lam Date: Wed, 10 Jul 2024 14:03:15 -0700 Subject: [PATCH 04/22] Apply ruff format. --- fme_env.py | 5 ++++- src/fmetools/features.py | 1 + src/fmetools/guiparams.py | 1 + src/fmetools/http.py | 1 + src/fmetools/localize.py | 1 + src/fmetools/logfile.py | 1 + src/fmetools/paramparsing.py | 1 + src/fmetools/plugins.py | 1 + src/fmetools/scripted_selection.py | 1 + src/fmetools/webservices.py | 1 + tests/test_features.py | 4 +++- 11 files changed, 16 insertions(+), 2 deletions(-) diff --git a/fme_env.py b/fme_env.py index 0cd18f6..4d8e3d1 100644 --- a/fme_env.py +++ b/fme_env.py @@ -1,6 +1,7 @@ """ A CLI script to set up a virtualenv to work with FME. """ + import os import argparse import shutil @@ -31,7 +32,9 @@ raise ValueError("Couldn't find " + site_packages_dir) if not args.fme_home: - raise ValueError("FME_HOME environment variable not defined, so --fme-home must be given") + raise ValueError( + "FME_HOME environment variable not defined, so --fme-home must be given" + ) print("Using FME install dir: " + args.fme_home) if not os.path.isdir(args.fme_home): raise ValueError("FME_HOME is not a directory") diff --git a/src/fmetools/features.py b/src/fmetools/features.py index b3dcc18..544e543 100644 --- a/src/fmetools/features.py +++ b/src/fmetools/features.py @@ -5,6 +5,7 @@ These utilities make it more convenient to work with FMEFeature objects, and help avoid common errors. """ + from __future__ import annotations from typing import Any, Iterable, Optional diff --git a/src/fmetools/guiparams.py b/src/fmetools/guiparams.py index 8bee0b9..ff21053 100644 --- a/src/fmetools/guiparams.py +++ b/src/fmetools/guiparams.py @@ -11,6 +11,7 @@ This module is made available to provide a migration path for existing transformers and advanced use cases. Most developers should instead use FMXJ transformer definitions and :mod:`fmetools.paramparsing`. """ + from collections import namedtuple from typing import Any, Mapping, Union diff --git a/src/fmetools/http.py b/src/fmetools/http.py index b948698..a8e7e15 100644 --- a/src/fmetools/http.py +++ b/src/fmetools/http.py @@ -5,6 +5,7 @@ a drop-in replacement for :class:`requests.Session`. Use this class to make HTTP requests that automatically follow FME settings. """ + import json import logging import os diff --git a/src/fmetools/localize.py b/src/fmetools/localize.py index ac6d42e..77e95f8 100644 --- a/src/fmetools/localize.py +++ b/src/fmetools/localize.py @@ -2,6 +2,7 @@ """ Helpers for enabling localized strings using the :mod:`gettext` module. """ + from __future__ import annotations import os diff --git a/src/fmetools/logfile.py b/src/fmetools/logfile.py index 1c3f9b8..93c101e 100644 --- a/src/fmetools/logfile.py +++ b/src/fmetools/logfile.py @@ -6,6 +6,7 @@ Developers should not need to directly use anything in this module, as the base classes in :mod:`.plugins` include preconfigured loggers. """ + import logging from typing import Optional diff --git a/src/fmetools/paramparsing.py b/src/fmetools/paramparsing.py index f105c51..98fc494 100644 --- a/src/fmetools/paramparsing.py +++ b/src/fmetools/paramparsing.py @@ -11,6 +11,7 @@ Packages using this module and targeting both FME Form and FME Flow should set ``minimum_fme_build`` in its package.yml accordingly. """ + from __future__ import annotations from typing import Any, Iterable, Optional, Union diff --git a/src/fmetools/plugins.py b/src/fmetools/plugins.py index 4df0543..8b36655 100644 --- a/src/fmetools/plugins.py +++ b/src/fmetools/plugins.py @@ -6,6 +6,7 @@ :class:`FMEEnhancedTransformer` is the recommended base class for transformers. Transformer developers should subclass it to implement their own transformers. """ + import logging import warnings from typing import Optional diff --git a/src/fmetools/scripted_selection.py b/src/fmetools/scripted_selection.py index 3cf5941..2f8b1ff 100644 --- a/src/fmetools/scripted_selection.py +++ b/src/fmetools/scripted_selection.py @@ -17,6 +17,7 @@ which includes a reference to the :class:`ScriptedSelectionCallback` implementation above, and other configuration, such as Input Parameters, search support, and more. """ + from abc import ABC, abstractmethod from typing import Any, Dict, List, Optional diff --git a/src/fmetools/webservices.py b/src/fmetools/webservices.py index 7cd1078..353e8a4 100644 --- a/src/fmetools/webservices.py +++ b/src/fmetools/webservices.py @@ -2,6 +2,7 @@ """ Helpers for working with FME Named Connections and FME Web Services. """ + from __future__ import annotations from typing import Union diff --git a/tests/test_features.py b/tests/test_features.py index 72bc281..8bbca1e 100644 --- a/tests/test_features.py +++ b/tests/test_features.py @@ -101,7 +101,9 @@ def test_set_and_get_attributes(attrs): def test_build_feature(): - f = build_feature("feattype", attrs={"foo": "bar"}, geometry=FMEPoint(1, 1, 1), coordsys="LL84") + f = build_feature( + "feattype", attrs={"foo": "bar"}, geometry=FMEPoint(1, 1, 1), coordsys="LL84" + ) assert not f.getSequencedAttributeNames() assert f.getFeatureType() == "feattype" assert isinstance(f.getGeometry(), FMEPoint) From f7e92a5584473f4883b1399b24a49b1f48c3da40 Mon Sep 17 00:00:00 2001 From: Carson Lam Date: Wed, 10 Jul 2024 14:04:22 -0700 Subject: [PATCH 05/22] Pin Creator-based test to v6 to pass under FME 2025. --- tests/test_paramparsing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_paramparsing.py b/tests/test_paramparsing.py index 54d5be0..0a7e739 100644 --- a/tests/test_paramparsing.py +++ b/tests/test_paramparsing.py @@ -19,7 +19,7 @@ @pytest.fixture def creator(): - return TransformerParameterParser("Creator") + return TransformerParameterParser("Creator", version=6) def test_system_transformer(creator): From 8e38a9fd602ea98c031574f647b249d86be2ad5b Mon Sep 17 00:00:00 2001 From: Carson Lam Date: Wed, 10 Jul 2024 14:18:22 -0700 Subject: [PATCH 06/22] In pytest, import untested modules for parsing coverage. --- tests/__init__.py | 0 tests/conftest.py | 4 +++- 2 files changed, 3 insertions(+), 1 deletion(-) delete mode 100644 tests/__init__.py diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/conftest.py b/tests/conftest.py index a982572..60360ce 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,6 @@ -# coding: utf-8 +# Import these modules just for parsing coverage. +# Remove when these modules get imported elsewhere for tests. +from fmetools import localize, scripted_selection, webservices # noqa F401 import json From 134f5f448f12b544a7adb69f86ccb46d797d3707 Mon Sep 17 00:00:00 2001 From: Carson Lam Date: Wed, 10 Jul 2024 14:25:00 -0700 Subject: [PATCH 07/22] Add future annotations import for FA100 and PY38 support. --- src/fmetools/guiparams.py | 2 ++ src/fmetools/http.py | 1 + src/fmetools/logfile.py | 2 ++ src/fmetools/plugins.py | 2 ++ src/fmetools/scripted_selection.py | 2 ++ 5 files changed, 9 insertions(+) diff --git a/src/fmetools/guiparams.py b/src/fmetools/guiparams.py index ff21053..4c4aabb 100644 --- a/src/fmetools/guiparams.py +++ b/src/fmetools/guiparams.py @@ -12,6 +12,8 @@ Most developers should instead use FMXJ transformer definitions and :mod:`fmetools.paramparsing`. """ +from __future__ import annotations + from collections import namedtuple from typing import Any, Mapping, Union diff --git a/src/fmetools/http.py b/src/fmetools/http.py index a8e7e15..9b1e112 100644 --- a/src/fmetools/http.py +++ b/src/fmetools/http.py @@ -6,6 +6,7 @@ Use this class to make HTTP requests that automatically follow FME settings. """ +from __future__ import annotations import json import logging import os diff --git a/src/fmetools/logfile.py b/src/fmetools/logfile.py index 93c101e..8161767 100644 --- a/src/fmetools/logfile.py +++ b/src/fmetools/logfile.py @@ -7,6 +7,8 @@ as the base classes in :mod:`.plugins` include preconfigured loggers. """ +from __future__ import annotations + import logging from typing import Optional diff --git a/src/fmetools/plugins.py b/src/fmetools/plugins.py index 8b36655..fa3191a 100644 --- a/src/fmetools/plugins.py +++ b/src/fmetools/plugins.py @@ -7,6 +7,8 @@ Transformer developers should subclass it to implement their own transformers. """ +from __future__ import annotations + import logging import warnings from typing import Optional diff --git a/src/fmetools/scripted_selection.py b/src/fmetools/scripted_selection.py index 2f8b1ff..2738caf 100644 --- a/src/fmetools/scripted_selection.py +++ b/src/fmetools/scripted_selection.py @@ -18,6 +18,8 @@ and other configuration, such as Input Parameters, search support, and more. """ +from __future__ import annotations + from abc import ABC, abstractmethod from typing import Any, Dict, List, Optional From 21b597a552da5efd1d2b12c064304a43a19a986f Mon Sep 17 00:00:00 2001 From: Carson Lam Date: Wed, 10 Jul 2024 14:30:55 -0700 Subject: [PATCH 08/22] Consolidate tooling around Tox and Ruff. --- .gitignore | 1 + pyproject.toml | 6 ++++++ requirements.txt | 11 ++++++----- setup.cfg | 8 -------- tox.ini | 38 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 51 insertions(+), 13 deletions(-) create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index eb8b697..a079a66 100644 --- a/.gitignore +++ b/.gitignore @@ -65,6 +65,7 @@ instance/ # Sphinx documentation docs/_build/ +_build/ # PyBuilder target/ diff --git a/pyproject.toml b/pyproject.toml index 4f06d8a..771192f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,3 +4,9 @@ addopts = "-s" testpaths = [ "tests", ] + +[tool.ruff] +required-version = '>=0.5.1' +target-version = 'py38' +[tool.ruff.lint] +extend-select = ["FA"] diff --git a/requirements.txt b/requirements.txt index c99aae9..0df5b7e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ -build -pytest -pytest-vcr -hypothesis -ruff +build==1.2.1 +pytest>=8.2.2 +pytest-cov>=5.0.0 +pytest-vcr==1.0.2 +hypothesis==6.105.1 +ruff==0.5.1 diff --git a/setup.cfg b/setup.cfg index 31ba663..06358bb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,13 +31,5 @@ include_package_data = True [options.packages.find] where=src -[options.extras_require] -dev = - ruff>=0.1.5 - pytest - hypothesis - sphinx>=6.0.0 - sphinx-rtd-theme>=1.2.2 - [bdist_wheel] universal=1 diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..5c43cad --- /dev/null +++ b/tox.ini @@ -0,0 +1,38 @@ +[tox] +envlist = py{38,39,310,311,312} +min_version = 4.16.0 +package = editable + +[testenv] +passenv = + FME_HOME +deps = + pytest>=8.2.2 + pytest-cov>=5.0.0 + pytest-vcr>=1.0.2 + hypothesis>=6.105.1 + fme-packager>=1.6.0 +commands = + fme-packager config-env --fme-home '{env:FME_HOME}' + pytest --junitxml test-reports/junit-{envname}.xml --junit-prefix={envname} --cov --cov-append --cov-report xml {posargs} + +[testenv:format] +deps = + ruff>=0.3.2 +skip_install = true +commands = + ruff format --check + +[testenv:check] +deps = + ruff>=0.3.2 +skip_install = true +commands = + ruff check + +[testenv:docs] +deps = + sphinx>=6.0.0 + sphinx-rtd-theme>=1.2.2 +commands = + sphinx-build docs _build From 21c44b7dec6a38757bf68ccf8840d7586fb0e801 Mon Sep 17 00:00:00 2001 From: Carson Lam Date: Wed, 10 Jul 2024 14:39:19 -0700 Subject: [PATCH 09/22] v0.9.4. --- CHANGES.md | 4 ++++ src/fmetools/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index fa331b3..eefb625 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,9 @@ # fmetools changes +## 0.9.4 + +* Fix support for Python 3.8 and require Python 3.8+. + ## 0.9.3 * Update docstrings for FMEEnhancedTransformer and FMEBaseTransformer. diff --git a/src/fmetools/__init__.py b/src/fmetools/__init__.py index 050f487..a5a06b8 100644 --- a/src/fmetools/__init__.py +++ b/src/fmetools/__init__.py @@ -1,6 +1,6 @@ # coding: utf-8 -__version__ = "0.9.3" +__version__ = "0.9.4" import gettext import os From b060f18a4508d3f0c9e76a68eee18e73dfb786af Mon Sep 17 00:00:00 2001 From: Carson Lam Date: Mon, 15 Jul 2024 14:35:02 -0700 Subject: [PATCH 10/22] v0.10.0. --- CHANGES.md | 2 +- src/fmetools/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index eefb625..99f6c0b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,6 @@ # fmetools changes -## 0.9.4 +## 0.10.0 * Fix support for Python 3.8 and require Python 3.8+. diff --git a/src/fmetools/__init__.py b/src/fmetools/__init__.py index a5a06b8..bc5b4a6 100644 --- a/src/fmetools/__init__.py +++ b/src/fmetools/__init__.py @@ -1,6 +1,6 @@ # coding: utf-8 -__version__ = "0.9.4" +__version__ = "0.10.0" import gettext import os From 1d496f4c9dc04a68b6b1572bf81c87aa52823eb9 Mon Sep 17 00:00:00 2001 From: Heidi Tong Date: Mon, 22 Jul 2024 09:47:32 -0700 Subject: [PATCH 11/22] Add deprecated.py --- src/fmetools/deprecated.py | 214 +++++++++++++++++++++++++++++++++++++ 1 file changed, 214 insertions(+) create mode 100644 src/fmetools/deprecated.py diff --git a/src/fmetools/deprecated.py b/src/fmetools/deprecated.py new file mode 100644 index 0000000..8ee3554 --- /dev/null +++ b/src/fmetools/deprecated.py @@ -0,0 +1,214 @@ +# coding: utf-8 +"""This module includes any classes that have been deprecated.""" +from typing import Optional + +from fmeobjects import FMEFeature + + +class FMEBaseTransformer: + """ + In FME 2024.2, this base class was deprecated and replaced with :class:`fme.BaseTransformer`. + + Base class that represents the interface expected by the FME + infrastructure for Python-based transformer implementations. + In particular, this is the class-based API required by the PythonFactory_:: + + FACTORY_DEF {*} PythonFactory + FACTORY_NAME { $(XFORMER_NAME) } + INPUT { FEATURE_TYPE $(XFORMER_NAME)_READY } + SYMBOL_NAME my_library.my_module.TransformerImpl + OUTPUT { PYOUTPUT FEATURE_TYPE $(XFORMER_NAME)_PROCESSED } + + When executed by FME, this is approximately equivalent to:: + + from my_library.my_module import TransformerImpl + transformer = TransformerImpl() + + PythonFactory does not require Python classes to inherit from this base class, + but it expects them to have the same interface as this class. + + This class can be used as a context manager to guarantee that :meth:`close` is called. + This is useful for writing tests. + + .. seealso:: + + PythonFactory_ in the `FME Factory and Function Documentation`_. + + .. _PythonFactory: https://docs.safe.com/fme/html/FME_FactFunc/doc_pages/pythonfactory.txt + .. _FME Factory and Function Documentation: https://docs.safe.com/fme/html/FME_FactFunc/index.html + """ + + def __init__(self): + """ + FME instantiates this class, so it must not require any constructor arguments. + """ + self.factory_name: str = self.__class__.__name__ + """ + This is the ``FACTORY_NAME`` parameter of the PythonFactory_ that instantiated this class. + Defaults to the name of this class. + + .. note:: + Do not modify this property. FME sets the value at runtime. + + .. _PythonFactory: https://docs.safe.com/fme/html/FME_FactFunc/doc_pages/pythonfactory.txt + """ + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + def input(self, feature: FMEFeature) -> None: + """ + Receive a feature from the transformer's single input port. + + This method is used instead of :meth:`input_from` if the transformer has no input tags, + meaning that the transformer's INPUT_TAGS parameter is listed as . + + Transformers typically receive a feature through this method, process it, + modify the feature by adding output attributes, and then output the feature using :meth:`pyoutput`. + However, transformers may output any number of features for each input feature, or none at all. + Transformers may also create new :class:`FMEFeature` instances and output them. + + :param feature: The feature to process. + """ + pass + + def input_from(self, feature: FMEFeature, input_tag: str) -> None: + """ + Receive a feature from input_tag. + + This method is used instead of :meth:`input` if the transformer has defined input tags + listed in the transformer's INPUT_TAGS parameter and the PythonFactory's + PY_INPUT_TAGS clause. Introduced in FME 2024.0. + + Example of a ``PythonFactory`` definition with two input tags:: + + FACTORY_DEF {*} PythonFactory + FACTORY_NAME { $(XFORMER_NAME) } + PY_INPUT_TAGS { INPUT0 INPUT1 } + $(INPUT_LINES) + SYMBOL_NAME { symbol_name } + PY_OUTPUT_TAGS { Output } + OUTPUT { Output FEATURE_TYPE $(OUTPUT_Output_FTYPE) + $(OUTPUT_Output_FUNCS) } + OUTPUT { FEATURE_TYPE $(OUTPUT__FTYPE) + $(OUTPUT__FUNCS) } + + Transformers typically receive a feature through this method, process it, + modify the feature by adding output attributes, and then output the feature using :meth:`pyoutput`. + However, transformers may output any number of features for each input feature, or none at all. + Transformers may also create new :class:`FMEFeature` instances and output them. + + :param feature: The feature to process. + :param input_tag: The input tag that feature came from. + """ + pass + + def process_group(self) -> None: + """ + If group processing is enabled, then this is called after all the + current group's features have been sent to :meth:`input`. + Can be left unimplemented if group processing is not supported. + + :meth:`pyoutput` may be called from this method. + """ + pass + + def close(self) -> None: + """ + Called at the end of translation. + Override this method to perform any necessary cleanup or finalization operations. + + :meth:`pyoutput` may be called from this method. + """ + pass + + def pyoutput(self, feature: FMEFeature, output_tag: Optional[str] = None) -> None: + """ + Output a feature from the transformer to an output tag. If an output tag is specified and does not exist on + the PythonFactory, an error will be raised. + + .. note:: + Do not implement this method. FME injects the implementation at runtime. + + :param feature: The feature to output. + :param output_tag: The output tag to direct feature to. If multiple output tags exist but this argument is not + specified, ``PYOUTPUT`` will be used as a fallback value. If the PythonFactory definition has a single + output tag, this tag will be the default. Introduced in FME 2024.0. + """ + # Stub. Implementation is injected at runtime. + + def total_features_passed_along(self) -> int: + """ + .. note:: + Do not implement this method. FME injects the implementation at runtime. + + :returns: A count of features that have been processed to date, in all groups. + """ + # Stub. Implementation is injected at runtime. + pass + + # noinspection PyMethodMayBeStatic + def has_support_for(self, support_type: int) -> bool: + """ + .. versionadded:: 2022.0 + This method and its corresponding constants in :mod:`fmeobjects`. + + This method is used by FME to check whether the transformer claims support for certain capabilities. + Currently, the only supported type is :data:`fmeobjects.FME_SUPPORT_FEATURE_TABLE_SHIM`, + which determines support for Bulk Mode. + + **Why support Bulk Mode** + + When a transformer supports Bulk Mode, + FME may pass features to :meth:`input` that come from a feature table object. + This allows significant performance gains when processing many features, + but requires the transformer to follow certain rules around how it handles features. + + .. seealso:: `How FME Improves Performance with Bulk Mode `_. + + **How to support Bulk Mode** + + * Features received by :meth:`input` must not be copied or cached for later use. + * Features received by :meth:`input` must not be read or modified after being passed to :meth:`pyoutput`. + * :meth:`pyoutput` should not be given new :class:`FMEFeature` instances. + Doing so will automatically downgrade feature processing to individual mode. + * Override this method. When ``support_type`` is :data:`fmeobjects.FME_SUPPORT_FEATURE_TABLE_SHIM`, + return ``True``. + + Violating these requirements may result in undefined behavior. + + **Illegal Examples** + + *Copy and access later:* :: + + def input(self, feature): + self._cached_features.append(feature) + + def close(self): + for feature in self._cached_features: # not allowed + self.pyoutput(feature) + + *Access after output:* :: + + def input(self, feature): + self.pyoutput(feature) + feature.setAttribute("attr name", "attr val") # not allowed + + *Group-by processing:* :: + + def input(self, feature): + self._cached_features.append(feature) + + def process_group(self): + for feature in self._cached_features: # not allowed + self.pyoutput(feature) + + :param support_type: The type of support to check for. + Currently, the only supported type is :data:`fmeobjects.FME_SUPPORT_FEATURE_TABLE_SHIM`. + :returns: ``True`` if the passed in support type is supported. + The default implementation returns ``False``. + """ + return False From c42fd4dbb890e1efdeacc921b23694240e897ee6 Mon Sep 17 00:00:00 2001 From: Heidi Tong Date: Mon, 22 Jul 2024 09:48:40 -0700 Subject: [PATCH 12/22] Update plugins.py to use fme.BaseTransformer if it exists --- src/fmetools/plugins.py | 217 ++-------------------------------------- 1 file changed, 7 insertions(+), 210 deletions(-) diff --git a/src/fmetools/plugins.py b/src/fmetools/plugins.py index fa3191a..84b7543 100644 --- a/src/fmetools/plugins.py +++ b/src/fmetools/plugins.py @@ -13,6 +13,11 @@ import warnings from typing import Optional +try: + from fme import BaseTransformer as FMEBaseTransformer +except ImportError: # Support < FME 2024.2 + from .deprecated import FMEBaseTransformer + from fmeobjects import FMEFeature try: @@ -28,7 +33,6 @@ # These are relevant externally. # Reader and writer base classes are omitted because they're not intended for general use. __all__ = [ - "FMEBaseTransformer", "FMEEnhancedTransformer", ] @@ -328,216 +332,9 @@ def __exit__(self, *args): self.close() -class FMEBaseTransformer: - """ - Base class that represents the interface expected by the FME - infrastructure for Python-based transformer implementations. - In particular, this is the class-based API required by the PythonFactory_:: - - FACTORY_DEF {*} PythonFactory - FACTORY_NAME { $(XFORMER_NAME) } - INPUT { FEATURE_TYPE $(XFORMER_NAME)_READY } - SYMBOL_NAME my_library.my_module.TransformerImpl - OUTPUT { PYOUTPUT FEATURE_TYPE $(XFORMER_NAME)_PROCESSED } - - When executed by FME, this is approximately equivalent to:: - - from my_library.my_module import TransformerImpl - transformer = TransformerImpl() - - PythonFactory does not require Python classes to inherit from this base class, - but it expects them to have the same interface as this class. - - This class can be used as a context manager to guarantee that :meth:`close` is called. - This is useful for writing tests. - - .. seealso:: - - PythonFactory_ in the `FME Factory and Function Documentation`_. - - .. _PythonFactory: https://docs.safe.com/fme/html/FME_FactFunc/doc_pages/pythonfactory.txt - .. _FME Factory and Function Documentation: https://docs.safe.com/fme/html/FME_FactFunc/index.html - """ - - def __init__(self): - """ - FME instantiates this class, so it must not require any constructor arguments. - """ - self.factory_name: str = self.__class__.__name__ - """ - This is the ``FACTORY_NAME`` parameter of the PythonFactory_ that instantiated this class. - Defaults to the name of this class. - - .. note:: - Do not modify this property. FME sets the value at runtime. - - .. _PythonFactory: https://docs.safe.com/fme/html/FME_FactFunc/doc_pages/pythonfactory.txt - """ - - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - - def input(self, feature: FMEFeature) -> None: - """ - Receive a feature from the transformer's single input port. - - This method is used instead of :meth:`input_from` if the transformer has no input tags, - meaning that the transformer's INPUT_TAGS parameter is listed as . - - Transformers typically receive a feature through this method, process it, - modify the feature by adding output attributes, and then output the feature using :meth:`pyoutput`. - However, transformers may output any number of features for each input feature, or none at all. - Transformers may also create new :class:`FMEFeature` instances and output them. - - :param feature: The feature to process. - """ - pass - - def input_from(self, feature: FMEFeature, input_tag: str) -> None: - """ - Receive a feature from input_tag. - - This method is used instead of :meth:`input` if the transformer has defined input tags - listed in the transformer's INPUT_TAGS parameter and the PythonFactory's - PY_INPUT_TAGS clause. Introduced in FME 2024.0. - - Example of a ``PythonFactory`` definition with two input tags:: - - FACTORY_DEF {*} PythonFactory - FACTORY_NAME { $(XFORMER_NAME) } - PY_INPUT_TAGS { INPUT0 INPUT1 } - $(INPUT_LINES) - SYMBOL_NAME { symbol_name } - PY_OUTPUT_TAGS { Output } - OUTPUT { Output FEATURE_TYPE $(OUTPUT_Output_FTYPE) - $(OUTPUT_Output_FUNCS) } - OUTPUT { FEATURE_TYPE $(OUTPUT__FTYPE) - $(OUTPUT__FUNCS) } - - Transformers typically receive a feature through this method, process it, - modify the feature by adding output attributes, and then output the feature using :meth:`pyoutput`. - However, transformers may output any number of features for each input feature, or none at all. - Transformers may also create new :class:`FMEFeature` instances and output them. - - :param feature: The feature to process. - :param input_tag: The input tag that feature came from. - """ - pass - - def process_group(self) -> None: - """ - If group processing is enabled, then this is called after all the - current group's features have been sent to :meth:`input`. - Can be left unimplemented if group processing is not supported. - - :meth:`pyoutput` may be called from this method. - """ - pass - - def close(self) -> None: - """ - Called at the end of translation. - Override this method to perform any necessary cleanup or finalization operations. - - :meth:`pyoutput` may be called from this method. - """ - pass - - def pyoutput(self, feature: FMEFeature, output_tag: Optional[str] = None) -> None: - """ - Output a feature from the transformer to an output tag. If an output tag is specified and does not exist on - the PythonFactory, an error will be raised. - - .. note:: - Do not implement this method. FME injects the implementation at runtime. - - :param feature: The feature to output. - :param output_tag: The output tag to direct feature to. If multiple output tags exist but this argument is not - specified, ``PYOUTPUT`` will be used as a fallback value. If the PythonFactory definition has a single - output tag, this tag will be the default. Introduced in FME 2024.0. - """ - # Stub. Implementation is injected at runtime. - - def total_features_passed_along(self) -> int: - """ - .. note:: - Do not implement this method. FME injects the implementation at runtime. - - :returns: A count of features that have been processed to date, in all groups. - """ - # Stub. Implementation is injected at runtime. - pass - - # noinspection PyMethodMayBeStatic - def has_support_for(self, support_type: int) -> bool: - """ - .. versionadded:: 2022.0 - This method and its corresponding constants in :mod:`fmeobjects`. - - This method is used by FME to check whether the transformer claims support for certain capabilities. - Currently, the only supported type is :data:`fmeobjects.FME_SUPPORT_FEATURE_TABLE_SHIM`, - which determines support for Bulk Mode. - - **Why support Bulk Mode** - - When a transformer supports Bulk Mode, - FME may pass features to :meth:`input` that come from a feature table object. - This allows significant performance gains when processing many features, - but requires the transformer to follow certain rules around how it handles features. - - .. seealso:: `How FME Improves Performance with Bulk Mode `_. - - **How to support Bulk Mode** - - * Features received by :meth:`input` must not be copied or cached for later use. - * Features received by :meth:`input` must not be read or modified after being passed to :meth:`pyoutput`. - * :meth:`pyoutput` should not be given new :class:`FMEFeature` instances. - Doing so will automatically downgrade feature processing to individual mode. - * Override this method. When ``support_type`` is :data:`fmeobjects.FME_SUPPORT_FEATURE_TABLE_SHIM`, - return ``True``. - - Violating these requirements may result in undefined behavior. - - **Illegal Examples** - - *Copy and access later:* :: - - def input(self, feature): - self._cached_features.append(feature) - - def close(self): - for feature in self._cached_features: # not allowed - self.pyoutput(feature) - - *Access after output:* :: - - def input(self, feature): - self.pyoutput(feature) - feature.setAttribute("attr name", "attr val") # not allowed - - *Group-by processing:* :: - - def input(self, feature): - self._cached_features.append(feature) - - def process_group(self): - for feature in self._cached_features: # not allowed - self.pyoutput(feature) - - :param support_type: The type of support to check for. - Currently, the only supported type is :data:`fmeobjects.FME_SUPPORT_FEATURE_TABLE_SHIM`. - :returns: True if the passed in support type is supported. - The default implementation returns ``False``. - """ - return False - - class FMETransformer(FMEBaseTransformer): def __init__(self): - super(FMEBaseTransformer, self).__init__() + super().__init__() warnings.warn( "Avoid confusion with fmeobjects.FMETransformer", DeprecationWarning ) @@ -783,7 +580,7 @@ def has_support_for(self, support_type: int) -> bool: """ Overrides the default implementation to report support for Bulk Mode. - :returns: True if ``support_type`` is :data:`fmeobjects.FME_SUPPORT_FEATURE_TABLE_SHIM`. + :returns: ``True`` if ``support_type`` is :data:`fmeobjects.FME_SUPPORT_FEATURE_TABLE_SHIM`. See :meth:`FMEBaseTransformer.has_support_for` for more details. """ if support_type == FME_SUPPORT_FEATURE_TABLE_SHIM: From ac88019d0d85d283c1a2879da2ae75e5745e83cb Mon Sep 17 00:00:00 2001 From: Heidi Tong Date: Mon, 22 Jul 2024 13:51:01 -0700 Subject: [PATCH 13/22] Rename deprecated.py to _deprecated.py --- src/fmetools/{deprecated.py => _deprecated.py} | 0 src/fmetools/plugins.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename src/fmetools/{deprecated.py => _deprecated.py} (100%) diff --git a/src/fmetools/deprecated.py b/src/fmetools/_deprecated.py similarity index 100% rename from src/fmetools/deprecated.py rename to src/fmetools/_deprecated.py diff --git a/src/fmetools/plugins.py b/src/fmetools/plugins.py index 84b7543..ca14315 100644 --- a/src/fmetools/plugins.py +++ b/src/fmetools/plugins.py @@ -16,7 +16,7 @@ try: from fme import BaseTransformer as FMEBaseTransformer except ImportError: # Support < FME 2024.2 - from .deprecated import FMEBaseTransformer + from ._deprecated import FMEBaseTransformer from fmeobjects import FMEFeature From 6c317244ca98b57073581c7af641236503998e71 Mon Sep 17 00:00:00 2001 From: Heidi Tong Date: Mon, 22 Jul 2024 09:51:43 -0700 Subject: [PATCH 14/22] v0.10.1 --- CHANGES.md | 5 +++++ src/fmetools/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 99f6c0b..74c66e7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,10 @@ # fmetools changes +## 0.10.1 + +* Use `fme.BaseTransformer` if it exists. +* Deprecate `fmetools.plugins.FMEBaseTransformer`. + ## 0.10.0 * Fix support for Python 3.8 and require Python 3.8+. diff --git a/src/fmetools/__init__.py b/src/fmetools/__init__.py index bc5b4a6..d8bd942 100644 --- a/src/fmetools/__init__.py +++ b/src/fmetools/__init__.py @@ -1,6 +1,6 @@ # coding: utf-8 -__version__ = "0.10.0" +__version__ = "0.10.1" import gettext import os From 0cbd738e4c08e1d3f635f75efd268aab670257f2 Mon Sep 17 00:00:00 2001 From: Heidi Tong Date: Tue, 30 Jul 2024 13:07:22 -0700 Subject: [PATCH 15/22] Update docstrings in plugins.py --- src/fmetools/plugins.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/fmetools/plugins.py b/src/fmetools/plugins.py index ca14315..66d28de 100644 --- a/src/fmetools/plugins.py +++ b/src/fmetools/plugins.py @@ -333,6 +333,10 @@ def __exit__(self, *args): class FMETransformer(FMEBaseTransformer): + """ + In FME versions prior to 2024.2, this will subclass :class:`fmetools._deprecated.FMEBaseTransformer` + instead. + """ def __init__(self): super().__init__() warnings.warn( @@ -347,6 +351,9 @@ class FMEEnhancedTransformer(FMEBaseTransformer): and adds methods to introduce more granularity to the transformer lifecycle to help developers organize their code. + In FME versions prior to 2024.2, this will subclass :class:`fmetools._deprecated.FMEBaseTransformer` + instead. + :meth:`input` is broken down to these methods for implementations to overwrite: - :meth:`pre_input` - :meth:`setup` which is only called for the first input feature @@ -376,7 +383,7 @@ class FMEEnhancedTransformer(FMEBaseTransformer): This class overrides :meth:`has_support_for` to return ``True`` for Bulk Mode support. This means that the transformer cannot cache or copy features for later use, and cannot output new :class:`fmeobjects.FMEFeature` instances. - See :meth:`FMEBaseTransformer.has_support_for` for details about these restrictions. + See :meth:`fme.BaseTransformer.has_support_for` for details about these restrictions. """ def __init__(self): @@ -581,7 +588,7 @@ def has_support_for(self, support_type: int) -> bool: Overrides the default implementation to report support for Bulk Mode. :returns: ``True`` if ``support_type`` is :data:`fmeobjects.FME_SUPPORT_FEATURE_TABLE_SHIM`. - See :meth:`FMEBaseTransformer.has_support_for` for more details. + See :meth:`fme.BaseTransformer.has_support_for` for more details. """ if support_type == FME_SUPPORT_FEATURE_TABLE_SHIM: return True From 036db186c7861da69625eef5dba212407dae4031 Mon Sep 17 00:00:00 2001 From: Heidi Tong Date: Tue, 30 Jul 2024 14:10:38 -0700 Subject: [PATCH 16/22] Add fmetools._deprecated.FMEBaseTransformer to plugins.rst --- docs/fmetools/plugins.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/fmetools/plugins.rst b/docs/fmetools/plugins.rst index a81fad5..f4b3ecd 100644 --- a/docs/fmetools/plugins.rst +++ b/docs/fmetools/plugins.rst @@ -5,3 +5,8 @@ fmetools.plugins :members: :private-members: :member-order: groupwise + +.. autoclass:: fmetools._deprecated.FMEBaseTransformer + :members: + :private-members: + :member-order: groupwise From 1a26d981d54741298dea544e8d168c5419591571 Mon Sep 17 00:00:00 2001 From: Heidi Tong Date: Tue, 30 Jul 2024 14:11:08 -0700 Subject: [PATCH 17/22] Link to deprecated FMEBaseTransformer --- src/fmetools/plugins.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fmetools/plugins.py b/src/fmetools/plugins.py index 66d28de..856f2aa 100644 --- a/src/fmetools/plugins.py +++ b/src/fmetools/plugins.py @@ -383,7 +383,7 @@ class FMEEnhancedTransformer(FMEBaseTransformer): This class overrides :meth:`has_support_for` to return ``True`` for Bulk Mode support. This means that the transformer cannot cache or copy features for later use, and cannot output new :class:`fmeobjects.FMEFeature` instances. - See :meth:`fme.BaseTransformer.has_support_for` for details about these restrictions. + See :meth:`fmetools._deprecated.FMEBaseTransformer.has_support_for` for details about these restrictions. """ def __init__(self): @@ -588,7 +588,7 @@ def has_support_for(self, support_type: int) -> bool: Overrides the default implementation to report support for Bulk Mode. :returns: ``True`` if ``support_type`` is :data:`fmeobjects.FME_SUPPORT_FEATURE_TABLE_SHIM`. - See :meth:`fme.BaseTransformer.has_support_for` for more details. + See :meth:`fmetools._deprecated.FMEBaseTransformer.has_support_for` for more details. """ if support_type == FME_SUPPORT_FEATURE_TABLE_SHIM: return True From d03c452282cb9196b2874b3ff508efcc0fea485c Mon Sep 17 00:00:00 2001 From: Heather Li Date: Tue, 6 Aug 2024 17:03:04 -0700 Subject: [PATCH 18/22] add read schema generators --- src/fmetools/plugins.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/fmetools/plugins.py b/src/fmetools/plugins.py index 856f2aa..f5abcc5 100644 --- a/src/fmetools/plugins.py +++ b/src/fmetools/plugins.py @@ -80,6 +80,9 @@ def __init__(self, reader_type_name, reader_keyword, mapping_file): self._aborted = False self._feature_types = [] + + self._list_feature_types = None + self._readSchema_generator, self._read_generator = None, None @property @@ -174,6 +177,39 @@ def setConstraints(self, feature): self._read_generator.close() self._read_generator = None + + def _feature_types_generator(self): + """ + Lists the names of feature types + """ + pass + + def _schema_features_generator(self): + """ + Generate schema features. + + When self._feature_types is empty, schema features for all possible feature types should be generated + Otherwise, a single schema feature (where attribute names correspond to names and attribute values correspond to mapped attribute types) + should be generated. + """ + pass + + + def readSchema(self): + """ + Creates schema features + """ + # pylint: disable=invalid-name + if not self._readSchema_generator: + if self._list_feature_types: + self._readSchema_generator = self._feature_types_generator() + else: + self._readSchema_generator = self._schema_features_generator() + try: + return next(self._readSchema_generator) + except StopIteration: + return None + def readSchemaGenerator(self): """ Generator form of :meth:`readSchema`. From 09d769201fa3e63c021d83f2d5d7b2e29bdfaeb2 Mon Sep 17 00:00:00 2001 From: Heather Li Date: Wed, 7 Aug 2024 12:39:50 -0700 Subject: [PATCH 19/22] add reader generator --- src/fmetools/plugins.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/fmetools/plugins.py b/src/fmetools/plugins.py index f5abcc5..10cc458 100644 --- a/src/fmetools/plugins.py +++ b/src/fmetools/plugins.py @@ -15,7 +15,7 @@ try: from fme import BaseTransformer as FMEBaseTransformer -except ImportError: # Support < FME 2024.2 +except ImportError: # Support < FME 2024.2 from ._deprecated import FMEBaseTransformer from fmeobjects import FMEFeature @@ -80,7 +80,6 @@ def __init__(self, reader_type_name, reader_keyword, mapping_file): self._aborted = False self._feature_types = [] - self._list_feature_types = None self._readSchema_generator, self._read_generator = None, None @@ -177,7 +176,6 @@ def setConstraints(self, feature): self._read_generator.close() self._read_generator = None - def _feature_types_generator(self): """ Lists the names of feature types @@ -224,6 +222,26 @@ def readSchemaGenerator(self): else: break + def _read_features_generator(self): + """ + Creates features for a feature type + """ + pass + + def read(self): + """ + Creates features for a feature type + """ + # pylint: disable=invalid-name + if not self._read_generator: + self._read_generator = self._read_features_generator() + + try: + return next(self._read_generator) + except StopIteration: + return None + + def readGenerator(self): """ Generator form of :meth:`read`. From c003d52c39bb6862c34ccb3a2f78eefc68c185dc Mon Sep 17 00:00:00 2001 From: Heather Li Date: Wed, 7 Aug 2024 16:43:01 -0700 Subject: [PATCH 20/22] touch up generator docstrings --- src/fmetools/plugins.py | 43 ++++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/src/fmetools/plugins.py b/src/fmetools/plugins.py index 10cc458..0ccdec8 100644 --- a/src/fmetools/plugins.py +++ b/src/fmetools/plugins.py @@ -54,6 +54,8 @@ class FMESimplifiedReader(FMEReader): :ivar bool _using_constraints: True if :meth:`setConstraints` was called. :ivar bool _aborted: True if :meth:`abort` was called. :ivar list[str] _feature_types: Ordered list of feature type names. + :ivar bool _list_feature_types: True if the reader was launched to produce + a list of feature types. :ivar _readSchema_generator: Use this member to store any generator used for :meth:`readSchema`. Doing so means it'll be explicitly closed for you in :meth:`close`. @@ -80,7 +82,7 @@ def __init__(self, reader_type_name, reader_keyword, mapping_file): self._aborted = False self._feature_types = [] - self._list_feature_types = None + self._list_feature_types = False self._readSchema_generator, self._read_generator = None, None @@ -178,24 +180,38 @@ def setConstraints(self, feature): def _feature_types_generator(self): """ - Lists the names of feature types + A generator which produces features for each potential feature type from + the reader's dataset. + + The feature types will populate the list displayed by the GUI Type FEATURE_TYPES. + + Must yield FMEFeatures with the feature type set. + Only the feature type is required; feature attributes will be ignored. """ pass def _schema_features_generator(self): """ - Generate schema features. + A generator which produces schema features for all requested feature types. + + When `self._feature_types` is empty, schema features for all possible + feature types should be generated. Otherwise, a single schema feature + should be generated for each feature type in `self._feature_types`. - When self._feature_types is empty, schema features for all possible feature types should be generated - Otherwise, a single schema feature (where attribute names correspond to names and attribute values correspond to mapped attribute types) - should be generated. + The function :meth:`features.build_feature` should be used to create schema features. + Schema features must contain the feature type, all possible geometry + types for the feature type, and exposed attributes for the feature. + The attribute value for a schema attribute should be set to the expected + format attribute type. """ pass - def readSchema(self): """ - Creates schema features + Creates schema features. + + Implementations should override :meth:`_feature_types_generator` + and :meth:`_schema_features_generator` instead of this method. """ # pylint: disable=invalid-name if not self._readSchema_generator: @@ -224,13 +240,18 @@ def readSchemaGenerator(self): def _read_features_generator(self): """ - Creates features for a feature type + Generator which yields data features for all requested feature types. + + The function :meth:`features.build_feature` should be used to create data features. """ pass def read(self): """ - Creates features for a feature type + Creates features for a feature type. + + Implementations should override :meth:`_read_features_generator` + instead of this method. """ # pylint: disable=invalid-name if not self._read_generator: @@ -241,7 +262,6 @@ def read(self): except StopIteration: return None - def readGenerator(self): """ Generator form of :meth:`read`. @@ -391,6 +411,7 @@ class FMETransformer(FMEBaseTransformer): In FME versions prior to 2024.2, this will subclass :class:`fmetools._deprecated.FMEBaseTransformer` instead. """ + def __init__(self): super().__init__() warnings.warn( From 5bea7970fdb6fa764850181bef3b29730a98d255 Mon Sep 17 00:00:00 2001 From: Heather Li Date: Tue, 13 Aug 2024 13:17:14 -0700 Subject: [PATCH 21/22] set _list_feature_types flag during open --- src/fmetools/plugins.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/fmetools/plugins.py b/src/fmetools/plugins.py index 0ccdec8..bd75920 100644 --- a/src/fmetools/plugins.py +++ b/src/fmetools/plugins.py @@ -154,6 +154,10 @@ def open(self, dataset_name, parameters): if open_parameters.get("FME_DEBUG"): self._debug = True + self._list_feature_types = self._mapping_file.get_flag( + "RETRIEVE_ALL_TABLE_NAMES" + ) + return self.enhancedOpen(open_parameters) def enhancedOpen(self, open_parameters): From e51ed5f1b97005556b124d107de48c1c28e2a69f Mon Sep 17 00:00:00 2001 From: Heather Li Date: Tue, 13 Aug 2024 16:33:45 -0700 Subject: [PATCH 22/22] add return type hints to reader methods --- src/fmetools/plugins.py | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/src/fmetools/plugins.py b/src/fmetools/plugins.py index bd75920..e7fa961 100644 --- a/src/fmetools/plugins.py +++ b/src/fmetools/plugins.py @@ -11,7 +11,7 @@ import logging import warnings -from typing import Optional +from typing import Optional, Generator try: from fme import BaseTransformer as FMEBaseTransformer @@ -87,12 +87,8 @@ def __init__(self, reader_type_name, reader_keyword, mapping_file): self._readSchema_generator, self._read_generator = None, None @property - def log(self): - """ - Provides access to the FME log. - - :rtype: logging.LoggerAdapter - """ + def log(self) -> logging.LoggerAdapter: + """Provides access to the FME log.""" if not self._log: # Instantiate a logger with the appropriate debug mode. self._log = get_configured_logger(self.__class__.__name__, self._debug) @@ -111,7 +107,7 @@ def debug(self, new_debug): self._debug = new_debug self._log = get_configured_logger(self.__class__.__name__, self._debug) - def hasSupportFor(self, support_type): + def hasSupportFor(self, support_type) -> bool: """ Return whether this reader supports a certain type. Currently, the only supported type is fmeobjects.FME_SUPPORT_FEATURE_TABLE_SHIM. @@ -129,7 +125,7 @@ def hasSupportFor(self, support_type): """ return False - def open(self, dataset_name, parameters): + def open(self, dataset_name, parameters) -> None: """Open the dataset for reading. Does these things for you: @@ -160,7 +156,7 @@ def open(self, dataset_name, parameters): return self.enhancedOpen(open_parameters) - def enhancedOpen(self, open_parameters): + def enhancedOpen(self, open_parameters) -> None: """ Implementations shall override this method instead of :meth:`open`. @@ -168,7 +164,7 @@ def enhancedOpen(self, open_parameters): """ pass - def setConstraints(self, feature): + def setConstraints(self, feature) -> None: """ Reset any existing feature generator that represents the state for :meth:`read`. @@ -182,7 +178,7 @@ def setConstraints(self, feature): self._read_generator.close() self._read_generator = None - def _feature_types_generator(self): + def _feature_types_generator(self) -> Generator[FMEFeature]: """ A generator which produces features for each potential feature type from the reader's dataset. @@ -194,7 +190,7 @@ def _feature_types_generator(self): """ pass - def _schema_features_generator(self): + def _schema_features_generator(self) -> Generator[FMEFeature]: """ A generator which produces schema features for all requested feature types. @@ -210,7 +206,7 @@ def _schema_features_generator(self): """ pass - def readSchema(self): + def readSchema(self) -> Optional[FMEFeature]: """ Creates schema features. @@ -228,7 +224,7 @@ def readSchema(self): except StopIteration: return None - def readSchemaGenerator(self): + def readSchemaGenerator(self) -> Generator[FMEFeature]: """ Generator form of :meth:`readSchema`. @@ -242,7 +238,7 @@ def readSchemaGenerator(self): else: break - def _read_features_generator(self): + def _read_features_generator(self) -> Generator[FMEFeature]: """ Generator which yields data features for all requested feature types. @@ -250,7 +246,7 @@ def _read_features_generator(self): """ pass - def read(self): + def read(self) -> Optional[FMEFeature]: """ Creates features for a feature type. @@ -266,7 +262,7 @@ def read(self): except StopIteration: return None - def readGenerator(self): + def readGenerator(self) -> Generator[FMEFeature]: """ Generator form of :meth:`read`. @@ -280,10 +276,10 @@ def readGenerator(self): else: break - def abort(self): + def abort(self) -> None: self._aborted = True - def close(self): + def close(self) -> None: """ This default implementation closes any existing read generators. """