Test lower bounds specified in main dependencies (#70)

* Save work

* Test torch

* Update tests for setup.cfg

* Add tests for lower

* Fix tests

* Add installation

* Remove notebook

* Add interface tests and run pre-commit

* Add tests for errors

* Better option for skipping setup

* Add TODO

* Fix mypy issues

* Fix flake8

* Add docs

* Add example to clarify docstring

* Rework loop

* Ignore type

* Refactor to simplify a little bit

* Fix docstring

docs/source/usage.rst

@@ -457,6 +457,90 @@ For instance, ``snowflake-connector-python[pandas]>=2.2.8,<2.3.9`` might be repl
                 --deps scikit-learn \
                 --export
 
+Testing lower bounds of packages
+--------------------------------
+
+.. warning::
+
+    This feature is experimental and may change in future releases. Limited functionality is available.
+
+Suppose you want to test if a new feature is compatible with dependencies' lower bounds.
+You can use an edgetest environment to test the lower bounds of dependencies in your ``setup.cfg`` or ``pyproject.toml`` as follows:
+
+.. tabs::
+
+    .. tab:: .cfg
+
+        .. code-block:: ini
+
+            [edgetest.envs.pandas_lower]
+            lower =
+                pandas
+
+    .. tab:: .toml
+
+        .. code-block:: toml
+
+            [edgetest.envs.pandas_lower]
+            lower = [
+                "pandas"
+            ]
+
+Running the edgetest command using this environment specification will:
+
+1. Create your virtual environment (as in the standard case),
+2. Install the local package,
+3. Install the lower version of ``pandas`` as specified in your project configuration file, and
+4. Run the ``pytest`` in the environment.
+
+For example, if your project configuration looks like this:
+
+.. tabs::
+
+    .. tab:: .cfg
+
+        .. code-block:: ini
+
+            [options]
+            install_requires =
+                pandas>=0.24.3
+
+            [edgetest.envs.pandas_lower]
+            lower =
+                pandas
+
+    .. tab:: .toml
+
+        .. code-block:: toml
+
+            [project]
+            dependencies = [
+                "pandas>=0.24.3",
+            ]
+
+            [edgetest.envs.pandas_lower]
+            lower =
+                pandas
+
+then edgetest will:
+
+1. Create the ``pandas_lower`` virtual environment,
+2. Install the local package,
+3. Install ``pandas==0.24.3``, and
+4. Run the ``pytest`` in the environment.
+
+Testing lower bounds of dependencies currently has the following limitations:
+
+- Can only parse main dependencies. Edgetest cannot parse lower bounds specified in extras.
+- Can only install lower bounds of packages specified with ``>=``. Edgetest cannot test the lower bounds of packages specified with ``>``.
+- Cannot automatically update or export lower bounds of dependencies to a project configuration file.
+
+.. note::
+    It is hard to install lower bounds of dependencies if they are very outdated.
+    Older versions of dependencies like pandas may be incompatible with newer Python versions
+    or may not be able to be installed with a given numpy version. In the case where installation
+    is not possible, edgetest will report that the setup was unsuccessful and move on to the next environment.
+
 Using plugins
 -------------
 

edgetest/core.py

@@ -25,6 +25,9 @@ class TestPackage:
         The name of the conda environment to create/use.
     upgrade : list
         The list of packages to upgrade
+    lower : list
+        The list of packages to install lower bounds alongside the lower bound value.
+        E.g. ``["pandas==1.5.2"]``.
     package_dir : str, optional (default None)
         The location of the local package to install and test.
 
@@ -41,17 +44,25 @@ def __init__(
         self,
         hook: _HookRelay,
         envname: str,
-        upgrade: List[str],
+        upgrade: Optional[List[str]] = None,
+        lower: Optional[List[str]] = None,
         package_dir: Optional[str] = None,
     ):
         """Init method."""
         self.hook = hook
         self._basedir = Path(Path.cwd(), ".edgetest")
         self._basedir.mkdir(exist_ok=True)
         self.envname = envname
+
+        if (upgrade is None and lower is None) or (
+            upgrade is not None and lower is not None
+        ):
+            raise ValueError("Exactly one of upgrade and lower must be provided.")
         self.upgrade = upgrade
+        self.lower = lower
         self.package_dir = package_dir or "."
 
+        self.setup_status: bool = False
         self.status: bool = False
 
     @property
@@ -69,6 +80,7 @@ def setup(
         self,
         extras: Optional[List[str]] = None,
         deps: Optional[List[str]] = None,
+        skip: bool = False,
         **options,
     ) -> None:
         """Set up the testing environment.
@@ -79,6 +91,9 @@ def setup(
             The list of extra installations to include.
         deps : list, optional (default None)
             A list of additional dependencies to install via ``pip``
+        skip : bool, optional (default False)
+            Whether to skip setup as a pre-made environment has already been
+            created.
         **options
             Additional options for ``self.hook.create_environment``.
 
@@ -91,12 +106,22 @@ def setup(
         RuntimeError
             This error will be raised if any part of the set up process fails.
         """
+        if skip:
+            self.setup_status = True
+            return
         # Create the conda environment
-        LOG.info(f"Creating the following environment: {self.envname}...")
-        self.hook.create_environment(
-            basedir=self._basedir, envname=self.envname, conf=options
-        )
-        LOG.info(f"Successfully created {self.envname}")
+        try:
+            LOG.info(f"Creating the following environment: {self.envname}...")
+            self.hook.create_environment(
+                basedir=self._basedir, envname=self.envname, conf=options
+            )
+            LOG.info(f"Successfully created {self.envname}")
+        except RuntimeError:
+            LOG.exception(
+                "Could not create the following environment: %s", self.envname
+            )
+            self.setup_status = False
+            return
 
         # Install the local package
         with pushd(self.package_dir):
@@ -105,31 +130,85 @@ def setup(
                 pkg += f"[{', '.join(extras)}]"
             if deps:
                 LOG.info(
-                    f"Installing specified additional dependencies into {self.envname}..."
+                    "Installing specified additional dependencies into %s: %s",
+                    self.envname,
+                    ", ".join(deps),
                 )
                 split = [shlex.split(dep) for dep in deps]
-                _run_command(
-                    self.python_path,
-                    "-m",
-                    "pip",
-                    "install",
-                    *[itm for lst in split for itm in lst],
+                try:
+                    _run_command(
+                        self.python_path,
+                        "-m",
+                        "pip",
+                        "install",
+                        *[itm for lst in split for itm in lst],
+                    )
+                except RuntimeError:
+                    LOG.exception(
+                        "Unable to install specified additional dependencies in %s",
+                        self.envname,
+                    )
+                    self.setup_status = False
+                    return
+                LOG.info(
+                    f"Successfully installed specified additional dependencies into {self.envname}"
                 )
+
             LOG.info(f"Installing the local package into {self.envname}...")
-            _run_command(self.python_path, "-m", "pip", "install", pkg)
-            LOG.info(f"Successfully installed the local package into {self.envname}...")
-
-        # Upgrade package(s)
-        LOG.info(
-            f"Upgrading the following packages in {self.envname}: {', '.join(self.upgrade)}"
-        )
-        self.hook.run_update(
-            basedir=self._basedir,
-            envname=self.envname,
-            upgrade=self.upgrade,
-            conf=options,
-        )
-        LOG.info(f"Successfully upgraded packages in {self.envname}")
+            try:
+                _run_command(self.python_path, "-m", "pip", "install", pkg)
+                LOG.info(
+                    f"Successfully installed the local package into {self.envname}..."
+                )
+            except RuntimeError:
+                LOG.exception(
+                    "Unable to install the local package into %s", self.envname
+                )
+                self.setup_status = False
+                return
+
+        if self.upgrade:
+            # Upgrade package(s)
+            LOG.info(
+                f"Upgrading the following packages in {self.envname}: {', '.join(self.upgrade)}"
+            )
+            try:
+                self.hook.run_update(
+                    basedir=self._basedir,
+                    envname=self.envname,
+                    upgrade=self.upgrade,
+                    conf=options,
+                )
+                self.setup_status = True
+            except RuntimeError:
+                self.setup_status = False
+                LOG.exception("Unable to upgrade packages in %s", self.envname)
+                return
+            LOG.info("Successfully upgraded packages in %s", self.envname)
+        else:
+            # Install lower bounds of package(s)
+            LOG.info(
+                "Installing lower bounds of packages in %s: %s",
+                {self.envname},
+                ", ".join(self.lower),  # type:ignore
+            )
+            try:
+                self.hook.run_install_lower(
+                    basedir=self._basedir,
+                    envname=self.envname,
+                    lower=self.lower,
+                    conf=options,
+                )
+                self.setup_status = True
+            except RuntimeError:
+                self.setup_status = False
+                LOG.exception(
+                    "Unable to install lower bounds of packages in %s", self.envname
+                )
+                return
+            LOG.info(
+                f"Successfully installed lower bounds of packages in {self.envname}"
+            )
 
     def upgraded_packages(self) -> List[Dict[str, str]]:
         """Get the list of upgraded packages for the test environment.
@@ -144,6 +223,8 @@ def upgraded_packages(self) -> List[Dict[str, str]]:
             The output of ``pip list --format json``, filtered to the packages upgraded
             for this environment.
         """
+        if self.upgrade is None:
+            return []
         # Get the version for the upgraded package(s)
         out, _ = _run_command(self.python_path, "-m", "pip", "list", "--format", "json")
         outjson = json.loads(out)
@@ -155,6 +236,21 @@ def upgraded_packages(self) -> List[Dict[str, str]]:
             if _isin_case_dashhyphen_ins(pkg.get("name", ""), upgrade_wo_extras)
         ]
 
+    def lowered_packages(self) -> List[Dict[str, str]]:
+        """Get list of lowered packages for the test environment.
+
+        Returns
+        -------
+        List[Dict[str, str]]
+            List of lowered packages and their versions
+        """
+        if self.lower is None:
+            return []
+        packages_split = (pkg_str.split("==") for pkg_str in self.lower)
+        return [
+            {"name": pkg_info[0], "version": pkg_info[1]} for pkg_info in packages_split
+        ]
+
     def run_tests(self, command: str) -> int:
         """Run the tests in the package directory.
 
@@ -168,6 +264,8 @@ def run_tests(self, command: str) -> int:
         int
             The exit code
         """
+        if not self.setup_status:
+            raise RuntimeError("Environment setup failed. Cannot run tests.")
         with pushd(self.package_dir):
             popen = Popen(
                 (self.python_path, "-m", *shlex.split(command)), universal_newlines=True

edgetest/hookspecs.py

@@ -95,6 +95,25 @@ def run_update(basedir: str, envname: str, upgrade: List, conf: Dict):
     """
 
 
+@hookspec(firstresult=True)
+def run_install_lower(basedir: str, envname: str, lower: Dict[str, str], conf: Dict):
+    """Install lower bounds of packages provided.
+
+    Parameters
+    ----------
+    basedir : str
+        The base directory location for the environment.
+    envname : str
+        Environment to install into.
+    lower_bounds : Dict[str, str]
+        Lower bounds of packages to install.
+    conf : Dict
+        The configuration dictionary for the environment. This is useful if you
+        want to add configuration arguments for additional dependencies that can
+        only be installed through the environment manager (e.g. Conda).
+    """
+
+
 @hookspec
 def post_run_hook(testers: List, conf: Dict):
     """Post testing hook.

edgetest/interface.py

@@ -150,17 +150,19 @@ def cli(
             TestPackage(
                 hook=pm.hook,
                 envname=env["name"],
-                upgrade=env["upgrade"],
+                upgrade=env.get("upgrade"),
+                lower=env.get("lower"),
                 package_dir=env["package_dir"],
             )
         )
         # Set up the test environment
         if nosetup:
             click.echo(f"Using existing environment for {env['name']}...")
+            testers[-1].setup(skip=True, **env)
         else:
             testers[-1].setup(**env)
         # Run the tests
-        if notest:
+        if notest or not testers[-1].setup_status:
             click.echo(f"Skipping tests for {env['name']}")
         else:
             testers[-1].run_tests(env["command"])