Merge pull request #58 from LUMC/release_1.1.0

Release 1.1.0

Author: rhpvorderman

.travis.yml

@@ -52,5 +52,5 @@ matrix:
         - conda config --add channels defaults
         - conda config --add channels bioconda
         - conda config --add channels conda-forge
-        - conda create -n my_env cromwell
+        - conda create -n my_env cromwell tox  # Install tox for good integration within the conda env.
         - source activate my_env

HISTORY.rst

@@ -7,6 +7,10 @@ Changelog
 .. NOTE: This document is user facing. Please word the changes in such a way
 .. that users understand how the changes affect the new version.
 
+version 1.1.0
+---------------------------
++ Enabled custom tests on workflow files.
+
 Version 1.0.0
 ---------------------------
 Lots of small fixes that improve the usability of pytest-workflow are included

README.rst

@@ -4,6 +4,14 @@ pytest-workflow
 
 .. Badges have empty alts. So nothing shows up if they do not work.
 .. This fixes readthedocs issues with badges.
+.. image:: https://img.shields.io/pypi/v/pytest-workflow.svg
+  :target: https://pypi.org/project/pytest-workflow/
+  :alt:
+
+.. image:: https://img.shields.io/pypi/pyversions/pytest-workflow.svg
+  :target: https://pypi.org/project/pytest-workflow/
+  :alt:
+
 .. image:: https://api.codacy.com/project/badge/Grade/f8bc14b0a507429eac7c06194fafcd59
   :target: https://www.codacy.com/app/LUMC/pytest-workflow?utm_source=github.com&utm_medium=referral&utm_content=LUMC/pytest-workflow&utm_campaign=Badge_Grade
   :alt:
@@ -16,10 +24,6 @@ pytest-workflow
   :target: https://codecov.io/gh/LUMC/pytest-workflow
   :alt:
 
-.. image:: https://img.shields.io/pypi/pyversions/pytest-workflow.svg
-  :target: https://pypi.org/project/pytest-workflow/
-  :alt:
-
 pytest-workflow is a pytest plugin that aims to make pipeline/workflow testing easy
 by using yaml files for the test configuration.
 
@@ -65,7 +69,8 @@ Below is an example of a YAML file that defines a test:
 This will run ``touch test.file`` and check afterwards if a file with path:
 ``test.file`` is present. It will also check if the ``command`` has exited
 with exit code ``0``, which is the only default test that is run. Testing
-workflows that exit with another exit code is also possible.
+workflows that exit with another exit code is also possible. Several other
+predefined tests as well as custom tests are possible.
 
 Documentation for more advanced use cases can be found on our
 `readthedocs page <https://pytest-workflow.readthedocs.io/>`_.

docs/writing_tests.rst

@@ -2,6 +2,9 @@
 Writing tests with pytest-workflow
 ==================================
 
+Getting started
+---------------
+
 In order to write tests that are discoverable by the plugin you need to
 complete the following steps.
 
@@ -23,7 +26,8 @@ This will run ``touch test.file`` and check afterwards if a file with path:
 with exit code ``0``, which is the only default test that is run. Testing
 workflows that exit with another exit code is also possible.
 
-A more advanced example:
+Test options
+------------
 
 .. code-block:: yaml
 
@@ -65,3 +69,40 @@ A more advanced example:
 
 
 The above YAML file contains all the possible options for a workflow test.
+
+Writing custom tests
+--------------------
+
+Pytest-workflow provides a way to run custom tests on files produced by a
+workflow.
+
+.. code-block:: python
+
+    import pathlib
+    import pytest
+
+    @pytest.mark.workflow(name='files containing numbers')
+    def test_div_by_three(workflow_dir):
+        number_file = workflow_dir / pathlib.Path("123.txt")
+
+        with number_file.open('rt') as file_h:
+            number_file_content = file_h.read()
+
+        assert int(number_file_content) % 3 == 0
+
+The ``@pytest.mark.workflow(name='files containing numbers')`` marks the test
+as belonging to a workflow named 'files containing numbers'. The mark can also
+be written without the explicit ``name`` key as ``@pytest.mark.workflow('files
+containing nummbers')``. This test will only run if the workflow 'files
+containing numbers' has run.
+
+``workflow_dir`` is a fixture. It does not work without a
+``pytest.mark.workflow('workflow_name')`` mark.  This is a
+`pathlib.Path <https://docs.python.org/3/library/pathlib.html>`_ object that
+points to the folder where the named workflow was executed. This allows writing of
+advanced python tests for each file produced by the workflow.
+
+.. container:: note
+
+    NOTE: stdout and stderr are available as files in the root of the
+    ``workflow_dir`` as ``log.out`` and ``log.err`` respectively.

mypy.ini

@@ -0,0 +1,3 @@
+[mypy]
+# This ignores a lot of errors that are a huge headache to fix.
+ignore_missing_imports = True

setup.py

@@ -21,7 +21,7 @@
 
 setup(
     name="pytest-workflow",
-    version="1.0.0",
+    version="1.1.0",
     description="A pytest plugin for configuring workflow/pipeline tests "
                 "using YAML files",
     author="Leiden University Medical Center",

src/pytest_workflow/content_tests.py

@@ -46,7 +46,7 @@ def check_content(strings: List[str],
 
     # Create two sets. By default all strings are not found.
     strings_to_check = set(strings)
-    found_strings = set()
+    found_strings = set()  # type: Set[str]
 
     for line in text_lines:
         # Break the loop if all strings are found
@@ -88,7 +88,7 @@ def file_to_string_generator(filepath: Path) -> Iterable[str]:
                  if filepath.suffix == ".gz" else
                  filepath.open)
     # Use 'rt' here explicitly as opposed to 'rb'
-    with file_open(mode='rt') as file_handler:
+    with file_open(mode='rt') as file_handler:  # type: ignore  # mypy goes crazy here otherwise  # noqa: E501
         for line in file_handler:
             yield line
 

src/pytest_workflow/plugin.py

@@ -19,7 +19,12 @@
 import tempfile
 import warnings
 from pathlib import Path
-from typing import Optional  # noqa: F401 needed for typing.
+from typing import List, Optional  # noqa: F401 needed for typing.
+
+from _pytest.config import Config as PytestConfig
+from _pytest.config.argparsing import Parser as PytestParser
+from _pytest.fixtures import SubRequest
+from _pytest.mark import MarkDecorator  # noqa: F401 used for typing
 
 import pytest
 
@@ -32,7 +37,7 @@
 from .workflow import Workflow, WorkflowQueue
 
 
-def pytest_addoption(parser):
+def pytest_addoption(parser: PytestParser):
     parser.addoption(
         "--keep-workflow-wd",
         action="store_true",
@@ -82,7 +87,7 @@ def pytest_collect_file(path, parent):
     return None
 
 
-def pytest_configure(config):
+def pytest_configure(config: PytestConfig):
     """This runs before tests start and adds values to the config."""
     # We need to add a workflow queue to some central variable. Instead of
     # using a global variable we add a value to the config.
@@ -92,6 +97,14 @@ def pytest_configure(config):
     workflow_queue = WorkflowQueue()
     setattr(config, "workflow_queue", workflow_queue)
 
+    # Save which workflows are run and which are not.
+    executed_workflows = []  # type: List[str]
+    setattr(config, "executed_workflows", executed_workflows)
+
+    # Save workflow for cleanup in this var.
+    workflow_cleanup_dirs = []  # type: List[str]
+    setattr(config, "workflow_cleanup_dirs", workflow_cleanup_dirs)
+
     # When multiple workflows are started they should all be set in the same
     # temporary directory
     # Running in a temporary directory will prevent the project repository
@@ -110,13 +123,13 @@ def pytest_configure(config):
     # So this is why the native pytest `tmpdir` fixture is not used.
 
     basetemp = config.getoption("basetemp")
-    workflow_dir = (
+    workflow_temp_dir = (
         Path(basetemp) if basetemp is not None
         else Path(tempfile.mkdtemp(prefix="pytest_workflow_")))
-    setattr(config, "workflow_dir", workflow_dir)
+    setattr(config, "workflow_temp_dir", workflow_temp_dir)
 
 
-def pytest_collection(session):
+def pytest_collection(session: pytest.Session):
     """This function is started at the beginning of collection"""
     # pylint: disable=unused-argument
     # needed for pytest
@@ -129,13 +142,81 @@ def pytest_collection(session):
     print()
 
 
-def pytest_runtestloop(session):
+def pytest_collection_modifyitems(config: PytestConfig,
+                                  items: List[pytest.Item]):
+    """Here we skip all tests related to workflows that are not executed"""
+
+    for item in items:
+        marker = item.get_closest_marker(
+            name="workflow")  # type: Optional[MarkDecorator] # noqa: E501
+
+        if marker is None:
+            continue
+
+        if 'name' in marker.kwargs:
+            workflow_name = marker.kwargs['name']
+        # If name key is not defined use the first arg.
+        elif 'name' not in marker.kwargs and len(marker.args) >= 1:
+            workflow_name = marker.args[0]
+            # Make sure a name attribute is added anyway for the
+            # fixture lookup.
+            marker.kwargs['name'] = workflow_name
+        else:
+            # If we raise an error here a number of things will happen:
+            # + Pytest will crash. Giving a lot of INTERNAL ERROR lines
+            # + No tests will run
+            # + No workflows will be started
+            # + All the threads that are waiting for a workflow to
+            #   finish will wait indefinitely. Causing an infinite
+            #   hang. Pytest will never finish.
+            # Therefore we do not crash here, but raise a warning.
+            item.warn(pytest.PytestWarning(
+                "A workflow name should be defined in the "
+                "workflow marker of {0}".format(item.nodeid)))
+            # Go on with the next item.
+            continue
+
+        if workflow_name not in config.executed_workflows:
+            skip_marker = pytest.mark.skip(
+                reason="'{0}' has not run.".format(workflow_name))
+            item.add_marker(skip_marker)
+
+
+def pytest_runtestloop(session: pytest.Session):
     """This runs after collection, but before the tests."""
     session.config.workflow_queue.process(
         session.config.getoption("workflow_threads")
     )
 
 
+def pytest_sessionfinish(session: pytest.Session):
+    if not session.config.getoption("keep_workflow_wd"):
+        for tempdir in session.config.workflow_cleanup_dirs:
+            shutil.rmtree(str(tempdir))
+
+
+@pytest.fixture()
+def workflow_dir(request: SubRequest):
+    """Returns the workflow_dir of the workflow named in the mark. This fixture
+    is only provided for tests that are marked with the workflow mark."""
+
+    # request.node refers to the node that has the mark. This is a pytest.Node
+    marker = request.node.get_closest_marker(name="workflow")
+
+    if marker is not None:
+        workflow_temp_dir = request.config.workflow_temp_dir
+        try:
+            workflow_name = marker.kwargs['name']
+        except KeyError:
+            raise TypeError(
+                "A workflow name should be defined in the "
+                "workflow marker of {0}".format(request.node.nodeid))
+        return workflow_temp_dir / Path(replace_whitespace(workflow_name))
+    else:
+        raise ValueError("workflow_dir can only be requested in tests marked"
+                         " with the workflow mark.")
+
+
 class YamlFile(pytest.File):
     """
     This class collects YAML files and turns them into test items.
@@ -162,8 +243,6 @@ class WorkflowTestsCollector(pytest.Collector):
     def __init__(self, workflow_test: WorkflowTest, parent: pytest.Collector):
         self.workflow_test = workflow_test
         super().__init__(workflow_test.name, parent=parent)
-        # Tempdir is stored for cleanup with teardown().
-        self.tempdir = None  # type: Optional[Path]
 
         # Attach tags to this node for easier workflow selection
         self.tags = [self.workflow_test.name] + self.workflow_test.tags
@@ -185,28 +264,34 @@ def queue_workflow(self):
         This is shorter than using pytest's terminal reporter.
         """
 
-        self.tempdir = (self.config.workflow_dir /
-                        Path(replace_whitespace(self.name, '_')))
+        tempdir = (self.config.workflow_temp_dir /
+                   Path(replace_whitespace(self.name, '_')))
 
         # Remove the tempdir if it exists. This is needed for shutil.copytree
         # to work properly.
-        if self.tempdir.exists():
+        if tempdir.exists():
             warnings.warn(
-                "'{0}' already exists. Deleting ...".format(self.tempdir))
-            shutil.rmtree(str(self.tempdir))
+                "'{0}' already exists. Deleting ...".format(tempdir))
+            shutil.rmtree(str(tempdir))
 
         # Copy the project directory to the temporary directory using pytest's
         # rootdir.
-        shutil.copytree(str(self.config.rootdir), str(self.tempdir))
+        shutil.copytree(str(self.config.rootdir), str(tempdir))
 
         # Create a workflow and make sure it runs in the tempdir
         workflow = Workflow(command=self.workflow_test.command,
-                            cwd=self.tempdir,
+                            cwd=tempdir,
                             name=self.workflow_test.name)
 
         # Add the workflow to the workflow queue.
         self.config.workflow_queue.put(workflow)
 
+        # Add the tempdir to the removal queue. We do not use a teardown method
+        # because this will remove the tempdir right after all the tests from
+        # this node have finished. If custom tests are defined this should not
+        # happen. The removal queue is processed just before pytest finishes
+        # and all tests have run.
+        self.config.workflow_cleanup_dirs.append(tempdir)
         return workflow
 
     def collect(self):
@@ -222,6 +307,9 @@ def collect(self):
         if not (set(self.config.getoption("workflow_tags")
                     ).issubset(set(self.tags))):
             return []
+        else:
+            # If we run the workflow, save this for reference later.
+            self.config.executed_workflows.append(self.workflow_test.name)
 
         # This creates a workflow that is queued for processing after the
         # collection phase.
@@ -253,13 +341,6 @@ def collect(self):
 
         return tests
 
-    def teardown(self):
-        """This function is executed after all tests from this collector have
-        finished. It is used to cleanup the tempdir."""
-        if (not self.config.getoption("keep_workflow_wd")
-                and self.tempdir is not None):
-            shutil.rmtree(str(self.tempdir))
-
 
 class ExitCodeTest(pytest.Item):
     def __init__(self, parent: pytest.Collector,

src/pytest_workflow/workflow.py

@@ -139,7 +139,11 @@ def stderr(self) -> bytes:
     @property
     def exit_code(self) -> int:
         self.wait()
-        return self._popen.returncode
+        if self._popen is not None:
+            return self._popen.returncode
+        else:
+            raise ValueError("No exit code after waiting. Please contact the "
+                             "developers and report this issue.")
 
 
 class WorkflowQueue(queue.Queue):