Fast failing test in ci (tensorflow#1666)

* Added a pytest option to skip the loading of custom ops.

* Added a ci run for python-only tests.

* Fix the usage of load_op_library.

* Revert "Fix the usage of load_op_library."

This reverts commit 94db295.

* Fixture is executed after collection.

* Removed assert False.

* Try running on macos for speed.

* Fix path.

* Run on ubuntu directly.

* Add caching for pip.

* Indent.

* Better wording.

* Remove the cache because we don't get any benefits.

.github/workflows/ci_test.yml

@@ -76,6 +76,17 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: bash tools/run_build.sh test_editable_mode
+  test_python_code_only:
+    name: Fast build to run python-only tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-python@v1
+        with:
+          python-version: 3.5
+      - uses: actions/checkout@v2
+      - run: pip install -r tools/install_deps/tensorflow-cpu.txt -r tools/install_deps/pytest.txt
+      - run: pip install -e ./
+      - run: pytest -v -n auto --skip-custom-ops ./tensorflow_addons
   test_cpu_in_small_docker_image:
     name: Run the cpu tests in a small python docker image
     runs-on: ubuntu-latest

CONTRIBUTING.md

@@ -200,6 +200,9 @@ Pytest has many cool options to help you make great tests:
 pytest -n 3 tensorflow_addons/
 pytest -n auto tensorflow_addons/
 
+# Run the whole test suite without compiling any custom ops (.so files).
+pytest -v --skip-custom-ops tensorflow_addons/
+
 # Open the debugger to inspect variables and execute code when 
 # an exception is raised.
 pytest --pdb tensorflow_addons/ 

tensorflow_addons/conftest.py

@@ -2,6 +2,8 @@
 from tensorflow_addons.utils.test_utils import cpu_and_gpu  # noqa: F401
 from tensorflow_addons.utils.test_utils import data_format  # noqa: F401
 from tensorflow_addons.utils.test_utils import set_seeds  # noqa: F401
+from tensorflow_addons.utils.test_utils import pytest_addoption  # noqa: F401
+from tensorflow_addons.utils.test_utils import set_global_variables  # noqa: F401
 
 # fixtures present in this file will be available
 # when running tests and can be referenced with strings

tensorflow_addons/tests/register_test.py

@@ -3,14 +3,25 @@
 import pytest
 import tensorflow as tf
 from tensorflow_addons.register import register_all, _get_all_shared_objects
+from tensorflow_addons.utils import resource_loader
 
 
 def test_multiple_register():
+    if resource_loader.SKIP_CUSTOM_OPS:
+        pytest.skip(
+            "Skipping the test because a custom ops "
+            "was being loaded while --skip-custom-ops was set."
+        )
     register_all()
     register_all()
 
 
 def test_get_all_shared_objects():
+    if resource_loader.SKIP_CUSTOM_OPS:
+        pytest.skip(
+            "Skipping the test because a custom ops "
+            "was being loaded while --skip-custom-ops was set."
+        )
     all_shared_objects = _get_all_shared_objects()
     assert len(all_shared_objects) >= 4
 

tensorflow_addons/utils/resource_loader.py

@@ -23,6 +23,7 @@
 MIN_TF_VERSION_FOR_ABI_COMPATIBILITY = "2.1.0"
 MAX_TF_VERSION_FOR_ABI_COMPATIBILITY = "2.2.0"
 abi_warning_already_raised = False
+SKIP_CUSTOM_OPS = False
 
 
 def get_project_root():
@@ -51,6 +52,13 @@ def __init__(self, relative_path):
 
     @property
     def ops(self):
+        if SKIP_CUSTOM_OPS:
+            import pytest
+
+            pytest.skip(
+                "Skipping the test because a custom ops "
+                "was being loaded while --skip-custom-ops was set."
+            )
         if self._ops is None:
             self.display_warning_if_incompatible()
             self._ops = tf.load_op_library(get_path_to_datafile(self.relative_path))

tensorflow_addons/utils/test_utils.py

@@ -23,6 +23,8 @@
 import pytest
 import tensorflow as tf
 
+from tensorflow_addons.utils import resource_loader
+
 # TODO: find public API alternative to these
 from tensorflow.python.framework.test_util import (  # noqa: F401
     run_all_in_graph_and_eager_modes,
@@ -202,6 +204,20 @@ def set_seeds():
     tf.random.set_seed(0)
 
 
+def pytest_addoption(parser):
+    parser.addoption(
+        "--skip-custom-ops",
+        action="store_true",
+        help="When a custom op is being loaded in a test, skip this test.",
+    )
+
+
+@pytest.fixture(scope="session", autouse=True)
+def set_global_variables(request):
+    if request.config.getoption("--skip-custom-ops"):
+        resource_loader.SKIP_CUSTOM_OPS = True
+
+
 def assert_allclose_according_to_type(
     a,
     b,