Reorganize the tests into unit/functional/integration/external (#1155)

* Split tests into unit/integration/functional/external * Include tests in the Jupytext package * Move the test notebooks to tests/data/notebooks * Turn the tests notebooks into fixtures * Remove the world population notebook from the examples In order to keep the package size under control when tests are included * Move the codecov.yml configuration file to .github * Filter more of the expected warnings * packaging is a dependency
mwouts · Nov 23, 2023 · 923798e · 923798e
1 parent 4b56512
commit 923798e
Show file tree

Hide file tree

Showing 609 changed files with 1,420 additions and 4,035 deletions.
diff --git a/.ci/environment-ci.yml b/.ci/environment-ci.yml
@@ -10,23 +10,6 @@ channels:
   - defaults
   - conda-forge
 dependencies:
-  - ipykernel
-  - nbconvert
+  - jupyterlab
   - nbformat>=5.1.2
-  - pyyaml
-  - toml
-  - markdown-it-py
-  - mdit-py-plugins
-  - pip
-  - pytest
-  - pytest-randomly
-  - pytest-cov
-  - coverage
-  - flake8
-  - autopep8
-  - black
-  - isort
   - pandoc==2.16.2
-  - sphinx-gallery<0.8
-  - pre-commit
-  - gitpython
diff --git a/.github/codecov.yml b/.github/codecov.yml
@@ -0,0 +1,34 @@
+codecov:
+  notify:
+    after_n_builds: 15
+
+comment:
+  after_n_builds: 15
+
+coverage:
+  status:
+    project:
+      source:
+        paths:
+          - "src/jupytext/"
+        target: 97%
+        threshold: 0.002
+      tests:
+        paths:
+          - "tests/"
+        target: 100%
+      unit-tests:
+        flags:
+          - unit
+      functional-tests:
+        flags:
+          - functional
+      integration-tests:
+        flags:
+          - integration
+      external-tests:
+        flags:
+          - external
+    patch:
+      default:
+        target: 80%  # new contributions should have a coverage at least equal to target
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -41,6 +41,10 @@ jobs:
     needs: [codeql]
     uses: ./.github/workflows/step_tests-pip.yml
 
+  test-unit-functional-integration:
+    needs: [codeql]
+    uses: ./.github/workflows/step_test_unit_functional.yml
+
   test-conda:
     needs: [codeql]
     uses: ./.github/workflows/step_tests-conda.yml
@@ -56,7 +60,7 @@ jobs:
       upload: ${{ inputs.upload-build-artifacts || false }}
 
   pass:
-    needs: [pre-commit, codeql, test-pip, test-conda, test-ui, build]
+    needs: [pre-commit, codeql, test-unit-functional-integration, test-pip, test-conda, test-ui, build]
     runs-on: ubuntu-latest
     steps:
       - name: Check jobs

diff --git a/.github/workflows/step_build.yml b/.github/workflows/step_build.yml
@@ -29,7 +29,7 @@ jobs:
           python -m pip install build wheel
 
           # NOTE: These builds and verifications of the builds can be done more
-          # robustily with jupyter-releaser.
+          # robustly with jupyter-releaser.
           #
           # Removed the check on size of package as we are distributing tests/ with
           # sdist now and they are around 8MB. Seems like original check was to make

diff --git a/.github/workflows/step_test_unit_functional.yml b/.github/workflows/step_test_unit_functional.yml
@@ -0,0 +1,45 @@
+name: test-categories
+run-name: Run Unit/Functional/Integration and External tests using Pip
+
+on:
+  workflow_call:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-test_classification
+  cancel-in-progress: true
+
+jobs:
+  test-pip:
+    continue-on-error: ${{ matrix.experimental || false }}
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [ "3.11" ]
+        coverage: [unit, functional, integration, external]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Base Setup
+        uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+        with:
+          python_version: ${{ matrix.python-version }}
+
+      - name: Install from source
+        run: HATCH_BUILD_HOOKS_ENABLE=false python -m pip install -e '.[test-cov,test-${{ matrix.coverage }}]' markdown-it-py~=3.0
+
+      - name: Install a Jupyter Kernel
+        run: python -m ipykernel install --name python_kernel --user
+
+      - name: Run the tests
+        run: pytest tests/${{ matrix.coverage }} --cov
+
+      - name: Upload the coverage
+        uses: codecov/codecov-action@v3
+        with:
+          flags: ${{ matrix.coverage }}
+          fail_ci_if_error: true
+          verbose: true
diff --git a/.github/workflows/step_tests-conda.yml b/.github/workflows/step_tests-conda.yml
@@ -13,7 +13,7 @@ jobs:
     strategy:
       matrix:
         os: [ 'ubuntu-latest', 'macos-latest', 'windows-latest' ]
-        python-version: [ 3.9 ]
+        python-version: [ 3.11 ]
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -38,9 +38,8 @@ jobs:
           use-only-tar-bz2: true
 
       - name: Install from source
-        # This is required for the pre-commit tests
         shell: pwsh
-        run: python -m pip install -e '.[test-cov]'
+        run: python -m pip install -e '.[test-external,test-cov]'
 
       - name: Create kernel
         shell: pwsh
@@ -61,8 +60,7 @@ jobs:
 
       - name: Test with pytest
         shell: pwsh
-        run: pytest --cov=src/jupytext --cov-report=xml
+        run: pytest --cov
 
       - name: Upload coverage
         uses: codecov/codecov-action@v3
-        if: ${{ matrix.os != 'windows-latest' }}
diff --git a/.github/workflows/step_tests-pip.yml b/.github/workflows/step_tests-pip.yml
@@ -18,7 +18,6 @@ jobs:
       matrix:
         python-version: [ "3.8", "3.9", "3.10", "3.11" ]
         markdown-it-py-version: ["~=2.0"]
-        kernel: [true]
         include:
           - python-version: "3.12-dev"
             experimental: true
@@ -28,14 +27,13 @@ jobs:
             markdown-it-py-version: ""
           - python-version: "3.x"
             markdown-it-py-version: "~=3.0"
-          - python-version: "3.11"
+          - python-version: "3.x"
             markdown-it-py-version: "~=4.0"
             experimental: true
           - python-version: "3.x"
-            kernel: false
+            no_kernel: true
           - python-version: "3.x"
             quarto: true
-            kernel: true
 
     steps:
       - name: Checkout
@@ -46,11 +44,11 @@ jobs:
         with:
           python_version: ${{ matrix.python-version }}
 
-      - name: Install from source (required for the pre-commit tests)
-        run: python -m pip install -e '.[test-cov]' ${{ matrix.markdown-it-py-version && format('markdown-it-py{0}', matrix.markdown-it-py-version) }}
+      - name: Install from source
+        run: python -m pip install -e '.[test-cov,test-external]' ${{ matrix.markdown-it-py-version && format('markdown-it-py{0}', matrix.markdown-it-py-version) }}
 
       - name: Install a Jupyter Kernel
-        if: ${{ matrix.kernel }}
+        if: ${{ !matrix.no_kernel }}
         run: python -m ipykernel install --name python_kernel --user
 
       - name: Install Quarto
@@ -66,12 +64,13 @@ jobs:
       - name: Test lab extension
         run: |
           # Check extension
+          pip install jupyterlab
           jupyter labextension list
           jupyter labextension list 2>&1 | grep -ie "jupyterlab-jupytext.*OK"
           python -m jupyterlab.browser_check
 
       - name: Test with pytest
-        run: pytest --cov=src/jupytext --cov-report=xml
+        run: pytest --cov
 
       - name: Upload coverage
         uses: codecov/codecov-action@v3
diff --git a/.github/workflows/step_tests-ui.yml b/.github/workflows/step_tests-ui.yml
@@ -20,14 +20,16 @@ jobs:
       - name: Base Setup
         uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
 
-      - name: Install from source (required for the pre-commit tests)
-        run: python -m pip install -e '.[test-cov]'
+      - name: Install from source
+        run: python -m pip install -e .
 
       - name: Install galata
         working-directory: jupyterlab/packages/jupyterlab-jupytext-extension/ui-tests
         env:
           PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
-        run: jlpm install
+        run: |
+          pip install jupyterlab
+          jlpm install
 
       - name: Install browser
         working-directory: jupyterlab/packages/jupyterlab-jupytext-extension/ui-tests

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -13,7 +13,7 @@
 exclude: >
     (?x)^(
       demo/.*|
-      tests/notebooks/.*|
+      tests/data/notebooks/.*|
     )$
 repos:
 

diff --git a/codecov.yml b/codecov.yml
diff --git a/docs/contributing.md b/docs/contributing.md
@@ -20,5 +20,5 @@ You want to submit an enhancement on Jupytext? Unless this is a small change, we
 A pull request for which you do not need to contact us in advance is the addition of a new language to Jupytext. In principle that should be easy - you would only have to:
 - document the language extension and comment by adding one line to `_SCRIPT_EXTENSIONS` in `jupytext/languages.py`.
 - add the language to `docs/languages.md`
-- contribute a sample notebook in `tests/notebooks/ipynb_[language]`.
-- run the tests suite (create a [development environment](developing.md), then execute `pytest` locally). The tests will generate various text representations corresponding to your notebook under  `tests/notebooks/mirror/`. Please verify that these files are valid scripts, and include them in your PR.
+- contribute a sample notebook in `tests/data/notebooks/inputs/ipynb_[language]`.
+- run the tests suite (create a [development environment](developing.md), then execute `pytest` locally). The tests will generate various text representations corresponding to your notebook under  `tests/data/notebooks/outputs/`. Please verify that these files are valid scripts, and include them in your PR.
diff --git a/docs/using-pre-commit.md b/docs/using-pre-commit.md
@@ -62,5 +62,5 @@ repos:
       language_version: python3
 ```
 
-Tested examples of how to use the pre-commit hook are available in our [tests](https://github.com/mwouts/jupytext/tree/main/tests) -
-see for instance [test_pre_commit_1_sync_with_config.py](https://github.com/mwouts/jupytext/blob/main/tests/test_pre_commit_1_sync_with_config.py).
+Tested examples of how to use the pre-commit hook are available in our [tests](https://github.com/mwouts/jupytext/tree/main/tests/functional/pre-commit) -
+see for instance [test_pre_commit_1_sync_with_config.py](https://github.com/mwouts/jupytext/blob/main/tests/functional/pre-commit/test_pre_commit_1_sync_with_config.py).
diff --git a/environment.yml b/environment.yml
@@ -6,23 +6,6 @@ dependencies:
   - python>=3.8
   - jupyterlab>=4.0.0
   - nbformat>=5.1.2
-  - pyyaml
-  - toml
-  - markdown-it-py>=1.0.0,<3.0.0
-  - mdit-py-plugins
-  - nbconvert
-  - ipykernel
-  - pytest
-  - pytest-xdist
-  - pytest-randomly
-  - pytest-cov
   - pre-commit
-  - gitpython
-  - pylint
-  - flake8
-  - black==23.11.0
-  - isort==5.12.0
-  - autopep8
-  - sphinx-gallery<0.8
-  - pandoc==2.16.2
   - nodejs>=20
+  - pandoc==2.16.2