Use pandoc>=3.0 in the tests

.ci/environment-ci.yml

@@ -12,4 +12,4 @@ channels:
 dependencies:
   - jupyterlab
   - nbformat>=5.1.2
-  - pandoc==2.16.2
+  - pandoc>=3.0

.github/workflows/step_tests-conda.yml

@@ -39,11 +39,8 @@ jobs:
         uses: conda-incubator/setup-miniconda@v3
         with:
           auto-update-conda: true
-          auto-activate-base: false
           activate-environment: jupytext-ci
-          channels: defaults,conda-forge
           environment-file: .ci/environment-ci.yml
-          use-only-tar-bz2: true
 
       - name: Install from source
         run: python -m pip install -e '.[test-external,test-cov]'

CHANGELOG.md

@@ -7,6 +7,9 @@ Jupytext ChangeLog
 **Changed**
 - Temporary text notebooks for the `--pipe` or `--check` commands are now created in the notebook directory ([#1206](https://github.com/mwouts/jupytext/issues/1206))
 
+**Fixed**
+- Jupytext is now tested with `pandoc>=3.0`. Please note that switching to `pandoc>=3.0` will add cell ids to your `pandoc:md` notebooks ([#1006](https://github.com/mwouts/jupytext/issues/1006))
+
 
 1.16.1 (2024-01-13)
 -------------------

environment.yml

@@ -8,4 +8,4 @@ dependencies:
   - nbformat>=5.1.2
   - pre-commit
   - nodejs>=20
-  - pandoc==2.16.2
+  - pandoc>=3.0

src/jupytext/jupytext.py

@@ -177,7 +177,7 @@
 
         return new_notebook(cells=cells, metadata=metadata)
 
-    def filter_notebook(self, nb, metadata):
+    def filter_notebook(self, nb, metadata, preserve_cell_ids=False):
         self.update_fmt_with_notebook_options(nb.metadata)
         unsupported_keys = set()
         metadata = insert_jupytext_info_and_filter_metadata(
@@ -192,14 +192,23 @@
                 _IGNORE_CELL_METADATA,
                 unsupported_keys=unsupported_keys,
             )
+
+            if preserve_cell_ids and hasattr(cell, "id"):
+                id = {"id": cell.id}
+            else:
+                id = {}
+
             if cell.cell_type == "code":
-                cells.append(new_code_cell(source=cell.source, metadata=cell_metadata))
+                cells.append(
+                    new_code_cell(source=cell.source, metadata=cell_metadata, **id)
+                )
             else:
                 cells.append(
                     NotebookNode(
                         source=cell.source,
                         metadata=cell_metadata,
                         cell_type=cell.cell_type,
+                        **id,
                     )
                 )
 
@@ -215,7 +224,9 @@
     def writes(self, nb, metadata=None, **kwargs):
         """Return the text representation of the notebook"""
         if self.fmt.get("format_name") == "pandoc":
-            return notebook_to_md(self.filter_notebook(nb, metadata))
+            return notebook_to_md(
+                self.filter_notebook(nb, metadata, preserve_cell_ids=True)
+            )
         if self.fmt.get("format_name") == "quarto" or self.ext == ".qmd":
             return notebook_to_qmd(self.filter_notebook(nb, metadata))
         if self.fmt.get(

tests/conftest.py

@@ -368,8 +368,8 @@ def pytest_runtest_setup(item):
             # https://github.com/mwouts/jupytext/commit/c07d919702999056ce47f92b74f63a15c8361c5d
             # The mirror files changed again when Pandoc 2.16 was introduced
             # https://github.com/mwouts/jupytext/pull/919/commits/1fa1451ecdaa6ad8d803bcb6fb0c0cf09e5371bf
-            if not is_pandoc_available(min_version="2.16.2", max_version="2.16.2"):
-                pytest.skip("pandoc==2.16.2 is not available")
+            if not is_pandoc_available(min_version="3.0"):
+                pytest.skip("pandoc>=3.0 is not available")
         if mark.name == "requires_quarto":
             if not is_quarto_available(min_version="0.2.0"):
                 pytest.skip("quarto>=0.2 is not available")
@@ -410,15 +410,15 @@ def pytest_collection_modifyitems(config, items):
             item.add_marker(pytest.mark.pre_commit)
 
 
-"""To make sure that cell ids are distinct we use a global counter.
+@pytest.fixture(autouse=True)
+def cell_id():
+    """To make sure that cell ids are distinct we use a global counter.
     This solves https://github.com/mwouts/jupytext/issues/747"""
-global_cell_count = 0
+    local_cell_count = 0
 
+    def enumerate_cell_ids():
+        nonlocal local_cell_count
+        local_cell_count += 1
+        return f"cell-{local_cell_count}"
 
-def generate_corpus_id():
-    global global_cell_count
-    global_cell_count += 1
-    return f"cell-{global_cell_count}"
-
-
-nbbase.random_cell_id = generate_corpus_id
+    nbbase.random_cell_id = enumerate_cell_ids

tests/data/notebooks/outputs/ipynb_to_pandoc/Notebook_with_R_magic.md

@@ -14,20 +14,20 @@ jupyter:
 This notebook shows the use of R cells to generate plots
 :::
 
-::: {.cell .code}
+::: {#cell-1 .cell .code}
 ``` python
 %load_ext rpy2.ipython
 ```
 :::
 
-::: {.cell .code}
+::: {#cell-2 .cell .code}
 ``` python
 %%R
 suppressMessages(require(tidyverse))
 ```
 :::
 
-::: {.cell .code}
+::: {#cell-3 .cell .code}
 ``` python
 %%R
 ggplot(iris, aes(x = Sepal.Length, y = Petal.Length, color=Species)) + geom_point()
@@ -38,7 +38,7 @@ ggplot(iris, aes(x = Sepal.Length, y = Petal.Length, color=Species)) + geom_poin
 The default plot dimensions are not good for us, so we use the -w and -h parameters in %%R magic to set the plot size
 :::
 
-::: {.cell .code}
+::: {#cell-4 .cell .code}
 ``` python
 %%R -w 400 -h 240
 ggplot(iris, aes(x = Sepal.Length, y = Petal.Length, color=Species)) + geom_point()

tests/data/notebooks/outputs/ipynb_to_pandoc/Notebook_with_more_R_magic_111.md

@@ -8,7 +8,7 @@ jupyter:
   nbformat_minor: 2
 ---
 
-::: {.cell .code}
+::: {#cell-1 .cell .code}
 ``` python
 %load_ext rpy2.ipython
 import pandas as pd
@@ -24,7 +24,7 @@ df = pd.DataFrame(
 ```
 :::
 
-::: {.cell .code}
+::: {#cell-2 .cell .code}
 ``` python
 %%R -i df
 library("ggplot2")

tests/data/notebooks/outputs/ipynb_to_pandoc/cat_variable.md

@@ -8,7 +8,7 @@ jupyter:
   nbformat_minor: 2
 ---
 
-::: {.cell .code}
+::: {#cell-1 .cell .code}
 ``` python
 cat = 42
 ```

tests/data/notebooks/outputs/ipynb_to_pandoc/convert_to_py_then_test_with_update83.md

@@ -8,7 +8,7 @@ jupyter:
   nbformat_minor: 2
 ---
 
-::: {.cell .code}
+::: {#cell-1 .cell .code}
 ``` python
 %%time
 
@@ -20,7 +20,7 @@ print('asdf')
 Thanks for jupytext!
 :::
 
-::: {.cell .code}
+::: {#cell-2 .cell .code}
 ``` python
 ```
 :::

tests/data/notebooks/outputs/ipynb_to_pandoc/frozen_cell.md

@@ -8,14 +8,14 @@ jupyter:
   nbformat_minor: 2
 ---
 
-::: {.cell .code}
+::: {#cell-1 .cell .code}
 ``` python
 # This is an unfrozen cell. Works as usual.
 print("I'm a regular cell so I run and print!")
 ```
 :::
 
-::: {.cell .code deletable="false" editable="false" run_control="{\"frozen\":true}"}
+::: {#cell-2 .cell .code deletable="false" editable="false" run_control="{\"frozen\":true}"}
 ``` python
 # This is an frozen cell
 print("I'm frozen so Im not executed :(")

tests/data/notebooks/outputs/ipynb_to_pandoc/ir_notebook.md

@@ -12,19 +12,19 @@ jupyter:
 This is a jupyter notebook that uses the IR kernel.
 :::
 
-::: {.cell .code}
+::: {#cell-1 .cell .code}
 ``` R
 sum(1:10)
 ```
 :::
 
-::: {.cell .code}
+::: {#cell-2 .cell .code}
 ``` R
 plot(cars)
 ```
 :::
 
-::: {.cell .code}
+::: {#cell-3 .cell .code}
 ``` R
 ```
 :::

tests/data/notebooks/outputs/ipynb_to_pandoc/julia_benchmark_plotly_barchart.md

@@ -8,7 +8,7 @@ jupyter:
   nbformat_minor: 1
 ---
 
-::: {.cell .code}
+::: {#cell-1 .cell .code}
 ``` julia
 # IJulia rocks! So does Plotly. Check it out
 
@@ -20,7 +20,7 @@ Plotly.signin(username, api_key)
 ```
 :::
 
-::: {.cell .code}
+::: {#cell-2 .cell .code}
 ``` julia
 # Following data taken from http://julialang.org/ frontpage 
 benchmarks = ["fib", "parse_int", "quicksort3", "mandel", "pi_sum", "rand_mat_stat", "rand_mat_mul"]
@@ -69,7 +69,7 @@ display("text/html", s)
 ```
 :::
 
-::: {.cell .code}
+::: {#cell-3 .cell .code}
 ``` julia
 # checkout https://plot.ly/api/ for more Julia examples!
 # But to show off some other Plotly features:

tests/data/notebooks/outputs/ipynb_to_pandoc/jupyter.md

@@ -14,7 +14,7 @@ jupyter:
 This notebook is a simple jupyter notebook. It only has markdown and code cells. And it does not contain consecutive markdown cells. We start with an addition:
 :::
 
-::: {.cell .code}
+::: {#cell-1 .cell .code}
 ``` python
 a = 1
 b = 2
@@ -26,13 +26,13 @@ a + b
 Now we return a few tuples
 :::
 
-::: {.cell .code}
+::: {#cell-2 .cell .code}
 ``` python
 a, b
 ```
 :::
 
-::: {.cell .code}
+::: {#cell-3 .cell .code}
 ``` python
 a, b, a+b
 ```

tests/data/notebooks/outputs/ipynb_to_pandoc/jupyter_again.md

@@ -8,7 +8,7 @@ jupyter:
   nbformat_minor: 2
 ---
 
-::: {.cell .code}
+::: {#cell-1 .cell .code}
 ``` python
 c = '''
 title: "Quick test"
@@ -22,14 +22,14 @@ editor_options:
 ```
 :::
 
-::: {.cell .code}
+::: {#cell-2 .cell .code}
 ``` python
 import yaml
 print(yaml.dump(yaml.load(c)))
 ```
 :::
 
-::: {.cell .code}
+::: {#cell-3 .cell .code}
 ``` python
 ?next
 ```

tests/data/notebooks/outputs/ipynb_to_pandoc/jupyter_with_raw_cell_in_body.md

@@ -8,7 +8,7 @@ jupyter:
   nbformat_minor: 2
 ---
 
-::: {.cell .code}
+::: {#cell-1 .cell .code}
 ``` python
 1+2+3
 ```

tests/data/notebooks/outputs/ipynb_to_pandoc/jupyter_with_raw_cell_on_top.md

@@ -22,13 +22,13 @@ editor_options:
 ```
 :::
 
-::: {.cell .code}
+::: {#cell-1 .cell .code}
 ``` python
 1+2+3
 ```
 :::
 
-::: {.cell .code}
+::: {#cell-2 .cell .code}
 ``` python
 ```
 :::

tests/data/notebooks/outputs/ipynb_to_pandoc/notebook_with_complex_metadata.md

@@ -8,7 +8,7 @@ jupyter:
   nbformat_minor: 2
 ---
 
-::: {.cell .code}
+::: {#cell-1 .cell .code}
 ``` python
 ```
 :::

tests/data/notebooks/outputs/ipynb_to_pandoc/nteract_with_parameter.md

@@ -10,27 +10,27 @@ jupyter:
   nbformat_minor: 2
 ---
 
-::: {.cell .code inputHidden="false" outputHidden="false" tags="[\"parameters\"]"}
+::: {#cell-1 .cell .code inputHidden="false" outputHidden="false" tags="[\"parameters\"]"}
 ``` python
 param = 4
 ```
 :::
 
-::: {.cell .code inputHidden="false" outputHidden="false"}
+::: {#cell-2 .cell .code inputHidden="false" outputHidden="false"}
 ``` python
 import pandas as pd
 ```
 :::
 
-::: {.cell .code inputHidden="false" outputHidden="false"}
+::: {#cell-3 .cell .code inputHidden="false" outputHidden="false"}
 ``` python
 df = pd.DataFrame({'A': [1, 2], 'B': [3 + param, 4]},
                   index=pd.Index(['x0', 'x1'], name='x'))
 df
 ```
 :::
 
-::: {.cell .code inputHidden="false" outputHidden="false"}
+::: {#cell-4 .cell .code inputHidden="false" outputHidden="false"}
 ``` python
 %matplotlib inline
 df.plot(kind='bar')

tests/data/notebooks/outputs/ipynb_to_pandoc/plotly_graphs.md

@@ -20,14 +20,14 @@ This notebook contains complex outputs, including plotly javascript graphs.
 We use Plotly\'s connected mode to make the notebook lighter - when connected, the notebook downloads the `plotly.js` library from the web.
 :::
 
-::: {.cell .code}
+::: {#cell-1 .cell .code}
 ``` python
 import plotly.offline as offline
 offline.init_notebook_mode(connected=True)
 ```
 :::
 
-::: {.cell .code}
+::: {#cell-2 .cell .code}
 ``` python
 import plotly.graph_objects as go
 fig = go.Figure(