From dcb06b022f8082e83a9480b2ac326a99e806b0b6 Mon Sep 17 00:00:00 2001 From: Dalton Smith Date: Tue, 16 Feb 2021 00:41:02 -0500 Subject: [PATCH] Cleanup Extension Structure (#12) --- .github/workflows/workflow.yml | 21 +- README.md | 11 + dev-requirements.txt | 1 + setup.py | 54 ++-- sphinxext/remoteliteralinclude.py | 259 ++++++++++-------- tests/conftest.py | 13 +- .../{test-simple => test-simple-full}/conf.py | 0 .../index.rst | 0 tests/roots/test-simple-short/conf.py | 7 + tests/roots/test-simple-short/index.rst | 2 + tests/test_options.py | 31 ++- 11 files changed, 251 insertions(+), 148 deletions(-) rename tests/roots/{test-simple => test-simple-full}/conf.py (100%) rename tests/roots/{test-simple => test-simple-full}/index.rst (100%) create mode 100644 tests/roots/test-simple-short/conf.py create mode 100644 tests/roots/test-simple-short/index.rst diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 320fdf2..1dbdb30 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -1,17 +1,33 @@ -name: Test and Deploy +name: CI on: push: branches: - main + pull_request: + branches: + - main create: tags: - '*' jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Black + run: | + pip install black + black --check --exclude /docs --diff . test-extension: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.6', '3.7', '3.8', 'pypy3'] + python-version: ['3.6', '3.7', '3.8', '3.9', 'pypy3'] + sphinx-version: ['2', '3'] + name: "Test Extension - Python(${{ matrix.python-version }}), Sphinx(${{ matrix.sphinx-version }})" steps: - uses: actions/checkout@v2 - name: Setup Python @@ -25,6 +41,7 @@ jobs: python -m site python -m pip install --upgrade pip setuptools wheel python -m pip install -r dev-requirements.txt + python -m pip install sphinx==${{ matrix.sphinx-version }} - name: Run Tests for ${{ matrix.python-version }} run: | python -m pytest -vv diff --git a/README.md b/README.md index 4b3c5bf..6b9b920 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,21 @@ # sphinxext-remoteliteralinclude + +![CI](https://github.com/wpilibsuite/sphinxext-remoteliteralinclude/workflows/CI/badge.svg) + Sphinx extension that extends the ``literalinclude`` directive to allow remote URLS ## Installation +Please install the extension via pip using the following command: + ``python3 -m pip install sphinxext-remoteliteralinclude`` +then in your ``conf.py`` under ``extensions``, it should look like the following: + +```python +extensions = ["sphinxext.remoteliteralinclude"] +``` + ## Usage Simply just use it as you normally would a normal ``literalinclude`` diff --git a/dev-requirements.txt b/dev-requirements.txt index d22f86b..8a78e27 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,3 +1,4 @@ sphinx +six pytest==5.4.3 wheel==0.34.2 \ No newline at end of file diff --git a/setup.py b/setup.py index 6072ea5..b29718f 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,17 @@ import setuptools import subprocess -ret = subprocess.run("git describe --tags --abbrev=0", stdout=subprocess.PIPE, - stderr=subprocess.PIPE, check=True, shell=True) -version = ret.stdout.decode("utf-8").strip() +try: + ret = subprocess.run( + "git describe --tags --abbrev=0", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + shell=True, + ) + version = ret.stdout.decode("utf-8").strip() +except: + version = "main" with open("README.md", "r", encoding="utf-8") as fh: @@ -18,26 +26,26 @@ long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/wpilibsuite/sphinxext-remoteliteralinclude", - install_requires = ['sphinx>=2.0'], - packages=['sphinxext'], + install_requires=["sphinx>=2.0", "six"], + packages=["sphinxext"], classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Environment :: Plugins', - 'Environment :: Web Environment', - 'Framework :: Sphinx :: Extension', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Natural Language :: English', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python', - 'Topic :: Documentation :: Sphinx', - 'Topic :: Documentation', - 'Topic :: Software Development :: Documentation', - 'Topic :: Text Processing', - 'Topic :: Utilities', + "Development Status :: 5 - Production/Stable", + "Environment :: Plugins", + "Environment :: Web Environment", + "Framework :: Sphinx :: Extension", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python", + "Topic :: Documentation :: Sphinx", + "Topic :: Documentation", + "Topic :: Software Development :: Documentation", + "Topic :: Text Processing", + "Topic :: Utilities", ], - python_requires='>=3.4', + python_requires=">=3.4", ) diff --git a/sphinxext/remoteliteralinclude.py b/sphinxext/remoteliteralinclude.py index c9408a9..80fd554 100644 --- a/sphinxext/remoteliteralinclude.py +++ b/sphinxext/remoteliteralinclude.py @@ -23,29 +23,30 @@ from sphinx.domains import Domain + class RemoteLiteralIncludeReader(object): INVALID_OPTIONS_PAIR = [ - ('lineno-match', 'lineno-start'), - ('lineno-match', 'append'), - ('lineno-match', 'prepend'), - ('start-after', 'start-at'), - ('end-before', 'end-at'), - ('diff', 'pyobject'), - ('diff', 'lineno-start'), - ('diff', 'lineno-match'), - ('diff', 'lines'), - ('diff', 'start-after'), - ('diff', 'end-before'), - ('diff', 'start-at'), - ('diff', 'end-at'), + ("lineno-match", "lineno-start"), + ("lineno-match", "append"), + ("lineno-match", "prepend"), + ("start-after", "start-at"), + ("end-before", "end-at"), + ("diff", "pyobject"), + ("diff", "lineno-start"), + ("diff", "lineno-match"), + ("diff", "lines"), + ("diff", "start-after"), + ("diff", "end-before"), + ("diff", "start-at"), + ("diff", "end-at"), ] def __init__(self, url, options, config): # type: (unicode, Dict, Config) -> None self.url = url self.options = options - self.encoding = options.get('encoding', config.source_encoding) - self.lineno_start = self.options.get('lineno-start', 1) + self.encoding = options.get("encoding", config.source_encoding) + self.lineno_start = self.options.get("lineno-start", 1) self.parse_options() @@ -53,27 +54,30 @@ def parse_options(self): # type: () -> None for option1, option2 in self.INVALID_OPTIONS_PAIR: if option1 in self.options and option2 in self.options: - raise ValueError(__('Cannot use both "%s" and "%s" options') % - (option1, option2)) + raise ValueError( + __('Cannot use both "%s" and "%s" options') % (option1, option2) + ) def read_file(self, url, location=None): # type: (unicode, Any) -> List[unicode] # try: - # with codecs.open(url, 'r', self.encoding, errors='strict') as f: # type: ignore # NOQA - # text = f.read() # type: unicode + # with codecs.open(url, 'r', self.encoding, errors='strict') as f: # type: ignore # NOQA + # text = f.read() # type: unicode response = requests.get(url) text = response.text - + if text: if not response.status_code == requests.codes.ok: - raise ValueError('HTTP request returned error code %s' % response.status_code) - - if 'tab-width' in self.options: - text = text.expandtabs(self.options['tab-width']) + raise ValueError( + "HTTP request returned error code %s" % response.status_code + ) + + if "tab-width" in self.options: + text = text.expandtabs(self.options["tab-width"]) return text.splitlines(True) else: - raise IOError(__('Include file %r not found or reading it failed') % url) + raise IOError(__("Include file %r not found or reading it failed") % url) # except (IOError, OSError): # raise IOError(__('Include file %r not found or reading it failed') % url) # except UnicodeError: @@ -83,80 +87,93 @@ def read_file(self, url, location=None): def read(self, location=None): # type: (Any) -> Tuple[unicode, int] - if 'diff' in self.options: + if "diff" in self.options: lines = self.show_diff() else: - filters = [self.pyobject_filter, - self.start_filter, - self.end_filter, - self.lines_filter, - self.prepend_filter, - self.append_filter, - self.dedent_filter] + filters = [ + self.pyobject_filter, + self.start_filter, + self.end_filter, + self.lines_filter, + self.prepend_filter, + self.append_filter, + self.dedent_filter, + ] lines = self.read_file(self.url, location=location) for func in filters: lines = func(lines, location=location) - return ''.join(lines), len(lines) + return "".join(lines), len(lines) def show_diff(self, location=None): # type: (Any) -> List[unicode] new_lines = self.read_file(self.url) - old_url = self.options.get('diff') + old_url = self.options.get("diff") old_lines = self.read_file(old_url) diff = unified_diff(old_lines, new_lines, old_url, self.url) return list(diff) def pyobject_filter(self, lines, location=None): # type: (List[unicode], Any) -> List[unicode] - pyobject = self.options.get('pyobject') + pyobject = self.options.get("pyobject") if pyobject: from sphinx.pycode import ModuleAnalyzer - analyzer = ModuleAnalyzer.for_file(self.url, '') + + analyzer = ModuleAnalyzer.for_file(self.url, "") tags = analyzer.find_tags() if pyobject not in tags: - raise ValueError(__('Object named %r not found in include file %r') % - (pyobject, self.url)) + raise ValueError( + __("Object named %r not found in include file %r") + % (pyobject, self.url) + ) else: start = tags[pyobject][1] end = tags[pyobject][2] - lines = lines[start - 1:end] - if 'lineno-match' in self.options: + lines = lines[start - 1 : end] + if "lineno-match" in self.options: self.lineno_start = start return lines def lines_filter(self, lines, location=None): # type: (List[unicode], Any) -> List[unicode] - linespec = self.options.get('lines') + linespec = self.options.get("lines") if linespec: linelist = parselinenos(linespec, len(lines)) if any(i >= len(lines) for i in linelist): - raise ValueError('Line number spec is out of range (1 - %s)' % len(lines)) + raise ValueError( + "Line number spec is out of range (1 - %s)" % len(lines) + ) - if 'lineno-match' in self.options: + if "lineno-match" in self.options: # make sure the line list is not "disjoint". first = linelist[0] if all(first + i == n for i, n in enumerate(linelist)): self.lineno_start += linelist[0] else: - raise ValueError(__('Cannot use "lineno-match" with a disjoint ' - 'set of "lines"')) + raise ValueError( + __( + 'Cannot use "lineno-match" with a disjoint ' + 'set of "lines"' + ) + ) lines = [lines[n] for n in linelist if n < len(lines)] if lines == []: - raise ValueError(__('Line spec %r: no lines pulled from include file %r') % - (linespec, self.url)) + raise ValueError( + __("Line spec %r: no lines pulled from include file %r") + % (linespec, self.url) + ) return lines def start_filter(self, lines, location=None): # type: (List[unicode], Any) -> List[unicode] - if 'start-at' in self.options: - start = self.options.get('start-at') + if "start-at" in self.options: + start = self.options.get("start-at") inclusive = False - elif 'start-after' in self.options: - start = self.options.get('start-after') + elif "start-after" in self.options: + start = self.options.get("start-after") inclusive = True else: start = None @@ -165,30 +182,30 @@ def start_filter(self, lines, location=None): for lineno, line in enumerate(lines): if start in line: if inclusive: - if 'lineno-match' in self.options: + if "lineno-match" in self.options: self.lineno_start += lineno + 1 - return lines[lineno + 1:] + return lines[lineno + 1 :] else: - if 'lineno-match' in self.options: + if "lineno-match" in self.options: self.lineno_start += lineno return lines[lineno:] if inclusive is True: - raise ValueError('start-after pattern not found: %s' % start) + raise ValueError("start-after pattern not found: %s" % start) else: - raise ValueError('start-at pattern not found: %s' % start) + raise ValueError("start-at pattern not found: %s" % start) return lines def end_filter(self, lines, location=None): # type: (List[unicode], Any) -> List[unicode] - if 'end-at' in self.options: - end = self.options.get('end-at') + if "end-at" in self.options: + end = self.options.get("end-at") inclusive = True - elif 'end-before' in self.options: - end = self.options.get('end-before') + elif "end-before" in self.options: + end = self.options.get("end-before") inclusive = False else: end = None @@ -197,39 +214,39 @@ def end_filter(self, lines, location=None): for lineno, line in enumerate(lines): if end in line: if inclusive: - return lines[:lineno + 1] + return lines[: lineno + 1] else: if lineno == 0: return [] else: return lines[:lineno] if inclusive is True: - raise ValueError('end-at pattern not found: %s' % end) + raise ValueError("end-at pattern not found: %s" % end) else: - raise ValueError('end-before pattern not found: %s' % end) + raise ValueError("end-before pattern not found: %s" % end) return lines def prepend_filter(self, lines, location=None): # type: (List[unicode], Any) -> List[unicode] - prepend = self.options.get('prepend') + prepend = self.options.get("prepend") if prepend: - lines.insert(0, prepend + '\n') + lines.insert(0, prepend + "\n") return lines def append_filter(self, lines, location=None): # type: (List[unicode], Any) -> List[unicode] - append = self.options.get('append') + append = self.options.get("append") if append: - lines.append(append + '\n') + lines.append(append + "\n") return lines def dedent_filter(self, lines, location=None): # type: (List[unicode], Any) -> List[unicode] - if 'dedent' in self.options: - return dedent_lines(lines, self.options.get('dedent'), location=location) + if "dedent" in self.options: + return dedent_lines(lines, self.options.get("dedent"), location=location) else: return lines @@ -246,38 +263,39 @@ class RemoteLiteralInclude(SphinxDirective): optional_arguments = 0 final_argument_whitespace = True option_spec = { - 'dedent': int, - 'linenos': directives.flag, - 'lineno-start': int, - 'lineno-match': directives.flag, - 'tab-width': int, - 'language': directives.unchanged_required, - 'encoding': directives.encoding, - 'pyobject': directives.unchanged_required, - 'lines': directives.unchanged_required, - 'start-after': directives.unchanged_required, - 'end-before': directives.unchanged_required, - 'start-at': directives.unchanged_required, - 'end-at': directives.unchanged_required, - 'prepend': directives.unchanged_required, - 'append': directives.unchanged_required, - 'emphasize-lines': directives.unchanged_required, - 'caption': directives.unchanged, - 'class': directives.class_option, - 'name': directives.unchanged, - 'diff': directives.unchanged_required, + "dedent": int, + "linenos": directives.flag, + "lineno-start": int, + "lineno-match": directives.flag, + "tab-width": int, + "language": directives.unchanged_required, + "encoding": directives.encoding, + "pyobject": directives.unchanged_required, + "lines": directives.unchanged_required, + "start-after": directives.unchanged_required, + "end-before": directives.unchanged_required, + "start-at": directives.unchanged_required, + "end-at": directives.unchanged_required, + "prepend": directives.unchanged_required, + "append": directives.unchanged_required, + "emphasize-lines": directives.unchanged_required, + "caption": directives.unchanged, + "class": directives.class_option, + "name": directives.unchanged, + "diff": directives.unchanged_required, } def run(self): # type: () -> List[nodes.Node] document = self.state.document if not document.settings.file_insertion_enabled: - return [document.reporter.warning('File insertion disabled', - line=self.lineno)] + return [ + document.reporter.warning("File insertion disabled", line=self.lineno) + ] # convert options['diff'] to absolute path - if 'diff' in self.options: - _, path = self.env.relfn2path(self.options['diff']) - self.options['diff'] = path + if "diff" in self.options: + _, path = self.env.relfn2path(self.options["diff"]) + self.options["diff"] = path try: location = self.state_machine.get_source_and_line(self.lineno) @@ -288,26 +306,30 @@ def run(self): retnode = nodes.literal_block(text, text, source=url) set_source_info(self, retnode) - if self.options.get('diff'): # if diff is set, set udiff - retnode['language'] = 'udiff' - elif 'language' in self.options: - retnode['language'] = self.options['language'] - retnode['linenos'] = ('linenos' in self.options or - 'lineno-start' in self.options or - 'lineno-match' in self.options) - retnode['classes'] += self.options.get('class', []) - extra_args = retnode['highlight_args'] = {} - if 'emphasize-lines' in self.options: - hl_lines = parselinenos(self.options['emphasize-lines'], lines) + if self.options.get("diff"): # if diff is set, set udiff + retnode["language"] = "udiff" + elif "language" in self.options: + retnode["language"] = self.options["language"] + retnode["linenos"] = ( + "linenos" in self.options + or "lineno-start" in self.options + or "lineno-match" in self.options + ) + retnode["classes"] += self.options.get("class", []) + extra_args = retnode["highlight_args"] = {} + if "emphasize-lines" in self.options: + hl_lines = parselinenos(self.options["emphasize-lines"], lines) if any(i >= lines for i in hl_lines): - logger.warning(__('line number spec is out of range(1-%d): %r') % - (lines, self.options['emphasize-lines']), - location=location) - extra_args['hl_lines'] = [x + 1 for x in hl_lines if x < lines] - extra_args['linenostart'] = reader.lineno_start - - if 'caption' in self.options: - caption = self.options['caption'] or self.arguments[0] + logger.warning( + __("line number spec is out of range(1-%d): %r") + % (lines, self.options["emphasize-lines"]), + location=location, + ) + extra_args["hl_lines"] = [x + 1 for x in hl_lines if x < lines] + extra_args["linenostart"] = reader.lineno_start + + if "caption" in self.options: + caption = self.options["caption"] or self.arguments[0] retnode = container_wrapper(self, retnode, caption) # retnode will be note_implicit_target that is linked from caption and numref. @@ -318,5 +340,12 @@ def run(self): except Exception as exc: return [document.reporter.warning(text_type(exc), line=self.lineno)] + def setup(app): - directives.register_directive('rli', RemoteLiteralInclude) + directives.register_directive("rli", RemoteLiteralInclude) + directives.register_directive("remoteliteralinclude", RemoteLiteralInclude) + + return { + "parallel_read_safe": True, + "parallel_write_safe": False, + } diff --git a/tests/conftest.py b/tests/conftest.py index b8eacd1..1ba5f54 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,13 +2,20 @@ from sphinx.application import Sphinx from sphinx.testing.path import path + pytest_plugins = "sphinx.testing.fixtures" + @pytest.fixture(scope="session") def rootdir(): return path(__file__).parent.abspath() / "roots" + +@pytest.fixture() +def content(app): + app.build() + yield app + + def pytest_configure(config): - config.addinivalue_line( - "markers", "sphinx" - ) + config.addinivalue_line("markers", "sphinx") diff --git a/tests/roots/test-simple/conf.py b/tests/roots/test-simple-full/conf.py similarity index 100% rename from tests/roots/test-simple/conf.py rename to tests/roots/test-simple-full/conf.py diff --git a/tests/roots/test-simple/index.rst b/tests/roots/test-simple-full/index.rst similarity index 100% rename from tests/roots/test-simple/index.rst rename to tests/roots/test-simple-full/index.rst diff --git a/tests/roots/test-simple-short/conf.py b/tests/roots/test-simple-short/conf.py new file mode 100644 index 0000000..63656b4 --- /dev/null +++ b/tests/roots/test-simple-short/conf.py @@ -0,0 +1,7 @@ +extensions = ["sphinxext.remoteliteralinclude"] + +master_doc = "index" +exclude_patterns = ["_build"] + +# removes most of the HTML +html_theme = "basic" diff --git a/tests/roots/test-simple-short/index.rst b/tests/roots/test-simple-short/index.rst new file mode 100644 index 0000000..d629bd3 --- /dev/null +++ b/tests/roots/test-simple-short/index.rst @@ -0,0 +1,2 @@ +.. rli:: http://example.com/ + :lines: 4-4 \ No newline at end of file diff --git a/tests/test_options.py b/tests/test_options.py index 83d8e27..f3d9d46 100644 --- a/tests/test_options.py +++ b/tests/test_options.py @@ -1,11 +1,32 @@ import pytest +from sphinx.application import Sphinx +from sphinx import version_info -@pytest.mark.sphinx("html", testroot="simple") -def test_simple(app): - app.builder.build_all() - content = (app.outdir / 'index.html').read_text() +@pytest.mark.sphinx("html", testroot="simple-short") +def test_simple_short(app: Sphinx): + app.build() - html = ('Example Domain') + content = read_text(app) + + html = 'Example Domain' assert html in content + + +@pytest.mark.sphinx("html", testroot="simple-full") +def test_simple_full(app: Sphinx): + app.build() + + content = read_text(app) + + html = 'Example Domain' + + assert html in content + + +def read_text(app: Sphinx): + if version_info[:2] < (3, 0): + return (app.outdir / "index.html").text().replace("\n", "") + else: + return (app.outdir / "index.html").read_text()