Use python-version-specific requirements.txt instead of single generic requirements.txt

.github/workflows/ci.yml

@@ -35,8 +35,12 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install tox
       run: pip install tox
+    - name: Set TOXENV for current python
+      run: |
+        TOXENV=$(python -c "import sys; print(f'py{sys.version_info.major}{sys.version_info.minor}')")
+        echo "TOXENV=$TOXENV" >> "$GITHUB_ENV"
     - name: Run the tests
-      run: tox -e py
+      run: tox
   deploy:
     # only run if the commit is tagged...
     if: startsWith(github.ref, 'refs/tags/v')

.gitignore

@@ -6,3 +6,4 @@ build/
 dist/
 .eggs/
 src/picosvg/_version.py
+.tox/

requirements/dev-requirements.in

@@ -0,0 +1,9 @@
+# This file only declares the top-level development requirements.
+# Run `pip-compile-all.sh` to generate the concrete py3*-requirements.txt
+# files that are used to set up the tox and CI environments.
+
+black
+pytest
+# As of November 2020, pytype requires: Python <3.9, >=3.6
+# https://pypi.org/project/pytype/2020.11.23
+pytype; python_version < '3.9'

requirements/install-requirements.in

@@ -0,0 +1,6 @@
+# Only list here top-level runtime dependencies.
+# Run `pip-compile-all.sh` to generate the concrete py3*-requirements.txt
+# files that are used to set up the tox and CI environments.
+dataclasses>=0.7; python_version < '3.7'
+lxml>=4.0
+skia-pathops>=0.4.1

requirements/pip-compile-all.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+# Runs pip-compile to freeze requirements.txt for all supported pythons
+# that are listed in the tox.ini default envlist.
+# It is recommended to run this every time any top-level requirements in either
+# install-requirements.in or dev-requirements.in are added, removed or changed.
+# The script must be run from the same directory where tox.ini file is located,
+# and it requires that all the supported python3.X binaries are installed
+# locally and available on $PATH.
+# It also requires that the venv module is present in all of them, in order to
+# create the temporary virtual environment where to install pip-compile.
+# On most python distributions venv is part of the standard library, however on
+# some Linux distros (e.g. Debian) it needs to be installed separately.
+
+set -e
+
+TMPDIR="$(mktemp -d)"
+
+function compile_requirements {
+    local python_cmd=${1}
+    echo "Updating ${python_cmd}-requirements.txt"
+
+    "${python_cmd}" -m venv "${TMPDIR}/${python_cmd}-venv"
+
+    local venv_bin="${TMPDIR}/${python_cmd}-venv/bin"
+    local pip_cmd="${venv_bin}/pip"
+    "${pip_cmd}" install -qq pip-tools
+
+    local pip_compile_cmd="${venv_bin}/pip-compile"
+    "${pip_compile_cmd}" -q --upgrade \
+        -o requirements/${python_cmd}-requirements.txt \
+        requirements/install-requirements.in \
+        requirements/dev-requirements.in
+}
+
+[ -f "tox.ini" ] || { echo "ERROR: tox.ini file not found" ; exit 1; }
+
+running=false
+# `tox -l` prints all the environments listed in the tox.ini's default 'envlist'
+for toxenv in $(tox -l); do
+    if [[ $toxenv =~ py([0-9])([0-9]+) ]]; then
+        version_major=${BASH_REMATCH[1]}
+        version_minor=${BASH_REMATCH[2]}
+        compile_requirements "python${version_major}.${version_minor}" &
+        running=true
+    fi
+done
+
+if $running; then
+    sleep 0.5
+    echo "Please wait while all the requirements files are updated..."
+    wait
+    echo "Done!"
+fi
+
+# clean up after us before leaving
+rm -r "${TMPDIR}"

requirements/python3.6-requirements.txt

@@ -0,0 +1,34 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --output-file=requirements/python3.6-requirements.txt requirements/dev-requirements.in requirements/install-requirements.in
+#
+appdirs==1.4.4            # via black
+attrs==20.3.0             # via pytest, pytype
+black==20.8b1             # via -r requirements/dev-requirements.in
+click==7.1.2              # via black
+dataclasses==0.8 ; python_version < "3.7"  # via -r requirements/install-requirements.in, black
+decorator==4.4.2          # via networkx
+importlab==0.5.1          # via pytype
+importlib-metadata==3.1.0  # via pluggy, pytest
+iniconfig==1.1.1          # via pytest
+lxml==4.6.2               # via -r requirements/install-requirements.in
+mypy-extensions==0.4.3    # via black
+networkx==2.5             # via importlab
+ninja==1.10.0.post2       # via pytype
+packaging==20.7           # via pytest
+pathspec==0.8.1           # via black
+pluggy==0.13.1            # via pytest
+py==1.9.0                 # via pytest
+pyparsing==2.4.7          # via packaging
+pytest==6.1.2             # via -r requirements/dev-requirements.in
+pytype==2020.11.23 ; python_version < "3.9"  # via -r requirements/dev-requirements.in
+pyyaml==5.3.1             # via pytype
+regex==2020.11.13         # via black
+six==1.15.0               # via importlab, pytype
+skia-pathops==0.5.1.post1  # via -r requirements/install-requirements.in
+toml==0.10.2              # via black, pytest
+typed-ast==1.4.1          # via black, pytype
+typing-extensions==3.7.4.3  # via black
+zipp==3.4.0               # via importlib-metadata

requirements/python3.7-requirements.txt

@@ -0,0 +1,33 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --output-file=requirements/python3.7-requirements.txt requirements/dev-requirements.in requirements/install-requirements.in
+#
+appdirs==1.4.4            # via black
+attrs==20.3.0             # via pytest, pytype
+black==20.8b1             # via -r requirements/dev-requirements.in
+click==7.1.2              # via black
+decorator==4.4.2          # via networkx
+importlab==0.5.1          # via pytype
+importlib-metadata==3.1.0  # via pluggy, pytest
+iniconfig==1.1.1          # via pytest
+lxml==4.6.2               # via -r requirements/install-requirements.in
+mypy-extensions==0.4.3    # via black
+networkx==2.5             # via importlab
+ninja==1.10.0.post2       # via pytype
+packaging==20.7           # via pytest
+pathspec==0.8.1           # via black
+pluggy==0.13.1            # via pytest
+py==1.9.0                 # via pytest
+pyparsing==2.4.7          # via packaging
+pytest==6.1.2             # via -r requirements/dev-requirements.in
+pytype==2020.11.23 ; python_version < "3.9"  # via -r requirements/dev-requirements.in
+pyyaml==5.3.1             # via pytype
+regex==2020.11.13         # via black
+six==1.15.0               # via importlab, pytype
+skia-pathops==0.5.1.post1  # via -r requirements/install-requirements.in
+toml==0.10.2              # via black, pytest
+typed-ast==1.4.1          # via black, pytype
+typing-extensions==3.7.4.3  # via black
+zipp==3.4.0               # via importlib-metadata

requirements/python3.8-requirements.txt

@@ -0,0 +1,31 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --output-file=requirements/python3.8-requirements.txt requirements/dev-requirements.in requirements/install-requirements.in
+#
+appdirs==1.4.4            # via black
+attrs==20.3.0             # via pytest, pytype
+black==20.8b1             # via -r requirements/dev-requirements.in
+click==7.1.2              # via black
+decorator==4.4.2          # via networkx
+importlab==0.5.1          # via pytype
+iniconfig==1.1.1          # via pytest
+lxml==4.6.2               # via -r requirements/install-requirements.in
+mypy-extensions==0.4.3    # via black
+networkx==2.5             # via importlab
+ninja==1.10.0.post2       # via pytype
+packaging==20.7           # via pytest
+pathspec==0.8.1           # via black
+pluggy==0.13.1            # via pytest
+py==1.9.0                 # via pytest
+pyparsing==2.4.7          # via packaging
+pytest==6.1.2             # via -r requirements/dev-requirements.in
+pytype==2020.11.23 ; python_version < "3.9"  # via -r requirements/dev-requirements.in
+pyyaml==5.3.1             # via pytype
+regex==2020.11.13         # via black
+six==1.15.0               # via importlab, pytype
+skia-pathops==0.5.1.post1  # via -r requirements/install-requirements.in
+toml==0.10.2              # via black, pytest
+typed-ast==1.4.1          # via black, pytype
+typing-extensions==3.7.4.3  # via black

requirements/python3.9-requirements.txt

@@ -0,0 +1,24 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --output-file=requirements/python3.9-requirements.txt requirements/dev-requirements.in requirements/install-requirements.in
+#
+appdirs==1.4.4            # via black
+attrs==20.3.0             # via pytest
+black==20.8b1             # via -r requirements/dev-requirements.in
+click==7.1.2              # via black
+iniconfig==1.1.1          # via pytest
+lxml==4.6.2               # via -r requirements/install-requirements.in
+mypy-extensions==0.4.3    # via black
+packaging==20.7           # via pytest
+pathspec==0.8.1           # via black
+pluggy==0.13.1            # via pytest
+py==1.9.0                 # via pytest
+pyparsing==2.4.7          # via packaging
+pytest==6.1.2             # via -r requirements/dev-requirements.in
+regex==2020.11.13         # via black
+skia-pathops==0.5.1.post1  # via -r requirements/install-requirements.in
+toml==0.10.2              # via black, pytest
+typed-ast==1.4.1          # via black
+typing-extensions==3.7.4.3  # via black

setup.py

@@ -12,11 +12,36 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Just a toy, enough setuptools to be able to install.
-"""
 from setuptools import setup, find_packages
+import os.path
 
-setup(
+
+def readlines(filename):
+    # Return a file's list of lines excluding # comments
+    lines = []
+    with open(filename, "r") as fp:
+        for line in fp:
+            line, _, _ = line.partition("#")
+            line = line.strip()
+            if not line:
+                continue
+            lines.append(line)
+    return lines
+
+
+# Store top-level depedencies in external requirements.in files, so that
+# pip-compile can use them to compile requirements.txt files with full
+# dependency graph exploded and all versions pinned (for reproducible tests).
+# pip-compile support for setup.py is quite limited: it ignores extras_require,
+# as well as environment markers from install_requires:
+# https://github.com/jazzband/pip-tools/issues/625
+# https://github.com/jazzband/pip-tools/issues/908
+# https://github.com/jazzband/pip-tools/issues/1139
+install_deps = readlines(os.path.join("requirements", "install-requirements.in"))
+develop_deps = readlines(os.path.join("requirements", "dev-requirements.in"))
+
+
+setup_args = dict(
     name="picosvg",
     use_scm_version={"write_to": "src/picosvg/_version.py"},
     package_dir={'': 'src'},
@@ -26,13 +51,11 @@
             'picosvg=picosvg.picosvg:main',
         ],
     },
-
     setup_requires=["setuptools_scm"],
-    install_requires=[
-        "dataclasses>=0.7; python_version < '3.7'",
-        "lxml>=4.0",
-        "skia-pathops>=0.4.1",
-    ],
+    install_requires=install_deps,
+    extras_require={
+        "dev": develop_deps,
+    },
     python_requires=">=3.6",
 
     # this is for type checker to use our inline type hints:
@@ -47,3 +70,7 @@
         "meant for use playing with COLR fonts"
     ),
 )
+
+
+if __name__ == "__main__":
+    setup(**setup_args)

tox.ini

@@ -6,32 +6,39 @@
 ;     # Only runs the linter checks
 ;   $ tox -e py38
 ;     # Runs all tests against python3.8 only
-;   $ tox -e py
-;     # Runs tests against the python interpreter where tox itself was installed
+;   $ export TOXENV=py39
+;   $ tox
+;     # If present use $TOXENV environment variable
 envlist = lint, py3{6,7,8,9}
 
 ; if any of the requested python interpreters is unavailable (e.g. on the local dev
 ; workstation), the tests are skipped and tox won't return an error
 skip_missing_interpreters = true
 
 [testenv]
+description = Run pytest against the specified python version
+; if the tox environment name contains `dev` (e.g. `py39-dev`), pip install the
+; current package in editable mode (-e .)
+usedevelop =
+    dev: true
+    !dev: false
+; install pinned dependencies specific for each python version: e.g. if tox env
+; is named `py39`, it will install from `python3.9-requirements.txt` file, etc.
 deps =
-    -r dev-requirements.txt
-    -r requirements.txt
+    -r requirements/{basepython}-requirements.txt
 ; downloads the latest pip, setuptools and wheel when creating the venv
 download = true
 ; any arguments passed to tox command line after the '--' separator are passed through
-; to pytest: e.g. `tox -e py -- -vv --lf -x`
+; to pytest: e.g. `tox -e py39 -- -vv --lf -x`
 commands =
     pytest {posargs}
 
 [testenv:lint]
-; no need to install current package for static linter, formatter, typechecker, etc.
+description = Check python style and typing annotation
+; no need to install current package for static analysis tool
 skip_install = true
 ; use the latest python that pytype supports
 basepython = python3.8
-deps =
-    -r dev-requirements.txt
 commands =
     black --check --diff src tests
     pytype