Normalize tox, Makefile, setup.py with lim-cli

Author: davedittrich

.travis.yml

@@ -28,19 +28,23 @@ matrix:
       env: TOXENV=pypi
 
 
-before_script:
-  - git clone https://github.com/ztombol/bats-support.git ../bats-support
-  - git clone https://github.com/jasonkarns/bats-assert-1.git ../bats-assert-1
-
 branches:
   only:
     - master
     - develop
 
 # command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors
 install:
-  - pip install .
-  - pip install -r requirements.txt
+  - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
+  - bash miniconda.sh -b -p $HOME/miniconda
+  - export PATH=$HOME/miniconda/bin:$PATH
+  - source "$HOME/miniconda/etc/profile.d/conda.sh"
+  - hash -r
+  - conda config --set always_yes yes --set changeps1 no
+  - conda update --yes conda  # Update CONDA without command line prompt
+  - conda install -c conda-forge --yes tox
+  # Useful for debugging any issues with conda
+  - conda info -a
 
 # command to run tests, e.g. python setup.py test
 script:

Makefile

@@ -5,13 +5,42 @@ VERSION=$(shell cat VERSION)
 REQUIRED_VENV:=python_secrets
 VENV_DIR=$(HOME)/.virtualenvs/$(REQUIRED_VENV)
 PROJECT:=$(shell basename `pwd`)
+PYTHON=python3
 
 .PHONY: default
 default: all
 
 .PHONY: all
 all: install-active
 
+.PHONY: help
+help:
+	@echo 'usage: make [VARIABLE=value] [target [target..]]'
+	@echo ''
+	@echo 'test - generic target for both "test-tox" and "test-bats"'
+	@echo 'test-tox - run tox tests'
+	@echo 'test-bats - run Bats unit tests'
+	@echo 'test-bats-runtime - run Bats runtime integration/system tests'
+	@echo 'release - produce a pypi production release'
+	@echo 'release-test - produce a pypi test release'
+	@echo 'release-prep - final documentation preparations for release'
+	@echo 'sdist - run "$(PYTHON) setup.py sdist"'
+	@echo 'bdist_wheel - build a universal binary wheel'
+	@echo 'twine-check - run "twine check"'
+	@echo 'clean - remove build artifacts'
+	@echo 'spotless - deep clean'
+	@echo 'build-packet-cafe - Build and bring up packet_cafe containers'
+	@echo 'up-packet-cafe - Bring up packet_cafe containers'
+	@echo 'down-packet-cafe - Bring up packet_cafe containers'
+	@echo 'clean-packet-cafe - remove packet_cafe contents'
+	@echo 'spotless-packet-cafe - Remove all packet_cafe files and containers'
+	@echo 'install - install pip package'
+	@echo 'install-active - run "$(PYTHON) -m pip install -U ."'
+	@echo 'docs-tests - generate bats test output for documentation'
+	@echo 'docs-help - generate "lim help" output for documentation'
+	@echo 'docs - build Sphinx docs'
+
+
 #HELP test - run 'tox' for testing
 .PHONY: test
 test: test-tox
@@ -21,25 +50,25 @@ test: test-tox
 test-tox:
 	@if [ -f .python_secrets_environment ]; then (echo '[!] Remove .python_secrets_environment prior to testing'; exit 1); fi
 	touch docs/psec_help.txt
-	tox
+	@# See comment in tox.ini file.
+	tox -e pep8,bandit,docs && tox -e py36,py37,py38,bats,pypi
+	@-git checkout ChangeLog
 
 .PHONY: test-bats
 test-bats: bats-libraries
 	@if [ "$(TRAVIS)" != "true" ]; then \
 		if ! type bats 2>/dev/null >/dev/null; then \
 			echo "[-] Skipping bats tests"; \
 		else \
-			echo "[+] Running bats unit tests:"; \
-			(cd tests && ls -1 [0-9][0-9]*.bats); \
-			bats --tap tests/[0-9][0-9]*.bats; \
+			echo "[+] Running bats tests: $(shell cd tests && echo [0-9][0-9]*.bats)"; \
+			PYTHONWARNINGS="ignore" bats --tap tests/[0-9][0-9]*.bats; \
 		fi \
 	 fi
 
 .PHONY: test-bats-runtime
 test-bats-runtime: bats-libraries
-	@echo "[+] Running bats runtime tests:"
-	@cd tests && ls -1 runtime_[0-9][0-9]*.bats
-	bats --tap tests/runtime_*.bats || true
+	@echo "[+] Running bats runtime tests: $(shell cd tests && echo runtime_[0-9][0-9]*.bats)"; \
+	PYTHONWARNINGS="ignore" bats --tap tests/runtime_[0-9][0-9]*.bats
 
 .PHONY: no-diffs
 no-diffs:
@@ -48,39 +77,44 @@ no-diffs:
 
 #HELP release - package and upload a release to pypi
 .PHONY: release
-release: clean twine-check
+release: clean docs sdist bdist_wheel twine-check
 	twine upload $(shell cat dist/.LATEST_*) -r pypi
 
+#HELP release-prep - final documentation preparations for release
+.PHONY: release-prep
+release-prep: install-active clean sdist docs-help docs-tests
+	@echo 'Check in help text docs and ChangeLog?'
+
 #HELP release-test - upload to "testpypi"
 .PHONY: release-test
 release-test: clean test docs-tests docs twine-check
 	$(MAKE) no-diffs
 	twine upload $(shell cat dist/.LATEST_*) -r testpypi
 
+#HELP sdist - build a source package
+.PHONY: sdist
+sdist: clean-docs docs
+	rm -f dist/.LATEST_SDIST
+	$(PYTHON) setup.py sdist
+	ls -t dist/*.tar.gz 2>/dev/null | head -n 1 > dist/.LATEST_SDIST
+	ls -l dist/*.tar.gz
+
 #HELP bdist_egg - build an egg package
 .PHONY: bdist_egg
 bdist_egg:
 	rm -f dist/.LATEST_EGG
-	python setup.py bdist_egg
+	$(PYTHON) setup.py bdist_egg
 	ls -t dist/*.egg 2>/dev/null | head -n 1 > dist/.LATEST_EGG
 	ls -lt dist/*.egg
 
 #HELP bdist_wheel - build a wheel package
 .PHONY: bdist_wheel
 bdist_wheel:
 	rm -f dist/.LATEST_WHEEL
-	python setup.py bdist_wheel
+	$(PYTHON) setup.py bdist_wheel
 	ls -t dist/*.whl 2>/dev/null | head -n 1 > dist/.LATEST_WHEEL
 	ls -lt dist/*.whl
 
-#HELP sdist - build a source package
-.PHONY: sdist
-sdist: clean-docs docs
-	rm -f dist/.LATEST_SDIST
-	python setup.py sdist
-	ls -t dist/*.tar.gz 2>/dev/null | head -n 1 > dist/.LATEST_SDIST
-	ls -l dist/*.tar.gz
-
 #HELP twine-check
 .PHONY: twine-check
 twine-check: sdist bdist_egg bdist_wheel
@@ -89,7 +123,7 @@ twine-check: sdist bdist_egg bdist_wheel
 #HELP clean - remove build artifacts
 .PHONY: clean
 clean: clean-docs
-	python setup.py clean
+	$(PYTHON) setup.py clean
 	rm -rf dist build *.egg-info
 	find . -name '*.pyc' -delete
 
@@ -109,23 +143,36 @@ install:
 		exit 1; \
 	else \
 		echo "Installing into $(REQUIRED_VENV) virtual environment"; \
-		$(VENV_DIR)/bin/python -m pip uninstall -y $(PROJECT); \
-		$(VENV_DIR)/bin/python setup.py install; \
+		$(VENV_DIR)/bin/$(PYTHON) -m pip uninstall -y $(PROJECT); \
+		$(VENV_DIR)/bin/$(PYTHON) setup.py install; \
 	fi
 
 #HELP install-active - install in the active Python virtual environment
+.PHONY: i
 .PHONY: install-active
-install-active:
-	python -m pip install -U .
+i install-active: bdist_wheel
+	$(PYTHON) -m pip uninstall -y $(PROJECT)
+	$(PYTHON) -m pip install -U "dist/$(shell cat dist/.LATEST_WHEEL)" | grep -v 'already satisfied'
+	git checkout ChangeLog
 
 #HELP docs-tests - generate bats test output for documentation
 .PHONY: docs-tests
 PR=pr --omit-header --omit-pagination --page-width 80
 docs-tests:
+	$(MAKE) -B docs/test-tox.txt
+	$(MAKE) -B docs/test-bats.txt
+	$(MAKE) -B docs/test-bats-runtime.txt
+
+docs/test-tox.txt:
 	(echo '$$ make test-tox' && $(MAKE) test-tox) |\
 	       $(PR) | tee docs/test-tox.txt
+
+docs/test-bats.txt:
 	$(MAKE) test-bats | $(PR) | tee docs/test-bats.txt
 
+docs/test-bats-runtime.txt:
+	(echo '$$ make test-bats-runtime' && $(MAKE) test-bats-runtime) |\
+	       $(PR) | tee docs/test-bats-runtime.txt
 
 #HELP docs - build Sphinx docs (NOT INTEGRATED YET FROM OPENSTACK CODE BASE)
 .PHONY: docs
@@ -135,6 +182,11 @@ docs: docs/psec_help.txt
 docs/psec_help.txt: install-active
 	psec help | tee docs/psec_help.txt
 
+#HELP examples - produce some example output for docs
+.PHONY: examples
+examples:
+	psec --help
+
 # Git submodules and subtrees are both a huge PITA. This is way simpler.
 
 .PHONY: bats-libraries

setup.py

@@ -82,9 +82,9 @@ def get_absolute_path(*args):
         'Intended Audience :: Other Audience',
         'Intended Audience :: Science/Research',
         'Intended Audience :: System Administrators',
-        'Operating System :: OS Independent',
         'License :: OSI Approved :: Apache Software License',
         'Natural Language :: English',
+        'Operating System :: OS Independent',
         'Programming Language :: Python :: 3',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',

tox.ini

@@ -1,9 +1,13 @@
 [tox]
-envlist = py36,py37,py38,py39,pep8,bandit,bats,docs,pypi
+# In practice, you can optimize by first running basic tests
+# 'pep8,bandit,docs' and only after those succeed go on to
+# run the remaining (default) tests. E.g.,
+# $ tox -e pep8,bandit,docs && tox -e py36,py37,py38,bats,pypi
+
+# envlist = pep8,bandit,docs,py36,py37,py38,bats,pypi
+envlist = py36,py37,py38,bats,pypi
 skip_missing_interpreters = true
-requires =
-  setuptools >= 40.9.0
-  pip >= 20.1
+requires = tox-conda
 
 [testenv]
 setenv =
@@ -12,9 +16,17 @@ setenv =
     PYTHONPATH={toxinidir}:{toxinidir}/psec
 distribute = False
 install_command = python -m pip install {opts} {packages}
-deps = -r{toxinidir}/requirements.txt
+conda_deps =
+       pytest
+conda_channels =
+       conda-forge
+# Make sure these match setup.py!
+deps = -Ur{toxinidir}/requirements.txt
+       setuptools>=40.9.0
+       pbr>=5.4.5
+       pip>=20.2.2
        pytest
-commands = pytest
+commands = pytest {posargs}
 
 ; If you want to make tox run the tests with the same versions, create a
 ; requirements.txt with the pinned versions and uncomment the following lines:
@@ -28,15 +40,21 @@ commands = make twine-check
 
 [testenv:pep8]
 basepython = python3.8
-deps = flake8
+deps = flake8>=3.8.3
 commands = flake8 psec docs/conf.py setup.py
 
 [testenv:bandit]
 basepython = python3.8
 ; Run security linter
-deps = bandit
+deps = bandit>=1.1.0
 commands = bandit -c bandit.yaml -r psec -x tests -n5
 
+[testenv:docs]
+basepython = python3.8
+deps = -Ur{toxinidir}/requirements.txt
+       sphinx
+commands = sphinx-build -b html docs docs/_build
+
 [testenv:bats]
 ; Run bats unit tests
 ; Deal with this by requiring docutils==0.15:
@@ -50,12 +68,8 @@ commands = bandit -c bandit.yaml -r psec -x tests -n5
 ; #     File "/Users/dittrich/git/python_secrets/.tox/bats/lib/python3.7/site-packages/pkg_resources/__init__.py", line 792, in resolve
 ; #       raise VersionConflict(dist, req).with_context(dependent_req)
 ; #   pkg_resources.ContextualVersionConflict: (docutils 0.16 (/Users/dittrich/git/python_secrets/.tox/bats/lib/python3.7/site-packages), Requirement.parse('docutils<0.16,>=0.10'), {'botocore'})
-deps = docutils==0.15
+deps = -Ur{toxinidir}/requirements.txt
+       docutils==0.15
 whitelist_externals = make
 commands = make test-bats
 
-[testenv:docs]
-basepython = python3.8
-deps = -Ur{toxinidir}/requirements.txt
-       sphinx
-commands = sphinx-build -b html docs docs/build/html