Workflow improvements (#1066)

* Added Makefile to easily run tests and improve code quality 
* Fix build for MAD docker image
 * Update development build to use the MAD docker image

.gitignore

@@ -23,6 +23,7 @@ configs/coords/*
 status.json
 update_log.json
 commands.json
+.tox/
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

.gitlint

@@ -0,0 +1,8 @@
+[general]
+ignore=title-trailing-punctuation, title-must-not-contain-word
+
+[title-max-length]
+line-length=120
+
+[body-max-line-length]
+line-length=140

.gitmessage

@@ -0,0 +1,5 @@
+# (ISSUE IF APPLICABLE) SHORT_DESCRIPTION
+
+# LONG_DESCRIPTION
+#   * Thing 1
+#   * Thing 2

.isort.cfg

@@ -0,0 +1,2 @@
+[settings]
+known_third_party = PIL,aioconsole,apkutils,bitstring,configargparse,cv2,dataclasses,flask,flask_caching,gevent,google,gpapi,gpxdata,imutils,loguru,mysql,numpy,pkg_resources,psutil,pytesseract,pytest,requests,s2sphere,urllib3,websockets,werkzeug

.pre-commit-config.yaml

@@ -0,0 +1,34 @@
+repos:
+  - repo: https://github.com/Seeefo/manage-commit-msg.git
+    rev: 1.0.3
+    hooks:
+      - id: restore-commit-msg
+  - repo: https://github.com/jorisroovers/gitlint
+    rev: v0.13.1
+    hooks:
+      - id: gitlint
+  - repo: https://github.com/Seeefo/manage-commit-msg.git
+    rev: 1.0.3
+    hooks:
+      - id: save-commit-msg
+  - repo: https://gitlab.com/pycqa/flake8
+    rev: 3.7.9
+    hooks:
+      - id: flake8
+        language_version: python3
+        additional_dependencies: [flake8-bugbear, flake8-logging-format, flake8-variables-names, pep8-naming, flake8-eradicate]
+  - repo: https://github.com/asottile/seed-isort-config
+    rev: v2.1.0
+    hooks:
+      - id: seed-isort-config
+        language_version: python3
+  - repo: https://github.com/pre-commit/mirrors-isort
+    rev: v4.3.21
+    hooks:
+      - id: isort
+        language_version: python3
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.5.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer

Makefile

@@ -0,0 +1,149 @@
+CMD ?= bash
+CONTAINER_NAME ?= mapadroid-dev
+include docker/.dev.env
+export
+
+define PIP_MISSING
+Pip is missing or not available in PATH. If pip is not installed
+instructions can be found at https://pip.pypa.io/en/stable/installing/
+endef
+
+define PRE_COMMIT_MISSING
+Install pre-commit @ https://pre-commit.com/#install then run
+source ~/.profile
+endef
+
+define DOCKER_MISSING
+Docker installation is missing or not available. This could be caused by
+not having docker installed or the user does not have access to docker.
+Installation instructions can be found at
+https://docs.docker.com/get-docker/
+endef
+
+define DOCKER_NOT_RUNNING
+Docker is not running or the user does not have access to the Docker
+Engine. Please verify that its running and you have access. On *nix
+systems you can run the following commands to grant access:
+sudo groupadd docker
+sudo usermod -aG docker $USER
+newgrp docker
+endef
+
+define DOCKER_COMPOSE_MISSING
+docker-compose -f ${COMPOSE_FILE_DEV} is not installed or not available in PATH. Installation
+instructions can be found at https://docs.docker.com/compose/install/
+If docker-compose -f ${COMPOSE_FILE_DEV} is installed PATH needs to be corrected to include
+the binary
+endef
+
+define DOCKER_COMPOSE_OLD
+docker-compose -f ${COMPOSE_FILE_DEV} is too old. Update docker-compose -f ${COMPOSE_FILE_DEV} from the instructions at
+https://docs.docker.com/compose/install/
+endef
+
+# Windows defines OS but *nix does not
+ifdef OS
+    SHELL := powershell.exe
+    pip := $(shell Get-Command pip | Select-Object -ExpandProperty Source)
+    precommit := $(shell Get-Command pre-commit | Select-Object -ExpandProperty Source)
+    docker := $(shell Get-Command docker | Select-Object -ExpandProperty Source)
+    docker_compose := $(shell Get-Command docker-compose -f ${COMPOSE_FILE_DEV} | Select-Object -ExpandProperty Source)
+    UID ?= 1000
+    GID ?= 1000
+else
+    ifneq ($(shell which pip),)
+        pip := $(shell which pip)
+    else ifneq ($(shell which pip3),)
+        pip := $(shell which pip3)
+    else
+        $(error $(PIP_MISSING))
+    endif
+    precommit := $(shell which pre-commit)
+    docker := $(shell which docker)
+    docker_compose := $(shell which docker-compose -f ${COMPOSE_FILE_DEV})
+    UID ?= $(shell id -u)
+    GID ?= $(shell id -g)
+endif
+
+
+ifeq (, $(pip))
+    $(error $(PIP_MISSING))
+endif
+ifneq ($(VIRTUAL_ENV), )
+    pip_precommit_installation := $(pip) install pre-commit
+else
+    pip_precommit_installation := $(pip) install --user pre-commit
+endif
+ifeq (, $(precommit))
+    $(error $(PRE_COMMIT_MISSING))
+endif
+ifeq (, $(docker))
+    $(error $(DOCKER_MISSING))
+endif
+ifeq (, $(shell docker info))
+    $(error $(DOCKER_NOT_RUNNING))
+endif
+ifeq (, $(docker_compose))
+    $(error $(DOCKER_COMPOSE_MISSING))
+endif
+compose_ver ?= $(shell docker-compose -f ${COMPOSE_FILE_DEV} --version | cut -d' ' -f3 | cut -d '.' -f2)
+ifeq (, compose_ver < 27)
+    $(error $(DOCKER_COMPOSE_OLD))
+endif
+
+clean: clean-tox down
+
+clean-tox:
+	rm -rf .tox
+
+build:
+	docker build --file docker/Dockerfile --tag ${LOCAL_MAD_IMAGE} .
+	docker-compose -f ${COMPOSE_FILE_DEV} build --no-cache
+
+rebuild:
+	docker build --file docker/Dockerfile --tag ${LOCAL_MAD_IMAGE} .
+	docker-compose -f ${COMPOSE_FILE_DEV} build
+
+setup-precommit:
+	$(pip_precommit_installation)
+	pre-commit install
+	pre-commit install --hook-type commit-msg
+
+setup: setup-precommit
+	git config commit.template .gitmessage
+
+up:
+	docker-compose -f ${COMPOSE_FILE_DEV} up --detach
+
+shell: up
+	docker-compose -f ${COMPOSE_FILE_DEV} exec -u $(UID) $(CONTAINER_NAME) $(CMD)
+
+root-shell: up
+	docker-compose -f ${COMPOSE_FILE_DEV} exec -u root $(CONTAINER_NAME) $(CMD)
+
+down:
+	docker-compose -f ${COMPOSE_FILE_DEV} down
+
+tests: up
+	docker-compose -f ${COMPOSE_FILE_DEV} exec -u $(UID) mapadroid-dev tox
+
+unittests: up
+	docker-compose -f ${COMPOSE_FILE_DEV} exec -u $(UID) mapadroid-dev tox -e py37
+
+# Run bash within a defined tox environment
+# Specify a valid tox environment as such:
+#       make shell-py37
+# To force a recreation of the environment, specify the RECREATE environment variable with any value
+#   make shell-py37 RECREATE=1
+shell-%: up
+ifdef RECREATE
+	docker-compose -f ${COMPOSE_FILE_DEV} exec -u $(UID) mapadroid-dev tox -e $* --recreate -- bash
+else
+	docker-compose -f ${COMPOSE_FILE_DEV} exec -u $(UID) mapadroid-dev tox -e $* -- bash
+endif
+
+versions:
+	$(pip) --version
+	$(precommit) --version
+	$(docker) --version
+	$(docker_compose) --version

docker/.dev.env

@@ -0,0 +1,16 @@
+COMPOSE_CONVERT_WINDOWS_PATHS=1
+COMPOSE_FILE_DEV=docker/docker-compose-dev.yaml
+DOCKER_USER=dockeruser
+
+LOCAL_MAD_IMAGE=local_mad_production
+LOCAL_MAD_DEV_IMAGE=local_mad_development
+
+MYSQL_ROOT_PASSWORD=password
+MYSQL_DATABASE=rocketdb
+MYSQL_USER=rocketdb
+MYSQL_PASSWORD=rocketdb
+
+THERAIDMAPPER_DBIP=mariadb
+THERAIDMAPPER_DBUSERNAME=${MYSQL_USER}
+THERAIDMAPPER_DBNAME=${MYSQL_DATABASE}
+THERAIDMAPPER_DBPASSWORD=${MYSQL_USER}

.dockerignore → docker/.dockerignore

@@ -6,3 +6,4 @@ temp/
 files/
 upload/
 update_log.json
+docker/

Dockerfile → docker/Dockerfile

@@ -1,10 +1,11 @@
 ############################
 # MAD
 ############################
-FROM python:3.7-slim
+FROM python:3.7-slim AS mad-core
 # Working directory for the application
 WORKDIR /usr/src/app
 
+
 # copy requirements only, to reduce image size and improve cache usage
 COPY requirements.txt /usr/src/app/
 

docker/Dockerfile-dev

@@ -0,0 +1,39 @@
+# Development env
+FROM local_mad_production:latest AS dev_test
+RUN pip install tox
+# Versions of python to install for pyenv. These are used when tox executes specific
+# python versions. The correct versions need to be added to tox.ini under tox/envlist
+ENV PYTHON_VERSIONS 3.6.0 3.7.0 3.8.0
+COPY requirements-test.txt /usr/src/app/
+ENTRYPOINT ["bash"]
+# Need to re-add some required dependencies for tox to compile the new envs
+RUN apt-get install -y --no-install-recommends \
+# pyenv
+build-essential libssl-dev zlib1g-dev libbz2-dev \
+libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \
+xz-utils tk-dev libffi-dev liblzma-dev python-openssl git \
+# python build
+libffi-dev libgdbm-dev libsqlite3-dev libssl-dev zlib1g-dev
+
+# Map the user to avoid perm conflict
+ARG USER_NAME=dockeruser
+ARG UID=1000
+ARG GID=1000
+RUN groupadd -g $GID $USER_NAME; useradd -l -r -m -u $UID -g $GID $USER_NAME
+ENV USER $USER_NAME
+# Install pyenv
+# @TODO - How to install as a user and not root?
+ENV HOME=/home/dockeruser
+ENV PYENV_ROOT $HOME/.pyenv
+ENV PATH="$PYENV_ROOT/bin:$PATH"
+ENV PATH="$PYENV_ROOT/shims:$PATH"
+RUN curl -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash
+RUN chown -R dockeruser:dockeruser -R $HOME
+RUN for version in $PYTHON_VERSIONS; do \
+      pyenv install $version; \
+      pyenv local $version; \
+      pip install --upgrade setuptools pip; \
+      pyenv local --unset; \
+    done
+RUN echo "pyenv local $PYTHON_VERSIONS" >> ~/.bashrc
+RUN pyenv local $PYTHON_VERSIONS

docker-compose-dev.yaml → docker/docker-compose-dev.yaml

@@ -1,33 +1,36 @@
-version: "2.4"
+version: "3.4"
+
+volumes:
+  pre-commit-cache:
+    name: pre-commit-cache
 
 services:
-  mad:
-    image: map-a-droid
-    build: .
+  mapadroid-dev:
+    image: mapadroid-dev
+    build:
+      context: ../.
+      dockerfile: docker/Dockerfile-dev
     # Use an empty entrypoint and mount local sources into container so that a simple
     # `docker-compose run --service-ports python3 start.py` can be used to test current
     # development in a defined environment.
     # `docker-compose run bash` launch a shell inside the container.
     entrypoint: ""
     volumes:
-      - ./:/usr/src/app:rw
+      - ../:/usr/src/app:rw
+      - pre-commit-cache:/home/${DOCKER_USER}/.cache/pre-commit
     depends_on:
       - mariadb
     ports:
       - "5000:5000"
       - "8000:8000"
       - "8080:8080"
-    environment:
-      THERAIDMAPPER_DBIP: mariadb
-      THERAIDMAPPER_DBUSERNAME: rocketdb
-      THERAIDMAPPER_DBNAME: rocketdb
-      THERAIDMAPPER_DBPASSWORD: rocketdb"
+    env_file:
+      - .dev.env
+    tty: true
+    command: ["bash"]
 
   mariadb:
     image: mariadb:10.3
     command: mysqld --character-set-server=utf8mb4 --collation-server=utf8mb4_general_ci
-    environment:
-      MYSQL_DATABASE: rocketdb
-      MYSQL_USER: rocketdb
-      MYSQL_ROOT_PASSWORD: rocketdb
-      MYSQL_PASSWORD: rocketdb
+    env_file:
+      - .dev.env