From 26489b3e97e6bbe5319c0a13740b039fc4b56041 Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Tue, 27 Jun 2023 00:53:34 +0000 Subject: [PATCH 1/8] chore(devcontainer): update customizations syntax --- .devcontainer/devcontainer.json | 41 ++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index e6ce0513..cb5b195b 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -8,24 +8,27 @@ "postStartCommand": ["/bin/bash", "bin/init.sh"], "postAttachCommand": ["/bin/bash", ".devcontainer/postAttach.sh"], - // Set *default* container specific settings.json values on container create. - "settings": { - "terminal.integrated.defaultProfile.linux": "bash", - "terminal.integrated.profiles.linux": { - "bash": { - "path": "/bin/bash" - } + "customizations": { + // Set *default* container specific settings.json values on container create. + "vscode": { + "settings": { + "terminal.integrated.defaultProfile.linux": "bash", + "terminal.integrated.profiles.linux": { + "bash": { + "path": "/bin/bash" + } + } + }, + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "bpruitt-goddard.mermaid-markdown-syntax-highlighting", + "eamodio.gitlens", + "esbenp.prettier-vscode", + "hashicorp.terraform", + "mhutchie.git-graph", + "ms-python.python", + "ms-python.vscode-pylance" + ] } - }, - - // Add the IDs of extensions you want installed when the container is created. - "extensions": [ - "bpruitt-goddard.mermaid-markdown-syntax-highlighting", - "eamodio.gitlens", - "esbenp.prettier-vscode", - "hashicorp.terraform", - "mhutchie.git-graph", - "ms-python.python", - "ms-python.vscode-pylance" - ] + } } From f8ba8d59f378d1bfbc87a6caca60000745625a60 Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Tue, 27 Jun 2023 00:54:21 +0000 Subject: [PATCH 2/8] chore(devcontainer): normalize settings with the benefits devcontainer --- .vscode/settings.json | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index a8b8d514..57f4dcd9 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -9,18 +9,20 @@ "[python]": { "editor.defaultFormatter": "ms-python.python" }, - "python.defaultInterpreterPath": "/usr/local/bin/python", "python.formatting.provider": "black", "python.languageServer": "Pylance", - "python.linting.flake8Enabled": true, "python.linting.enabled": true, + "python.linting.flake8Enabled": true, "python.testing.pytestArgs": ["tests"], - "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true, + "python.testing.unittestEnabled": false, "[terraform]": { "editor.defaultFormatter": "hashicorp.terraform" }, "[terraform-vars]": { "editor.defaultFormatter": "hashicorp.terraform" + }, + "workbench.editorAssociations": { + "*.db": "sqlite-viewer.option" } } From 5eadec5fc0a31c7c1a37428a3421b11eb19e103d Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Tue, 27 Jun 2023 00:02:59 +0000 Subject: [PATCH 3/8] chore(data): add header row test_init_db_command fails with user count 26 != asserted 25 --- data/server.csv | 1 + 1 file changed, 1 insertion(+) diff --git a/data/server.csv b/data/server.csv index cfc85ca0..3ef76d78 100644 --- a/data/server.csv +++ b/data/server.csv @@ -1,3 +1,4 @@ +sub,name,type 71162,Box,courtesy_card 32587,Gonzales,courtesy_card 03552,McCulley,courtesy_card From b4c398aa040abb9b4eed1b1beda3964744c21cf0 Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Tue, 27 Jun 2023 15:25:10 -0700 Subject: [PATCH 4/8] fix(db): use csv.DictReader to parse header row --- eligibility_server/db/setup.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/eligibility_server/db/setup.py b/eligibility_server/db/setup.py index 84a1eaac..5441fa75 100644 --- a/eligibility_server/db/setup.py +++ b/eligibility_server/db/setup.py @@ -94,17 +94,18 @@ def import_csv_users(csv_path, remote): # open in read mode explicitly since the file may still be open if we downloaded from remote # newline="" is important here, see https://docs.python.org/3/library/csv.html#id3 with open(csv_path, mode="r", encoding="utf-8", newline="") as file: - data = csv.reader( + data = csv.DictReader( file, delimiter=config.csv_delimiter, quoting=config.csv_quoting, quotechar=config.csv_quotechar, ) - # unpack each record in data to the 3 columns - for sub, name, types in data: + + for row in data: # type lists are expected to be a comma-separated value and quoted if the CSV delimiter is a comma - types = [type.replace(config.csv_quotechar, "") for type in types.split(",") if type] - save_user(sub, name, types) + types = row["type"] + types = [types.replace(config.csv_quotechar, "") for type in types.split(",") if type] + save_user(row["sub"], row["name"], types) # close and remove the temp file if needed if temp_csv: From cc935bfc35859822111848b3bc6ec189d2ac701b Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Fri, 7 Jul 2023 13:28:08 -0700 Subject: [PATCH 5/8] refactor: use pyproject.toml exclusively move version to within app __init__ --- .devcontainer/requirements.txt | 3 --- eligibility_server/__init__.py | 3 +++ pyproject.toml | 45 ++++++++++++++++++++++++++++++---- requirements.txt | 5 ---- setup.py | 37 ---------------------------- tests/requirements.txt | 3 --- 6 files changed, 43 insertions(+), 53 deletions(-) delete mode 100644 .devcontainer/requirements.txt delete mode 100644 requirements.txt delete mode 100644 setup.py delete mode 100644 tests/requirements.txt diff --git a/.devcontainer/requirements.txt b/.devcontainer/requirements.txt deleted file mode 100644 index 0164b67a..00000000 --- a/.devcontainer/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -black -flake8 -pre-commit diff --git a/eligibility_server/__init__.py b/eligibility_server/__init__.py index e69de29b..6432a425 100644 --- a/eligibility_server/__init__.py +++ b/eligibility_server/__init__.py @@ -0,0 +1,3 @@ +__version__ = "2023.06.1" + +VERSION = __version__ diff --git a/pyproject.toml b/pyproject.toml index 8548c2bc..cd6c8e3b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,11 +1,46 @@ -# Configuration for Black. +[build-system] +requires = ["setuptools>=64", "wheel"] +build-backend = "setuptools.build_meta" -# NOTE: you have to use single-quoted strings in TOML for regular expressions. -# It's the equivalent of r-strings in Python. Multiline strings are treated as -# verbose regular expressions by Black. Use [ ] to denote a significant space -# character. +[project] +classifiers = ["Programming Language :: Python :: 3 :: Only"] +description = "Server implementation of the Eligibility Verification API" +dependencies = [ + "eligibility-api==2023.6.1", + "Flask==2.3.2", + "Flask-RESTful==0.3.10", + "Flask-SQLAlchemy==3.0.5", + "requests==2.31.0" +] +dynamic = ["version"] +keywords = ["flask"] +license = { file = "LICENSE" } +name = "eligibility-server" +readme = "README.md" +requires-python = ">=3.9" +[project.optional-dependencies] +dev = [ + "black", + "flake8", + "pre-commit", +] +test = [ + "coverage", + "pytest", + "pytest-mock", +] + +[project.urls] +Code = "https://github.com/cal-itp/eligibility-server" +Documentation = "https://docs.calitp.org/eligibility-server" +Issues = "https://github.com/cal-itp/eligibility-server/issues" + +# Configuration for black [tool.black] line-length = 127 target-version = ['py310'] include = '\.pyi?$' + +[tool.setuptools] +packages = ["eligibility_server"] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 73edb3dd..00000000 --- a/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -eligibility-api==2023.6.1 -Flask==2.3.2 -Flask-RESTful==0.3.10 -Flask-SQLAlchemy==3.0.5 -requests==2.31.0 diff --git a/setup.py b/setup.py deleted file mode 100644 index 8b5a1f22..00000000 --- a/setup.py +++ /dev/null @@ -1,37 +0,0 @@ -from setuptools import find_packages, setup - - -with open("requirements.txt") as f: - requirements = f.read().strip().split("\n") - install_requires = [r for r in requirements if not r.startswith("git+")] - dependency_links = list(set(requirements) - set(install_requires)) - -with open("README.md") as f: - long_description = f.read() - - -CLASSIFIERS = [ - "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3", -] - -setup( - name="eligibility-server", - version="2023.06.1", - description="Server implementation of the Eligibility Verification API", - long_description=long_description, - long_description_content_type="text/markdown", - python_requires=">=3.7", - classifiers=CLASSIFIERS, - project_urls={ - "Source": "https://github.com/cal-itp/eligibility-server", - "Tracker": "https://github.com/cal-itp/eligibility-server/issues", - }, - packages=find_packages(), - include_package_data=True, - install_requires=install_requires, - dependency_links=dependency_links, - license="AGPL-3.0", -) diff --git a/tests/requirements.txt b/tests/requirements.txt deleted file mode 100644 index 8be74ac0..00000000 --- a/tests/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -coverage -pytest -pytest-mock From 9982e7d2121e966cd81372f5a682b8e054eec19f Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Fri, 7 Jul 2023 13:28:54 -0700 Subject: [PATCH 6/8] refactor(docker): update build to install pyproject.toml --- .devcontainer/Dockerfile | 16 +++++++++------- Dockerfile | 10 ++++------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 8cbdca97..7e90d990 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,13 +1,15 @@ FROM eligibility_server:latest -# install python tooling -COPY .devcontainer/requirements.txt .devcontainer/requirements.txt -RUN pip install --no-cache-dir -r .devcontainer/requirements.txt +# install devcontainer requirements +RUN pip install -e .[dev,test] -# install docs tooling +# docs requirements are in a separate file for the GitHub Action COPY docs/requirements.txt docs/requirements.txt RUN pip install --no-cache-dir -r docs/requirements.txt -# install test tooling -COPY tests/requirements.txt tests/requirements.txt -RUN pip install -r tests/requirements.txt +# install pre-commit environments in throwaway Git repository +# https://stackoverflow.com/a/68758943 +COPY .pre-commit-config.yaml . +RUN git init . && \ + pre-commit install-hooks && \ + rm -rf .git diff --git a/Dockerfile b/Dockerfile index 5c9ada29..feb3b0ac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,15 +2,13 @@ FROM ghcr.io/cal-itp/docker-python-web:main ENV FLASK_APP=eligibility_server/app.py -# install python app dependencies -COPY requirements.txt requirements.txt -RUN pip install --no-cache-dir -r requirements.txt +# upgrade pip +RUN python -m pip install --upgrade pip # copy source files -COPY bin/ bin/ +COPY bin bin COPY eligibility_server/ eligibility_server/ -COPY *.py . -COPY README.md . +COPY pyproject.toml pyproject.toml # install source as a package RUN pip install -e . From 3880b7e81fad9b6597bc9b9597e5d3d1d0004b3f Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Fri, 7 Jul 2023 21:39:11 +0000 Subject: [PATCH 7/8] chore(github): update workflows and dependabot --- .github/dependabot.yml | 2 +- .github/workflows/run-tests.yml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 1bfaee9a..563c6a33 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,7 +6,7 @@ version: 2 updates: - package-ecosystem: "pip" - directory: "/" # requirements.txt + directory: "/" # pyproject.toml schedule: interval: "daily" commit-message: diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 311bf2a8..be47c86d 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -17,8 +17,7 @@ jobs: - name: Install package and dependencies run: | python -m pip install --upgrade pip - pip install -r tests/requirements.txt - pip install -r requirements.txt + pip install -e .[test] - name: Load environment variables uses: cardinalby/export-env-action@v2 From f7c2ad18fbdb2da270fe9145a011c05cdb1ec799 Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Mon, 24 Jul 2023 15:56:22 -0700 Subject: [PATCH 8/8] chore(terraform): route uptime check through Front Door more realistic and end-to-end test of the infrastructure and app --- terraform/uptime.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/uptime.tf b/terraform/uptime.tf index 7a0d2176..df8f5190 100644 --- a/terraform/uptime.tf +++ b/terraform/uptime.tf @@ -6,7 +6,7 @@ module "healthcheck" { application_insights = azurerm_application_insights.main # not strictly necessary to include the environment name, but helps to make the alerts more clear name = "mst-courtesy-cards-eligibility-server-${local.env_name}-healthcheck" - url = "https://${azurerm_linux_web_app.main.default_hostname}/healthcheck" + url = "https://${azurerm_cdn_frontdoor_endpoint.main.host_name}/healthcheck" } # ignore when app restarts as data is being reloaded