diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..3d1e5c5 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,2 @@ +[run] +omit = *tests*,app.py \ No newline at end of file diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..3e0dcff --- /dev/null +++ b/.dockerignore @@ -0,0 +1,21 @@ +tests/ +docs/ +cached/ +.github +.gitignore +Dockerfile +Makefile +Pipfile +Pipfile.lock +README.md +requirements-dev.txt +setup.py +tox.ini +.coverage +.dockerignore +.git +.mypy_cache +.pytest_cache +.tox +__pycache__ +tern_rest_api.egg-info \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb5cace..c99a05a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,4 +22,4 @@ jobs: run: pip install tox tox-gh-actions - name: Run Python tests - run: tox + run: tox -r diff --git a/.gitignore b/.gitignore index b6e4761..be915d9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# tern-rest-api specifics +cached +docs/.test_swagger.json # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f44eb7f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,30 @@ +# Copyright (c) 2022 VMware, Inc. All Rights Reserved. +# SPDX-License-Identifier: BSD-2-Clause + +FROM python:3.9-slim-buster as base + +RUN echo "deb http://deb.debian.org/debian bullseye main" > /etc/apt/sources.list.d/bullseye.list \ + && echo "Package: *\nPin: release n=bullseye\nPin-Priority: 50" > /etc/apt/preferences.d/bullseye \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + attr \ + findutils \ + fuse-overlayfs/bullseye \ + fuse3/bullseye \ + git \ + jq \ + skopeo \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt requirements.txt +RUN pip install --no-cache -r ./requirements.txt + +RUN mkdir /opt/tern-rest-api + +ADD . /opt/tern-rest-api +WORKDIR /opt/tern-rest-api + +ENV TERN_API_CACHE_DIR=/var/opt/tern-rest-api/cache +ENV TERN_DEFAULT_REGISTRY="registry.hub.docker.com" + +ENTRYPOINT [ "bash", "docker_start.sh" ] \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..3ed578f --- /dev/null +++ b/Makefile @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (c) 2022 VMware, Inc. All Rights Reserved. +# SPDX-License-Identifier: BSD-2-Clause + +build-dev: + docker-compose build --force-rm tern-rest-api + +serve-dev: build-dev + docker-compose up --remove-orphans + +tests: build-dev + docker-compose run --rm --volume=$(PWD):/opt/tern-rest-api --entrypoint="/bin/sh" tern-rest-api -c 'pip install tox && tox' + +stop: + docker-compose down -v + +update-requirements: + pipenv lock -r > requirements.txt + pipenv lock -r -d > requirements-dev.txt + +doc: + python -c "import app; app.export_swagger_json('docs/swagger.json')" diff --git a/Pipfile b/Pipfile index 4d5e929..825d21f 100644 --- a/Pipfile +++ b/Pipfile @@ -7,6 +7,8 @@ name = "pypi" flask = "*" flask-restx = "*" tern = "*" +flask-executor = "*" +gunicorn = "*" [dev-packages] black = "*" diff --git a/Pipfile.lock b/Pipfile.lock index a6a397f..c8fd3f5 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "6d5cd08ed1e881cbdf1b0f6d2b85741bf13aea25b22687cb9606168c27a686e3" + "sha256": "bc709a78553a8ae95dd6dc4ba0fc0d8f4a07fe1a4eb0668baf4e3fa1856d99ed" }, "pipfile-spec": 6, "requires": { @@ -57,11 +57,11 @@ }, "click": { "hashes": [ - "sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3", - "sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b" + "sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1", + "sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb" ], "markers": "python_version >= '3.6'", - "version": "==8.0.3" + "version": "==8.0.4" }, "debian-inspector": { "hashes": [ @@ -94,6 +94,14 @@ "index": "pypi", "version": "==2.0.3" }, + "flask-executor": { + "hashes": [ + "sha256:074885fc6d04764c86ab7f070818ea87dd08ee54767a216a8e7a00c39550bea2", + "sha256:ddb51b9e10f0fbfcff6c2386a5d92957402b7c81b7614eb1f3c77be64bcfd684" + ], + "index": "pypi", + "version": "==0.10.0" + }, "flask-restx": { "hashes": [ "sha256:63c69a61999a34f1774eaccc6fc8c7f504b1aad7d56a8ec672264e52d9ac05f4", @@ -118,6 +126,14 @@ "markers": "python_version >= '3.7'", "version": "==3.1.26" }, + "gunicorn": { + "hashes": [ + "sha256:9dcc4547dbb1cb284accfb15ab5667a0e5d1881cc443e0677b4882a4067a807e", + "sha256:e0a968b5ba15f8a328fdfd7ab1fcb5af4470c28aaf7e55df02a99bc13138e6e8" + ], + "index": "pypi", + "version": "==20.1.0" + }, "idna": { "hashes": [ "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", @@ -128,11 +144,11 @@ }, "itsdangerous": { "hashes": [ - "sha256:5174094b9637652bdb841a3029700391451bd092ba3db90600dea710ba28e97c", - "sha256:9e724d68fc22902a1435351f84c3fb8623f303fffcc566a4cb952df8c572cff0" + "sha256:29285842166554469a56d427addc0843914172343784cb909695fdbe90a3e129", + "sha256:d848fcb8bc7d507c4546b448574e8a44fc4ea2ba84ebf8d783290d53e81992f5" ], - "markers": "python_version >= '3.6'", - "version": "==2.0.1" + "markers": "python_version >= '3.7'", + "version": "==2.1.0" }, "jinja2": { "hashes": [ @@ -152,78 +168,49 @@ }, "markupsafe": { "hashes": [ - "sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298", - "sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64", - "sha256:0446679737af14f45767963a1a9ef7620189912317d095f2d9ffa183a4d25d2b", - "sha256:04635854b943835a6ea959e948d19dcd311762c5c0c6e1f0e16ee57022669194", - "sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567", - "sha256:0955295dd5eec6cb6cc2fe1698f4c6d84af2e92de33fbcac4111913cd100a6ff", - "sha256:0d4b31cc67ab36e3392bbf3862cfbadac3db12bdd8b02a2731f509ed5b829724", - "sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74", - "sha256:168cd0a3642de83558a5153c8bd34f175a9a6e7f6dc6384b9655d2697312a646", - "sha256:1d609f577dc6e1aa17d746f8bd3c31aa4d258f4070d61b2aa5c4166c1539de35", - "sha256:1f2ade76b9903f39aa442b4aadd2177decb66525062db244b35d71d0ee8599b6", - "sha256:20dca64a3ef2d6e4d5d615a3fd418ad3bde77a47ec8a23d984a12b5b4c74491a", - "sha256:2a7d351cbd8cfeb19ca00de495e224dea7e7d919659c2841bbb7f420ad03e2d6", - "sha256:2d7d807855b419fc2ed3e631034685db6079889a1f01d5d9dac950f764da3dad", - "sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26", - "sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38", - "sha256:37205cac2a79194e3750b0af2a5720d95f786a55ce7df90c3af697bfa100eaac", - "sha256:3c112550557578c26af18a1ccc9e090bfe03832ae994343cfdacd287db6a6ae7", - "sha256:3dd007d54ee88b46be476e293f48c85048603f5f516008bee124ddd891398ed6", - "sha256:4296f2b1ce8c86a6aea78613c34bb1a672ea0e3de9c6ba08a960efe0b0a09047", - "sha256:47ab1e7b91c098ab893b828deafa1203de86d0bc6ab587b160f78fe6c4011f75", - "sha256:49e3ceeabbfb9d66c3aef5af3a60cc43b85c33df25ce03d0031a608b0a8b2e3f", - "sha256:4dc8f9fb58f7364b63fd9f85013b780ef83c11857ae79f2feda41e270468dd9b", - "sha256:4efca8f86c54b22348a5467704e3fec767b2db12fc39c6d963168ab1d3fc9135", - "sha256:53edb4da6925ad13c07b6d26c2a852bd81e364f95301c66e930ab2aef5b5ddd8", - "sha256:5855f8438a7d1d458206a2466bf82b0f104a3724bf96a1c781ab731e4201731a", - "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a", - "sha256:5b6d930f030f8ed98e3e6c98ffa0652bdb82601e7a016ec2ab5d7ff23baa78d1", - "sha256:5bb28c636d87e840583ee3adeb78172efc47c8b26127267f54a9c0ec251d41a9", - "sha256:60bf42e36abfaf9aff1f50f52644b336d4f0a3fd6d8a60ca0d054ac9f713a864", - "sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914", - "sha256:6300b8454aa6930a24b9618fbb54b5a68135092bc666f7b06901f897fa5c2fee", - "sha256:63f3268ba69ace99cab4e3e3b5840b03340efed0948ab8f78d2fd87ee5442a4f", - "sha256:6557b31b5e2c9ddf0de32a691f2312a32f77cd7681d8af66c2692efdbef84c18", - "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8", - "sha256:6a7fae0dd14cf60ad5ff42baa2e95727c3d81ded453457771d02b7d2b3f9c0c2", - "sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d", - "sha256:6fcf051089389abe060c9cd7caa212c707e58153afa2c649f00346ce6d260f1b", - "sha256:7d91275b0245b1da4d4cfa07e0faedd5b0812efc15b702576d103293e252af1b", - "sha256:89c687013cb1cd489a0f0ac24febe8c7a666e6e221b783e53ac50ebf68e45d86", - "sha256:8d206346619592c6200148b01a2142798c989edcb9c896f9ac9722a99d4e77e6", - "sha256:905fec760bd2fa1388bb5b489ee8ee5f7291d692638ea5f67982d968366bef9f", - "sha256:97383d78eb34da7e1fa37dd273c20ad4320929af65d156e35a5e2d89566d9dfb", - "sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833", - "sha256:99df47edb6bda1249d3e80fdabb1dab8c08ef3975f69aed437cb69d0a5de1e28", - "sha256:9f02365d4e99430a12647f09b6cc8bab61a6564363f313126f775eb4f6ef798e", - "sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415", - "sha256:ab3ef638ace319fa26553db0624c4699e31a28bb2a835c5faca8f8acf6a5a902", - "sha256:aca6377c0cb8a8253e493c6b451565ac77e98c2951c45f913e0b52facdcff83f", - "sha256:add36cb2dbb8b736611303cd3bfcee00afd96471b09cda130da3581cbdc56a6d", - "sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9", - "sha256:b7f2d075102dc8c794cbde1947378051c4e5180d52d276987b8d28a3bd58c17d", - "sha256:baa1a4e8f868845af802979fcdbf0bb11f94f1cb7ced4c4b8a351bb60d108145", - "sha256:be98f628055368795d818ebf93da628541e10b75b41c559fdf36d104c5787066", - "sha256:bf5d821ffabf0ef3533c39c518f3357b171a1651c1ff6827325e4489b0e46c3c", - "sha256:c47adbc92fc1bb2b3274c4b3a43ae0e4573d9fbff4f54cd484555edbf030baf1", - "sha256:cdfba22ea2f0029c9261a4bd07e830a8da012291fbe44dc794e488b6c9bb353a", - "sha256:d6c7ebd4e944c85e2c3421e612a7057a2f48d478d79e61800d81468a8d842207", - "sha256:d7f9850398e85aba693bb640262d3611788b1f29a79f0c93c565694658f4071f", - "sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53", - "sha256:deb993cacb280823246a026e3b2d81c493c53de6acfd5e6bfe31ab3402bb37dd", - "sha256:e0f138900af21926a02425cf736db95be9f4af72ba1bb21453432a07f6082134", - "sha256:e9936f0b261d4df76ad22f8fee3ae83b60d7c3e871292cd42f40b81b70afae85", - "sha256:f0567c4dc99f264f49fe27da5f735f414c4e7e7dd850cfd8e69f0862d7c74ea9", - "sha256:f5653a225f31e113b152e56f154ccbe59eeb1c7487b39b9d9f9cdb58e6c79dc5", - "sha256:f826e31d18b516f653fe296d967d700fddad5901ae07c622bb3705955e1faa94", - "sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509", - "sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51", - "sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872" + "sha256:023af8c54fe63530545f70dd2a2a7eed18d07a9a77b94e8bf1e2ff7f252db9a3", + "sha256:09c86c9643cceb1d87ca08cdc30160d1b7ab49a8a21564868921959bd16441b8", + "sha256:142119fb14a1ef6d758912b25c4e803c3ff66920635c44078666fe7cc3f8f759", + "sha256:1d1fb9b2eec3c9714dd936860850300b51dbaa37404209c8d4cb66547884b7ed", + "sha256:204730fd5fe2fe3b1e9ccadb2bd18ba8712b111dcabce185af0b3b5285a7c989", + "sha256:24c3be29abb6b34052fd26fc7a8e0a49b1ee9d282e3665e8ad09a0a68faee5b3", + "sha256:290b02bab3c9e216da57c1d11d2ba73a9f73a614bbdcc027d299a60cdfabb11a", + "sha256:3028252424c72b2602a323f70fbf50aa80a5d3aa616ea6add4ba21ae9cc9da4c", + "sha256:30c653fde75a6e5eb814d2a0a89378f83d1d3f502ab710904ee585c38888816c", + "sha256:3cace1837bc84e63b3fd2dfce37f08f8c18aeb81ef5cf6bb9b51f625cb4e6cd8", + "sha256:4056f752015dfa9828dce3140dbadd543b555afb3252507348c493def166d454", + "sha256:454ffc1cbb75227d15667c09f164a0099159da0c1f3d2636aa648f12675491ad", + "sha256:598b65d74615c021423bd45c2bc5e9b59539c875a9bdb7e5f2a6b92dfcfc268d", + "sha256:599941da468f2cf22bf90a84f6e2a65524e87be2fce844f96f2dd9a6c9d1e635", + "sha256:5ddea4c352a488b5e1069069f2f501006b1a4362cb906bee9a193ef1245a7a61", + "sha256:62c0285e91414f5c8f621a17b69fc0088394ccdaa961ef469e833dbff64bd5ea", + "sha256:679cbb78914ab212c49c67ba2c7396dc599a8479de51b9a87b174700abd9ea49", + "sha256:6e104c0c2b4cd765b4e83909cde7ec61a1e313f8a75775897db321450e928cce", + "sha256:736895a020e31b428b3382a7887bfea96102c529530299f426bf2e636aacec9e", + "sha256:75bb36f134883fdbe13d8e63b8675f5f12b80bb6627f7714c7d6c5becf22719f", + "sha256:7d2f5d97fcbd004c03df8d8fe2b973fe2b14e7bfeb2cfa012eaa8759ce9a762f", + "sha256:80beaf63ddfbc64a0452b841d8036ca0611e049650e20afcb882f5d3c266d65f", + "sha256:84ad5e29bf8bab3ad70fd707d3c05524862bddc54dc040982b0dbcff36481de7", + "sha256:8da5924cb1f9064589767b0f3fc39d03e3d0fb5aa29e0cb21d43106519bd624a", + "sha256:961eb86e5be7d0973789f30ebcf6caab60b844203f4396ece27310295a6082c7", + "sha256:96de1932237abe0a13ba68b63e94113678c379dca45afa040a17b6e1ad7ed076", + "sha256:a0a0abef2ca47b33fb615b491ce31b055ef2430de52c5b3fb19a4042dbc5cadb", + "sha256:b2a5a856019d2833c56a3dcac1b80fe795c95f401818ea963594b345929dffa7", + "sha256:b8811d48078d1cf2a6863dafb896e68406c5f513048451cd2ded0473133473c7", + "sha256:c532d5ab79be0199fa2658e24a02fce8542df196e60665dd322409a03db6a52c", + "sha256:d3b64c65328cb4cd252c94f83e66e3d7acf8891e60ebf588d7b493a55a1dbf26", + "sha256:d4e702eea4a2903441f2735799d217f4ac1b55f7d8ad96ab7d4e25417cb0827c", + "sha256:d5653619b3eb5cbd35bfba3c12d575db2a74d15e0e1c08bf1db788069d410ce8", + "sha256:d66624f04de4af8bbf1c7f21cc06649c1c69a7f84109179add573ce35e46d448", + "sha256:e67ec74fada3841b8c5f4c4f197bea916025cb9aa3fe5abf7d52b655d042f956", + "sha256:e6f7f3f41faffaea6596da86ecc2389672fa949bd035251eab26dc6697451d05", + "sha256:f02cf7221d5cd915d7fa58ab64f7ee6dd0f6cddbb48683debf5d04ae9b1c2cc1", + "sha256:f0eddfcabd6936558ec020130f932d479930581171368fd728efcfb6ef0dd357", + "sha256:fabbe18087c3d33c5824cb145ffca52eccd053061df1d79d4b66dafa5ad2a5ea", + "sha256:fc3150f85e2dbcf99e65238c842d1cfe69d3e7649b19864c1cc043213d9cd730" ], - "markers": "python_version >= '3.6'", - "version": "==2.0.1" + "markers": "python_version >= '3.7'", + "version": "==2.1.0" }, "packageurl-python": { "hashes": [ @@ -513,58 +500,58 @@ }, "click": { "hashes": [ - "sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3", - "sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b" + "sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1", + "sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb" ], "markers": "python_version >= '3.6'", - "version": "==8.0.3" + "version": "==8.0.4" }, "coverage": { "hashes": [ - "sha256:1245ab82e8554fa88c4b2ab1e098ae051faac5af829efdcf2ce6b34dccd5567c", - "sha256:1bc6d709939ff262fd1432f03f080c5042dc6508b6e0d3d20e61dd045456a1a0", - "sha256:25e73d4c81efa8ea3785274a2f7f3bfbbeccb6fcba2a0bdd3be9223371c37554", - "sha256:276b13cc085474e482566c477c25ed66a097b44c6e77132f3304ac0b039f83eb", - "sha256:2aed4761809640f02e44e16b8b32c1a5dee5e80ea30a0ff0912158bde9c501f2", - "sha256:2dd70a167843b4b4b2630c0c56f1b586fe965b4f8ac5da05b6690344fd065c6b", - "sha256:352c68e233409c31048a3725c446a9e48bbff36e39db92774d4f2380d630d8f8", - "sha256:3f2b05757c92ad96b33dbf8e8ec8d4ccb9af6ae3c9e9bd141c7cc44d20c6bcba", - "sha256:448d7bde7ceb6c69e08474c2ddbc5b4cd13c9e4aa4a717467f716b5fc938a734", - "sha256:463e52616ea687fd323888e86bf25e864a3cc6335a043fad6bbb037dbf49bbe2", - "sha256:482fb42eea6164894ff82abbcf33d526362de5d1a7ed25af7ecbdddd28fc124f", - "sha256:56c4a409381ddd7bbff134e9756077860d4e8a583d310a6f38a2315b9ce301d0", - "sha256:56d296cbc8254a7dffdd7bcc2eb70be5a233aae7c01856d2d936f5ac4e8ac1f1", - "sha256:5e15d424b8153756b7c903bde6d4610be0c3daca3986173c18dd5c1a1625e4cd", - "sha256:618eeba986cea7f621d8607ee378ecc8c2504b98b3fdc4952b30fe3578304687", - "sha256:61d47a897c1e91f33f177c21de897267b38fbb45f2cd8e22a710bcef1df09ac1", - "sha256:621f6ea7260ea2ffdaec64fe5cb521669984f567b66f62f81445221d4754df4c", - "sha256:6a5cdc3adb4f8bb8d8f5e64c2e9e282bc12980ef055ec6da59db562ee9bdfefa", - "sha256:6c3f6158b02ac403868eea390930ae64e9a9a2a5bbfafefbb920d29258d9f2f8", - "sha256:704f89b87c4f4737da2860695a18c852b78ec7279b24eedacab10b29067d3a38", - "sha256:72128176fea72012063200b7b395ed8a57849282b207321124d7ff14e26988e8", - "sha256:78fbb2be068a13a5d99dce9e1e7d168db880870f7bc73f876152130575bd6167", - "sha256:7bff3a98f63b47464480de1b5bdd80c8fade0ba2832c9381253c9b74c4153c27", - "sha256:84f2436d6742c01136dd940ee158bfc7cf5ced3da7e4c949662b8703b5cd8145", - "sha256:9976fb0a5709988778ac9bc44f3d50fccd989987876dfd7716dee28beed0a9fa", - "sha256:9ad0a117b8dc2061ce9461ea4c1b4799e55edceb236522c5b8f958ce9ed8fa9a", - "sha256:9e3dd806f34de38d4c01416344e98eab2437ac450b3ae39c62a0ede2f8b5e4ed", - "sha256:9eb494070aa060ceba6e4bbf44c1bc5fa97bfb883a0d9b0c9049415f9e944793", - "sha256:9fde6b90889522c220dd56a670102ceef24955d994ff7af2cb786b4ba8fe11e4", - "sha256:9fff3ff052922cb99f9e52f63f985d4f7a54f6b94287463bc66b7cdf3eb41217", - "sha256:a06c358f4aed05fa1099c39decc8022261bb07dfadc127c08cfbd1391b09689e", - "sha256:a4f923b9ab265136e57cc14794a15b9dcea07a9c578609cd5dbbfff28a0d15e6", - "sha256:c5b81fb37db76ebea79aa963b76d96ff854e7662921ce742293463635a87a78d", - "sha256:d5ed164af5c9078596cfc40b078c3b337911190d3faeac830c3f1274f26b8320", - "sha256:d651fde74a4d3122e5562705824507e2f5b2d3d57557f1916c4b27635f8fbe3f", - "sha256:de73fca6fb403dd72d4da517cfc49fcf791f74eee697d3219f6be29adf5af6ce", - "sha256:e647a0be741edbb529a72644e999acb09f2ad60465f80757da183528941ff975", - "sha256:e92c7a5f7d62edff50f60a045dc9542bf939758c95b2fcd686175dd10ce0ed10", - "sha256:eeffd96882d8c06d31b65dddcf51db7c612547babc1c4c5db6a011abe9798525", - "sha256:f5a4551dfd09c3bd12fca8144d47fe7745275adf3229b7223c2f9e29a975ebda", - "sha256:fac0bcc5b7e8169bffa87f0dcc24435446d329cbc2b5486d155c2e0f3b493ae1" + "sha256:03e2a7826086b91ef345ff18742ee9fc47a6839ccd517061ef8fa1976e652ce9", + "sha256:07e6db90cd9686c767dcc593dff16c8c09f9814f5e9c51034066cad3373b914d", + "sha256:18d520c6860515a771708937d2f78f63cc47ab3b80cb78e86573b0a760161faf", + "sha256:1ebf730d2381158ecf3dfd4453fbca0613e16eaa547b4170e2450c9707665ce7", + "sha256:21b7745788866028adeb1e0eca3bf1101109e2dc58456cb49d2d9b99a8c516e6", + "sha256:26e2deacd414fc2f97dd9f7676ee3eaecd299ca751412d89f40bc01557a6b1b4", + "sha256:2c6dbb42f3ad25760010c45191e9757e7dce981cbfb90e42feef301d71540059", + "sha256:2fea046bfb455510e05be95e879f0e768d45c10c11509e20e06d8fcaa31d9e39", + "sha256:34626a7eee2a3da12af0507780bb51eb52dca0e1751fd1471d0810539cefb536", + "sha256:37d1141ad6b2466a7b53a22e08fe76994c2d35a5b6b469590424a9953155afac", + "sha256:46191097ebc381fbf89bdce207a6c107ac4ec0890d8d20f3360345ff5976155c", + "sha256:4dd8bafa458b5c7d061540f1ee9f18025a68e2d8471b3e858a9dad47c8d41903", + "sha256:4e21876082ed887baed0146fe222f861b5815455ada3b33b890f4105d806128d", + "sha256:58303469e9a272b4abdb9e302a780072c0633cdcc0165db7eec0f9e32f901e05", + "sha256:5ca5aeb4344b30d0bec47481536b8ba1181d50dbe783b0e4ad03c95dc1296684", + "sha256:68353fe7cdf91f109fc7d474461b46e7f1f14e533e911a2a2cbb8b0fc8613cf1", + "sha256:6f89d05e028d274ce4fa1a86887b071ae1755082ef94a6740238cd7a8178804f", + "sha256:7a15dc0a14008f1da3d1ebd44bdda3e357dbabdf5a0b5034d38fcde0b5c234b7", + "sha256:8bdde1177f2311ee552f47ae6e5aa7750c0e3291ca6b75f71f7ffe1f1dab3dca", + "sha256:8ce257cac556cb03be4a248d92ed36904a59a4a5ff55a994e92214cde15c5bad", + "sha256:8cf5cfcb1521dc3255d845d9dca3ff204b3229401994ef8d1984b32746bb45ca", + "sha256:8fbbdc8d55990eac1b0919ca69eb5a988a802b854488c34b8f37f3e2025fa90d", + "sha256:9548f10d8be799551eb3a9c74bbf2b4934ddb330e08a73320123c07f95cc2d92", + "sha256:96f8a1cb43ca1422f36492bebe63312d396491a9165ed3b9231e778d43a7fca4", + "sha256:9b27d894748475fa858f9597c0ee1d4829f44683f3813633aaf94b19cb5453cf", + "sha256:9baff2a45ae1f17c8078452e9e5962e518eab705e50a0aa8083733ea7d45f3a6", + "sha256:a2a8b8bcc399edb4347a5ca8b9b87e7524c0967b335fbb08a83c8421489ddee1", + "sha256:acf53bc2cf7282ab9b8ba346746afe703474004d9e566ad164c91a7a59f188a4", + "sha256:b0be84e5a6209858a1d3e8d1806c46214e867ce1b0fd32e4ea03f4bd8b2e3359", + "sha256:b31651d018b23ec463e95cf10070d0b2c548aa950a03d0b559eaa11c7e5a6fa3", + "sha256:b78e5afb39941572209f71866aa0b206c12f0109835aa0d601e41552f9b3e620", + "sha256:c76aeef1b95aff3905fb2ae2d96e319caca5b76fa41d3470b19d4e4a3a313512", + "sha256:dd035edafefee4d573140a76fdc785dc38829fe5a455c4bb12bac8c20cfc3d69", + "sha256:dd6fe30bd519694b356cbfcaca9bd5c1737cddd20778c6a581ae20dc8c04def2", + "sha256:e5f4e1edcf57ce94e5475fe09e5afa3e3145081318e5fd1a43a6b4539a97e518", + "sha256:ec6bc7fe73a938933d4178c9b23c4e0568e43e220aef9472c4f6044bfc6dd0f0", + "sha256:f1555ea6d6da108e1999b2463ea1003fe03f29213e459145e70edbaf3e004aaa", + "sha256:f5fa5803f47e095d7ad8443d28b01d48c0359484fec1b9d8606d0e3282084bc4", + "sha256:f7331dbf301b7289013175087636bbaf5b2405e57259dd2c42fdcc9fcc47325e", + "sha256:f9987b0354b06d4df0f4d3e0ec1ae76d7ce7cbca9a2f98c25041eb79eec766f1", + "sha256:fd9e830e9d8d89b20ab1e5af09b32d33e1a08ef4c4e14411e559556fd788e6b2" ], "index": "pypi", - "version": "==6.3.1" + "version": "==6.3.2" }, "distlib": { "hashes": [ @@ -575,11 +562,11 @@ }, "filelock": { "hashes": [ - "sha256:7b23620a293cf3e19924e469cb96672dc72b36c26e8f80f85668310117fcbe4e", - "sha256:d1eccb164ed020bc84edd9e45bf6cdb177f64749f6b8fe066648832d2e98726d" + "sha256:9cd540a9352e432c7246a48fe4e8712b10acb1df2ad1f30e8c070b82ae1fed85", + "sha256:f8314284bfffbdcfa0ff3d7992b023d4c628ced6feb957351d4c48d059f56bc0" ], "markers": "python_version >= '3.7'", - "version": "==3.5.1" + "version": "==3.6.0" }, "flake8": { "hashes": [ @@ -635,11 +622,11 @@ }, "platformdirs": { "hashes": [ - "sha256:30671902352e97b1eafd74ade8e4a694782bd3471685e78c32d0fdfd3aa7e7bb", - "sha256:8ec11dfba28ecc0715eb5fb0147a87b1bf325f349f3da9aab2cd6b50b96b692b" + "sha256:7535e70dfa32e84d4b34996ea99c5e432fa29a708d0f4e394bbcb2a8faa4f16d", + "sha256:bcae7cab893c2d310a711b70b24efb93334febe65f8de776ee320b517471e227" ], "markers": "python_version >= '3.7'", - "version": "==2.5.0" + "version": "==2.5.1" }, "pluggy": { "hashes": [ @@ -731,11 +718,11 @@ }, "virtualenv": { "hashes": [ - "sha256:45e1d053cad4cd453181ae877c4ffc053546ae99e7dd049b9ff1d9be7491abf7", - "sha256:e0621bcbf4160e4e1030f05065c8834b4e93f4fcc223255db2a823440aca9c14" + "sha256:dd448d1ded9f14d1a4bfa6bfc0c5b96ae3be3f2d6c6c159b23ddcfd701baa021", + "sha256:e9dd1a1359d70137559034c0f5433b34caf504af2dc756367be86a5a32967134" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==20.13.1" + "version": "==20.13.3" } } } diff --git a/README.md b/README.md index e578d0e..75fa781 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ files to help build your virtual environment. We also recommend using [Pipenv](https://pipenv.pypa.io/en/latest/) to manage your virtual environment. ```shell -$ Pip install pipenv +$ pip install pipenv $ pipenv shell ``` @@ -59,7 +59,26 @@ $ pipenv install -d ### Running the development Tern REST API +#### As a Docker Container +```shell +$ make serve-dev +``` +Open http://localhost/ in your browser. + +Changing the source code will automatically reload the server inside the +container and makes the development easier. + +You can stop the sever using ``Ctrl+C`` and running ``make stop`` +Container environment variables: + +- ``TERN_API_CACHE_DIR``: The directory where the tern reports are cached. +Default: ``/var/opt/tern-rest-api/cache`` +- ``TERN_DEFAULT_REGISTRY``: Optional default registry to use when no registry +is passed to the API requests. Default: ``docker.io`` + + +#### On your local machine Runing the API locally ```shell @@ -68,16 +87,29 @@ $ flask run --reload Open http://localhost:5000/ in your browser. -## Tests +### Tests We use [Tox](https://tox.wiki/en/latest/) to manage running the tests. -Running tests +#### As a Docker Container +```shell +$ make tests +``` + +#### On your local machine ```shell $ tox ``` -## Managing the requirements +### Documentation + +```shell +$ make doc +``` + +The documentation tests are done also by ``tox``. + +### Requirements Installing new requirements @@ -97,6 +129,5 @@ $ pipenv update Updating the ``requirements.txt`` and ``requirements-dev.txt`` ```shell -$ pipenv lock -r > requirements.txt -$ pipenv lock -r -d > requirements-dev.txt -``` \ No newline at end of file +$ make update-requirements +``` diff --git a/app.py b/app.py index 3043291..7a18284 100644 --- a/app.py +++ b/app.py @@ -8,9 +8,9 @@ from flask_restx import Api -from tern_api import __version__, tern_api +from tern_api import __version__, tern_app from tern_api.api.v1.common_models import api_models_namespace -from tern_api.api.v1.reports import ns as report_v1 +from tern_api.api.v1.reports import ns as reports_v1 from tern_api.api.v1.version import ns as version_v1 logging.basicConfig( @@ -26,20 +26,21 @@ api = Api( - tern_api, + tern_app, version=__version__.version, title="Tern REST API", description="Tern Project REST API", ) + api.add_namespace(api_models_namespace) api.add_namespace(version_v1, path="/api/v1/version") -api.add_namespace(report_v1, path="/api/v1/report") +api.add_namespace(reports_v1, path="/api/v1/reports") def export_swagger_json(filepath): - tern_api.config["SERVER_NAME"] = "localhost" - with tern_api.app_context().__enter__(): + tern_app.config["SERVER_NAME"] = "localhost" + with tern_app.app_context().__enter__(): with open(filepath, "w") as f: swagger_json = json.dumps(api.__schema__, indent=4) f.write(swagger_json) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..41f051a --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,21 @@ +version: '3' + +volumes: + tern-cache: + +services: + tern-rest-api: + build: + context: . + args: + DEVEL: "yes" + command: "bash bash_start.sh" + environment: + - ENVIRONMENT=development + - TERN_DEFAULT_REGISTRY=registry.hub.docker.com + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - tern-cache:/var/opt/tern-rest-api/cache:z + - ./tern_api:/opt/tern-rest-api/tern_api:z + ports: + - "5001:80" diff --git a/docker_start.sh b/docker_start.sh new file mode 100755 index 0000000..2204d26 --- /dev/null +++ b/docker_start.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +if [[ ${ENVIRONMENT^^} == "DEVELOPMENT" ]]; then + echo "Starting tern-rest-api in development mode" + gunicorn --reload -b 0.0.0.0:80 app:tern_app +else + gunicorn -b 0.0.0.0:80 app:tern_app +fi \ No newline at end of file diff --git a/docs/swagger.json b/docs/swagger.json index 29a88b0..d8016ac 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -2,7 +2,7 @@ "swagger": "2.0", "basePath": "/", "paths": { - "/api/v1/report": { + "/api/v1/reports": { "post": { "responses": { "200": { @@ -30,7 +30,7 @@ ] } }, - "/api/v1/report/status": { + "/api/v1/reports/status": { "post": { "responses": { "200": { @@ -145,15 +145,14 @@ }, "report_parameters": { "required": [ + "cache", "image", "tag" ], "properties": { "registry": { "type": "string", - "description": "Registry Server", - "default": "https://registry.hub.docker.com", - "example": "http://registry.example.com" + "description": "Registry Server" }, "image": { "type": "string", @@ -164,6 +163,11 @@ "type": "string", "description": "Image tag", "example": "3.0" + }, + "cache": { + "type": "boolean", + "description": "Use cache data if available?", + "example": true } }, "type": "object" @@ -181,6 +185,7 @@ }, "async_response_model": { "required": [ + "cache", "id" ], "properties": { @@ -193,6 +198,11 @@ "type": "string", "description": "Unique Identification for request", "example": "19f035a711644eab84ef5a38ceb5572e" + }, + "cache": { + "type": "boolean", + "description": "Request uses cache?", + "example": true } }, "type": "object" @@ -220,21 +230,38 @@ }, "data_status_response": { "required": [ + "cache", "status" ], "properties": { + "cache": { + "type": "boolean", + "description": "Requested using cache?", + "example": true + }, + "id": { + "type": "string", + "description": "Unique Identification for request", + "example": "19f035a711644eab84ef5a38ceb5572e" + }, + "message": { + "type": "string", + "description": "Message" + }, + "report": { + "$ref": "#/definitions/report_mode" + }, "status": { "type": "string", "description": "Status of request", - "example": "DONE", + "example": "SUCCESS", "enum": [ - "UNKNOWN", - "FAILED", - "DONE" + "PENDING", + "RECEIVED", + "RUNNING", + "SUCCESS", + "FAILURE" ] - }, - "result": { - "$ref": "#/definitions/report_mode" } }, "type": "object" @@ -260,6 +287,7 @@ }, "image_report_data": { "required": [ + "cache", "name", "repotag", "tag" @@ -276,6 +304,11 @@ "tag": { "type": "string", "example": "3.0" + }, + "cache": { + "type": "string", + "description": "Use cache if available?", + "default": true } }, "type": "object" diff --git a/requirements-dev.txt b/requirements-dev.txt index 6e786aa..06b400e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,32 +15,34 @@ black==22.1.0 certifi==2021.10.8 chardet==4.0.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' charset-normalizer==2.0.10; python_version >= '3' -click==8.0.3; python_version >= '3.6' -coverage==6.3.1 +click==8.0.4; python_version >= '3.6' +coverage==6.3.2 debian-inspector==30.0.0; python_version >= '3.6' and python_version < '4' distlib==0.3.4 docker==5.0.3; python_version >= '3.6' dockerfile-parse==1.2.0 -filelock==3.5.1; python_version >= '3.7' +filelock==3.6.0; python_version >= '3.7' flake8==4.0.1 +flask-executor==0.10.0 flask-restx==0.5.1 flask==2.0.3 gitdb==4.0.9; python_version >= '3.6' gitpython==3.1.26; python_version >= '3.7' +gunicorn==20.1.0 idna==3.3; python_version >= '3' iniconfig==1.1.1 isort==5.10.1 -itsdangerous==2.0.1; python_version >= '3.6' +itsdangerous==2.1.0; python_version >= '3.7' jinja2==3.0.3; python_version >= '3.6' jsonschema==4.4.0; python_version >= '3.7' -markupsafe==2.0.1; python_version >= '3.6' +markupsafe==2.1.0; python_version >= '3.7' mccabe==0.6.1 mypy-extensions==0.4.3 packageurl-python==0.9.6; python_version >= '3.6' packaging==21.3; python_version >= '3.6' pathspec==0.9.0 pbr==5.8.0; python_version >= '2.6' -platformdirs==2.5.0; python_version >= '3.7' +platformdirs==2.5.1; python_version >= '3.7' pluggy==1.0.0; python_version >= '3.6' prettytable==3.0.0; python_version >= '3.7' py==1.11.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' @@ -62,7 +64,7 @@ tomli==2.0.1; python_version >= '3.7' tox==3.24.5 typing-extensions==4.1.1; python_version < '3.10' urllib3==1.26.8; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4' -virtualenv==20.13.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +virtualenv==20.13.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' wcwidth==0.2.5 websocket-client==1.2.3; python_version >= '3.6' werkzeug==2.0.3; python_version >= '3.6' diff --git a/requirements.txt b/requirements.txt index eaccc09..49250d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,19 +11,21 @@ attrs==21.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, certifi==2021.10.8 chardet==4.0.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' charset-normalizer==2.0.10; python_version >= '3' -click==8.0.3; python_version >= '3.6' +click==8.0.4; python_version >= '3.6' debian-inspector==30.0.0; python_version >= '3.6' and python_version < '4' docker==5.0.3; python_version >= '3.6' dockerfile-parse==1.2.0 +flask-executor==0.10.0 flask-restx==0.5.1 flask==2.0.3 gitdb==4.0.9; python_version >= '3.6' gitpython==3.1.26; python_version >= '3.7' +gunicorn==20.1.0 idna==3.3; python_version >= '3' -itsdangerous==2.0.1; python_version >= '3.6' +itsdangerous==2.1.0; python_version >= '3.7' jinja2==3.0.3; python_version >= '3.6' jsonschema==4.4.0; python_version >= '3.7' -markupsafe==2.0.1; python_version >= '3.6' +markupsafe==2.1.0; python_version >= '3.7' packageurl-python==0.9.6; python_version >= '3.6' pbr==5.8.0; python_version >= '2.6' prettytable==3.0.0; python_version >= '3.7' diff --git a/tern_api/__init__.py b/tern_api/__init__.py index 6d1bd3b..5d378d7 100644 --- a/tern_api/__init__.py +++ b/tern_api/__init__.py @@ -1,7 +1,16 @@ +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # Copyright (c) 2022 VMware, Inc. All Rights Reserved. # SPDX-License-Identifier: BSD-2-Clause +import os + from flask import Flask +from flask_executor import Executor + +tern_app = Flask(__name__) +tern_app.config["TERN_API_CACHE_DIR"] = os.getenv("TERN_API_CACHE_DIR") +tern_app.config["TERN_DEFAULT_REGISTRY"] = os.getenv("TERN_DEFAULT_REGISTRY") -tern_api = Flask(__name__) +tern_tasks = Executor(tern_app) +tern_app.config["TERN_TASKS"] = tern_tasks diff --git a/tern_api/api/v1/common_models.py b/tern_api/api/v1/common_models.py index 1170558..9adbbfc 100644 --- a/tern_api/api/v1/common_models.py +++ b/tern_api/api/v1/common_models.py @@ -18,21 +18,6 @@ }, ) -async_response_model = api_models_namespace.model( - "async_response_model", - { - "message": fields.String( - description="Status message", - requored=True, - example="Request submitted.", - ), - "id": fields.String( - description="Unique Identification for request", - required=True, - example="19f035a711644eab84ef5a38ceb5572e", - ), - }, -) image_report_data = api_models_namespace.model( "image_report_data", @@ -52,6 +37,12 @@ example="3.0", required=True, ), + "cache": fields.String( + description="Use cache if available?", + exampple=True, + required=True, + default=True, + ), }, ) image_report_model = api_models_namespace.model( @@ -60,3 +51,24 @@ report_model = api_models_namespace.model( "report_mode", {"images": fields.List(fields.Nested(image_report_model))} ) + +async_response_model = api_models_namespace.model( + "async_response_model", + { + "message": fields.String( + description="Status message", + requored=True, + example="Request submitted.", + ), + "id": fields.String( + description="Unique Identification for request", + required=True, + example="19f035a711644eab84ef5a38ceb5572e", + ), + "cache": fields.Boolean( + description="Request uses cache?", + required=True, + example=True, + ), + }, +) diff --git a/tern_api/api/v1/reports.py b/tern_api/api/v1/reports.py index a0f82bb..e2161f4 100644 --- a/tern_api/api/v1/reports.py +++ b/tern_api/api/v1/reports.py @@ -3,13 +3,16 @@ # # Copyright (c) 2022 VMware, Inc. All Rights Reserved. # SPDX-License-Identifier: BSD-2-Clause +from flask import request from flask_restx import Namespace, Resource, fields +from tern_api import constants, tern_app from tern_api.api.v1.common_models import ( async_response_model, error_model, report_model, ) +from tern_api.reports import status, submit ns = Namespace("/reports", description="Tern Bill of Materials Report") @@ -22,8 +25,8 @@ class Report(Resource): "registry": fields.String( description="Registry Server", required=False, - default="https://registry.hub.docker.com", - example="http://registry.example.com", + default=tern_app.config["TERN_DEFAULT_REGISTRY"], + example=tern_app.config["TERN_DEFAULT_REGISTRY"], ), "image": fields.String( description="Image name", @@ -35,6 +38,11 @@ class Report(Resource): required=True, example="3.0", ), + "cache": fields.Boolean( + description="Use cache data if available?", + required=True, + example=True, + ), }, ) report_response_request = ns.model( @@ -46,12 +54,15 @@ class Report(Resource): ) @ns.response(200, "OK", report_response_request) - @ns.expect(report_parameters) + @ns.expect(report_parameters, validate=True) def post(self): """Tern BoM report **Note**: This request will be processed assynchronous. """ + payload = request.json + response = submit(payload) + return response.to_response() @ns.route("/status") @@ -69,13 +80,28 @@ class ReportStatus(Resource): data_status_response = ns.model( "data_status_response", { + "cache": fields.Boolean( + description="Requested using cache?", + required=True, + example=True, + ), + "id": fields.String( + description="Unique Identification for request", + required=False, + example="19f035a711644eab84ef5a38ceb5572e", + ), + "message": fields.String( + description="Message", + required=False, + exampple="Request is running", + ), + "report": fields.Nested(report_model), "status": fields.String( description="Status of request", required=True, - example="DONE", - enum=["UNKNOWN", "FAILED", "DONE"], + example=constants.task_status.SUCCESS.value, + enum=[s.value for s in constants.task_status], ), - "result": fields.Nested(report_model), }, ) report_status_response = ns.model( @@ -87,6 +113,10 @@ class ReportStatus(Resource): ) @ns.response(200, "OK", report_status_response) - @ns.expect(report_status_parameters) + @ns.expect(report_status_parameters, validate=True) def post(self): """Request Tern BoM report status/result""" + + payload = request.json + response = status(payload.get("id")) + return response.to_response() diff --git a/tern_api/constants.py b/tern_api/constants.py new file mode 100644 index 0000000..c2edf76 --- /dev/null +++ b/tern_api/constants.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (c) 2022 VMware, Inc. All Rights Reserved. +# SPDX-License-Identifier: BSD-2-Clause + +from enum import Enum + + +class task_status(Enum): + PENDING = "PENDING" + RECEIVED = "RECEIVED" + RUNNING = "RUNNING" + SUCCESS = "SUCCESS" + FAILURE = "FAILURE" diff --git a/tern_api/reports.py b/tern_api/reports.py index a4bb83b..b166490 100644 --- a/tern_api/reports.py +++ b/tern_api/reports.py @@ -3,3 +3,194 @@ # # Copyright (c) 2022 VMware, Inc. All Rights Reserved. # SPDX-License-Identifier: BSD-2-Clause + +import json +import logging +import os +import subprocess +import sys +from dataclasses import asdict, dataclass, field +from typing import Any, Dict, Optional +from uuid import uuid4 + +from tern_api import tern_app, tern_tasks +from tern_api.constants import task_status +from tern_api.utils import TernAPIResponse + + +class TernError(Exception): + """Failure on the Tern execution.""" + + +@dataclass +class DataResponse: + id: str + cache: bool = field(default=True) + message: str = field(default="") + status: str = field(default=task_status.PENDING.value) + report: Dict[str, Any] = field(default_factory=dict) + + def to_dict(self): + """Returns the DataResponse as a dictionary.""" + return asdict(self) + + +def tern(command: list) -> Dict[str, Any]: + """ + Runs the tern CLI using the ``subprocess``. + + Args: + command: Command line as a list (required by subprocess). + + Returns: + report as a dictionary (JSON). + + Raises: + TernError: failure during running the command from tern CLI. + """ + logging.debug(command) + + if type(command) != list: + raise TypeError("command must be a list") + + tern_cmd = subprocess.run(command, capture_output=True) + if tern_cmd.stdout: + json_report = json.loads(tern_cmd.stdout) + return json_report + else: + if tern_cmd.stderr: + logging.debug(tern_cmd.stderr) + + # Is important to return a specific error if the error comes from + # tern CLI, it means the task FINISH, for example, invalid image. + logging.info(tern_cmd.stderr) + raise TernError(tern_cmd.stderr.decode()) + + +@tern_tasks.job +def tern_report( + command: list, cache: bool, cache_file: Optional[str] +) -> Dict[str, Any]: + """ + Tern Report as a task (background) and manages the cache + + Args: + command: Command line as a list (required by subprocess). + cache: Use caching + cache_file: If cache, inform the cache_file + + Return: + Report as Dictionary + """ + if cache: + # If a API user is using the cache, first try to load the cached + # instead doing a new call to the tern. + try: + with open(cache_file, "r") as f: + report = json.load(f) + return report + except FileNotFoundError: + logging.debug(f"Cache file not found: {cache_file}") + # call the tern and dump it to the cache + report = tern(command) + with open(cache_file, "w") as f: + json.dump(report, f, indent=2) + else: + report = tern(command) + + return report + + +def submit(payload: dict) -> TernAPIResponse: + """ + Get the Payload from API and prepare to request the report. + The request will be handled in the background as tern tasks. + + Args: + payload: API Payload + + Return: + API Response + """ + TERN_API_CACHE_DIR = tern_app.config["TERN_API_CACHE_DIR"] + TERN_DEFAULT_REGISTRY = tern_app.config["TERN_DEFAULT_REGISTRY"] + task_id = uuid4().hex + registry = payload.get( + "registry", + ) + image = payload.get("image") + tag = payload.get("tag") + cache = payload.get("cache", True) + cache_file_dir = os.path.join(TERN_API_CACHE_DIR, registry, image) + cache_file = os.path.join(cache_file_dir, f"{tag}.json") + + os.makedirs(cache_file_dir, exist_ok=True) + + report_request_response = DataResponse(id=task_id, cache=cache) + if registry != TERN_DEFAULT_REGISTRY: + registry_image_tag = f"{registry}/{image}:{tag}" + else: + registry_image_tag = f"{image}:{tag}" + + command = ["tern", "report", "-i", registry_image_tag, "-f", "json"] + logging.debug(command) + + tern_report.submit_stored( + task_id, command=command, cache=cache, cache_file=cache_file + ) + + report_request_response.message = "Request submitted." + return TernAPIResponse(report_request_response.to_dict()) + + +def status(task_id: str) -> TernAPIResponse: + """ + Request to the status/result from the tern tasks. + + The tern tasks can have basically the following status: + - PENDING: Not known (yet) by the task manager (initial status). + - RUNNING: Task is running by the task manager. + - SUCCESS: Task has fineshed in the task manager. + - FAILURE: Task has failed before finished. + + Args: + task_id: the unique task ID + """ + data_response = DataResponse(id=task_id, status=task_status.PENDING.value) + + try: + if not tern_tasks.futures.done(task_id): + status = tern_tasks.futures._state(task_id) + if status == task_status.RUNNING.value: + data_response.status = status + + else: + # if the task state is not done, then the task is still running + # we can't know the status, so we return the status as pending + # and remove the task from the task manager + data_response.status = task_status.PENDING.value + tern_tasks.futures.pop(task_id) + + return TernAPIResponse(data_response.to_dict()) + + report = tern_tasks.futures.pop(task_id) + data_response.report = report.result() + data_response.status = task_status.SUCCESS.value + + # It means the task was finished by the task manager (SUCCESS), but the + # report has no data and the error is given to the API user. + except TernError as e: + data_response.status = task_status.SUCCESS.value + response = TernAPIResponse(data_response.to_dict()) + response.errors = {"message": str(e)} + return response + + # Any kind of not expected failure means that the task didn't finished as + # expected (FAIL). + except: # noqa + data_response.status = task_status.FAILURE.value + data_response.message = ( + f"Task could not finish due: {str(sys.exc_info())}" + ) + + return TernAPIResponse(data_response.to_dict()) diff --git a/tern_api/utils.py b/tern_api/utils.py index 9af4e6d..3dac5f1 100644 --- a/tern_api/utils.py +++ b/tern_api/utils.py @@ -6,7 +6,6 @@ from dataclasses import dataclass, field from typing import Any, Dict -from attr import asdict from flask import jsonify from flask.wrappers import Response @@ -17,14 +16,6 @@ class TernAPIResponse: status_code: int = 200 errors: Dict[str, Any] = field(default_factory=dict) - def to_dict(self) -> Dict[str, Any]: - """Converts the dataclass data to a dictionary. - - :return: Response data as a dictionary - :rtype: ``dict`` - """ - return asdict(self) - def to_response(self) -> Response: """Converts the dataclass data to a Flask jsonified format, building a consistent response format for the requests to the API. diff --git a/tests/conftest.py b/tests/conftest.py index d044db9..79e600b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,15 +6,25 @@ import pytest from werkzeug.test import TestResponse -from app import tern_api +from app import tern_app from tests.utils import RequestDataTest +@pytest.fixture(scope="module") +def test_tern_app(): + # Configuration + tern_app.config["TERN_API_CACHE_DIR"] = "FakeCacheDir" + tern_app.config["TERN_DEFAULT_REGISTRY"] = "registry_fake_tests" + + return tern_app + + @pytest.fixture -def api_request(): +def api_request(test_tern_app): def _api_request(request_data: RequestDataTest) -> TestResponse: - with tern_api.test_client() as api_client: - with tern_api.app_context(): + with test_tern_app.test_client() as api_client: + with test_tern_app.app_context(): + if request_data.method.lower() == "get": response = api_client.get( request_data.endpoint, json=request_data.payload @@ -41,3 +51,13 @@ def _api_request(request_data: RequestDataTest) -> TestResponse: return response return _api_request + + +@pytest.fixture +def fake_id(): + class FakeID: + @property + def hex(self): + return "fake-id" + + return FakeID diff --git a/tests/tern_api/test_reports.py b/tests/tern_api/test_reports.py new file mode 100644 index 0000000..b578173 --- /dev/null +++ b/tests/tern_api/test_reports.py @@ -0,0 +1,368 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) 2022 VMware, Inc. All Rights Reserved. +# SPDX-License-Identifier: BSD-2-Clause +import json +import time +from dataclasses import dataclass +from unittest import mock + +import pytest + +from tern_api import reports +from tern_api.constants import task_status +from tern_api.utils import TernAPIResponse + + +@dataclass +class FakeSubprocessResult: + stdout_: bytes + stderr_: bytes + returncode_: int + + @property + def stdout(self): + return self.stdout_ + + @property + def stderr(self): + return self.stderr_ + + @property + def returncode(self): + return self.returncode_ + + +TERN_REPORT = b'{"images": [{"image": "photon", "tag": "3.0"}]}' + + +class TestReports: + @mock.patch("tern_api.reports.subprocess.run") + def test_tern(self, mock_subprocess_run): + """Test tern report""" + + fake_subprocess_result = FakeSubprocessResult( + stdout_=b'{"images": [{"image": "photon", "tag": "3.0"}]}', + stderr_=b"", + returncode_=0, + ) + + mock_subprocess_run.return_value = fake_subprocess_result + test_response = reports.tern( + ["tern", "report", "-i", "phothon:3.0", "-f", "json"] + ) + + assert test_response == json.loads(fake_subprocess_result.stdout_) + + @mock.patch("tern_api.reports.subprocess.run") + def test_tern_command_failed(self, mock_subprocess_run): + """Test tern report with a failure running tern command""" + + fake_subprocess_result = FakeSubprocessResult( + stdout_=b"", + stderr_=b"Failed to run tern command", + returncode_=1, + ) + + mock_subprocess_run.return_value = fake_subprocess_result + + with pytest.raises(reports.TernError) as e: + reports.tern(["tern", "report", "-i", "phothon:3.0", "-f", "json"]) + + assert e.value.args[0] == "Failed to run tern command" + + def test_tern_with_invalid_command_type(self): + """Test tern report with invalid command type for subprocess""" + + with pytest.raises(TypeError) as e: + reports.tern("tern report -i phothon:3.0 -f json") + + assert e.value.args[0] == "command must be a list" + + @mock.patch("builtins.open", mock.mock_open(read_data=TERN_REPORT)) + def test_tern_report_from_cache(self, test_tern_app, fake_id): + """Test tern report using cache and cache is available""" + command = ["tern", "report", "-i", "phothon:3.0", "-f", "json"] + + task_id = fake_id().hex + + with test_tern_app.test_request_context(): + future = reports.tern_report.submit_stored( + task_id, + command=command, + cache=True, + cache_file="fake_cache_file", + ) + + time.sleep(1) + assert future._state == "FINISHED" # futures state + assert future.done() is True + assert future.result() == json.loads(TERN_REPORT) + + @mock.patch("builtins.open") + @mock.patch("tern_api.reports.tern") + @mock.patch("tern_api.reports.json") + def test_tern_report_cache_not_available( + self, mock_json, mock_tern, mock_open, test_tern_app, fake_id + ): + """Test tern report using cache and cache is not available""" + # clean test tern tasks + test_tern_app.config["TERN_TASKS"].futures.pop("fake-id") + + command = ["tern", "report", "-i", "phothon:3.0", "-f", "json"] + task_id = fake_id().hex + + mock_open.side_effect = [ + FileNotFoundError, + mock.mock_open().return_value, + ] + mock_tern.return_value = json.loads(TERN_REPORT) + mock_json.dump.return_value = None + with test_tern_app.test_request_context(): + future = reports.tern_report.submit_stored( + task_id, + command=command, + cache=True, + cache_file="fake_cache_file", + ) + + time.sleep(1) + assert future._state == "FINISHED" # futures state + assert future.done() is True + assert future.result() == json.loads(TERN_REPORT) + + @mock.patch("tern_api.reports.tern") + def test_tern_report_no_cache(self, mock_tern, test_tern_app, fake_id): + """Test tern report without using cache (cache=False)""" + # clean test tern tasks + test_tern_app.config["TERN_TASKS"].futures.pop("fake-id") + + command = ["tern", "report", "-i", "phothon:3.0", "-f", "json"] + task_id = fake_id().hex + + mock_tern.return_value = json.loads(TERN_REPORT) + with test_tern_app.test_request_context(): + future = reports.tern_report.submit_stored( + task_id, + command=command, + cache=False, + cache_file="fake_cache_file", + ) + + time.sleep(1) + assert future._state == "FINISHED" # futures state + assert future.done() is True + assert future.result() == json.loads(TERN_REPORT) + + @mock.patch("tern_api.reports.os") + @mock.patch("tern_api.reports.uuid4") + @mock.patch("tern_api.reports.tern_report") + def test_request( + self, mock_os, mock_uuid4, mock_tern_report, test_tern_app, fake_id + ): + """Test report request""" + + expected_response = TernAPIResponse( + data={ + "message": "Request submitted.", + "id": "fake-id", + "cache": True, + "status": task_status.PENDING.value, + "report": {}, + } + ) + + payload = { + "registry": "registry_fake_tests", + "image": "photon", + "tag": "3.0", + "cache": True, + } + + mock_os.return_value = None + mock_uuid4.return_value.hex = fake_id().hex + mock_tern_report.submit_stored.return_value = None + + with test_tern_app.app_context(): + test_response = reports.submit(payload=payload) + + assert test_response == expected_response + assert test_response.status_code == 200 + + @mock.patch("tern_api.reports.os") + @mock.patch("tern_api.reports.uuid4") + @mock.patch("tern_api.reports.tern_report") + def test_request_using_different_registry( + self, mock_os, mock_uuid4, mock_tern_report, test_tern_app, fake_id + ): + """Test report request using non-default registry""" + + expected_response = TernAPIResponse( + data={ + "message": "Request submitted.", + "id": "fake-id", + "cache": True, + "status": task_status.PENDING.value, + "report": {}, + } + ) + + payload = { + "registry": "another_registry", + "image": "photon", + "tag": "3.0", + "cache": True, + } + + mock_os.return_value = None + mock_uuid4.return_value.hex = fake_id().hex + mock_tern_report.submit_stored.return_value = None + + with test_tern_app.app_context(): + test_response = reports.submit(payload=payload) + + assert test_response == expected_response + + def test_status(self, test_tern_app, fake_id): + """Test report status + + The very basic status is a non existent task id (PENDING). + """ + + # clean test tern tasks + test_tern_app.config["TERN_TASKS"].futures.pop("fake-id") + + task_id = fake_id().hex + expected_response = TernAPIResponse( + reports.DataResponse(id=task_id).to_dict() + ) + + test_result = reports.status(task_id) + assert test_result == expected_response + + @mock.patch("tern_api.reports.tern_tasks") + def test_status_task_not_done_running( + self, mock_tern_tasks, test_tern_app, fake_id + ): + """Test report task status is not done and running""" + + # clean test tern tasks + test_tern_app.config["TERN_TASKS"].futures.pop("fake-id") + + task_id = fake_id().hex + expected_response = TernAPIResponse( + reports.DataResponse( + id=task_id, status=task_status.RUNNING.value + ).to_dict() + ) + + # Task didn't finish yet + mock_tern_tasks.futures.done.return_value = False + # Task return as running + mock_tern_tasks.futures._state.return_value = task_status.RUNNING.value + + test_result = reports.status(task_id) + assert test_result == expected_response + + @mock.patch("tern_api.reports.tern_tasks") + def test_status_task_not_done_state_not_running( + self, mock_tern_tasks, test_tern_app, fake_id + ): + """Test report task status is not done and state not running""" + + # clean test tern tasks + test_tern_app.config["TERN_TASKS"].futures.pop("fake-id") + + task_id = fake_id().hex + expected_response = TernAPIResponse( + reports.DataResponse( + id=task_id, status=task_status.PENDING.value + ).to_dict() + ) + + # Task didn't finish yet + mock_tern_tasks.futures.done.return_value = False + # Task return as running + mock_tern_tasks.futures._state.return_value = "NOT_RUNNING" + + test_result = reports.status(task_id) + assert test_result == expected_response + + @mock.patch("tern_api.reports.tern_tasks") + def test_status_task_success( + self, mock_tern_tasks, test_tern_app, fake_id + ): + """Test report task status is not done and state not running""" + + # clean test tern tasks + test_tern_app.config["TERN_TASKS"].futures.pop("fake-id") + + task_id = fake_id().hex + expected_response = TernAPIResponse( + reports.DataResponse( + id=task_id, + status=task_status.SUCCESS.value, + report=json.loads(TERN_REPORT), + ).to_dict() + ) + + # Task didn't finish yet + mock_tern_tasks.futures.done.return_value = True + + mocked_result = mock.Mock() + mocked_result.result.return_value = json.loads(TERN_REPORT) + mock_tern_tasks.futures.pop.return_value = mocked_result + + test_result = reports.status(task_id) + assert test_result == expected_response + + @mock.patch("tern_api.reports.tern_tasks") + def test_status_tern_error(self, mock_tern_tasks, test_tern_app, fake_id): + """Test report task status success but tern error (finished)""" + + # clean test tern tasks + test_tern_app.config["TERN_TASKS"].futures.pop("fake-id") + + task_id = fake_id().hex + expected_response = TernAPIResponse( + reports.DataResponse( + id=task_id, + status=task_status.SUCCESS.value, + ).to_dict() + ) + expected_response.errors = {"message": "fake error"} + + # Task raise a handled error from Tern, what means the task + # is success finished. + mock_tern_tasks.futures.done.side_effect = reports.TernError( + "fake error" + ) + + test_result = reports.status(task_id) + assert test_result == expected_response + + @mock.patch("tern_api.reports.tern_tasks") + def test_status_task_error(self, mock_tern_tasks, test_tern_app, fake_id): + """Test report task status failed during the process (unfinished)""" + + # clean test tern tasks + test_tern_app.config["TERN_TASKS"].futures.pop("fake-id") + + task_id = fake_id().hex + expected_response = TernAPIResponse( + reports.DataResponse( + id=task_id, + status=task_status.FAILURE.value, + message="Task could not finish due", + ).to_dict() + ) + # Task raise a error, what means the task is didn't finished. + mock_tern_tasks.futures.done.side_effect = TypeError( + "wrong type during process" + ) + + test_result = reports.status(task_id) + assert test_result.status_code == 200 + assert expected_response.data.get("message") in test_result.data.get( + "message" + ), test_result.data.get("message") diff --git a/tests/tern_api/test_utils.py b/tests/tern_api/test_utils.py new file mode 100644 index 0000000..153e0af --- /dev/null +++ b/tests/tern_api/test_utils.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) 2022 VMware, Inc. All Rights Reserved. +# SPDX-License-Identifier: BSD-2-Clause +from tern_api.utils import TernAPIResponse + + +class TestTernAPIResponse: + def test_ternapi_response(self, test_tern_app): + """Test ternapi_response() basic functionality""" + with test_tern_app.app_context(): + response = TernAPIResponse() + + assert response.status_code == 200 + assert response.data == {} + assert response.errors == {} + assert response.to_response().status_code == 200 + assert response.to_response().json == {"data": {}} + + def test_to_response_with_data(self, test_tern_app): + """Test to_response() functionality with data""" + + with test_tern_app.app_context(): + response = TernAPIResponse( + data={"key": "value"}, + ) + + assert response.to_response().status_code == 200 + assert response.to_response().json == {"data": {"key": "value"}} + + def test_to_response_with_errors(self, test_tern_app): + """Test to_response() with errors""" + with test_tern_app.app_context(): + response = TernAPIResponse( + errors={"message": "error"}, + ) + + assert response.to_response().status_code == 200 + assert response.to_response().json == { + "data": {}, + "error": {"message": "error"}, + } + + def test_to_response_not_200(self, test_tern_app): + """Test bto response when is not 200 HTTP code""" + with test_tern_app.app_context(): + response = TernAPIResponse( + errors={"message": "Bad Request"}, status_code=400 + ) + + assert response.to_response().status_code == 400 + assert response.to_response().json == { + "data": {}, + "error": {"message": "Bad Request"}, + } diff --git a/tests/tern_api/v1/test_reports.py b/tests/tern_api/v1/test_reports.py new file mode 100644 index 0000000..7f2cd5d --- /dev/null +++ b/tests/tern_api/v1/test_reports.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) 2022 VMware, Inc. All Rights Reserved. +# SPDX-License-Identifier: BSD-2-Clause +from unittest import mock + +from tern_api.utils import TernAPIResponse +from tests.utils import RequestDataTest + + +class TestAPIReports: + @mock.patch("tern_api.api.v1.reports.submit") + def test_post_reports(self, mock_reports_submit, api_request): + expected_response = { + "data": { + "message": "Request submitted.", + "id": "fake-id", + "cache": True, + } + } + + payload = { + "registry": "registry.hub.docker.com", + "image": "photon", + "tag": "3.0", + "cache": True, + } + + mock_reports_submit.return_value = TernAPIResponse(expected_response) + test_response = api_request( + RequestDataTest( + method="POST", endpoint="/api/v1/reports", payload=payload + ) + ) + assert test_response.status_code == 200 + assert test_response.json.get("data") == expected_response + + def test_post_reports_missing_required_payload(self, api_request): + expected_response = { + "errors": { + "cache": "'cache' is a required property", + "image": "'image' is a required property", + "tag": "'tag' is a required property", + }, + "message": "Input payload validation failed", + } + + test_response = api_request( + RequestDataTest( + method="POST", endpoint="/api/v1/reports", payload={} + ) + ) + assert test_response.status_code == 400 + assert test_response.json == expected_response, test_response.json + + @mock.patch("tern_api.api.v1.reports.status") + def test_post_reports_status(self, mock_reports_status, api_request): + expected_response = { + "data": { + "cache": True, + "id": "19f035a711644eab84ef5a38ceb5572e", + "message": "", + "report": {}, + "status": "PENDING", + } + } + + payload = { + "id": "fake-id", + } + + mock_reports_status.return_value = TernAPIResponse(expected_response) + + test_response = api_request( + RequestDataTest( + method="POST", + endpoint="/api/v1/reports/status", + payload=payload, + ) + ) + assert test_response.status_code == 200 + assert test_response.json.get("data") == expected_response + + @mock.patch("tern_api.api.v1.reports.status") + def test_post_reports_status_invalid_payload( + self, mock_reports_status, api_request + ): + expected_response = { + "errors": { + "id": "'id' is a required property", + }, + "message": "Input payload validation failed", + } + + payload = {} + + mock_reports_status.return_value = TernAPIResponse(expected_response) + + test_response = api_request( + RequestDataTest( + method="POST", + endpoint="/api/v1/reports/status", + payload=payload, + ) + ) + assert test_response.status_code == 200 + assert test_response.json.get("data") == expected_response diff --git a/tests/tern_api/v1/test_version.py b/tests/tern_api/v1/test_version.py index 3581e25..eca7941 100644 --- a/tests/tern_api/v1/test_version.py +++ b/tests/tern_api/v1/test_version.py @@ -6,7 +6,7 @@ from tests.utils import RequestDataTest -class TestVersion: +class TestAPIVersion: def test_version(self, api_request): expected_response = { "api": version, diff --git a/tox.ini b/tox.ini index 3ba0654..80c93e5 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py39,py310,lint +envlist = py39,lint,test,doc-offline-api [flake8] exclude = ownca/__init__.py,venv,.venv,settings.py,.git,.tox,dist,docs,*lib/python*,*egg,build,tools @@ -15,13 +15,18 @@ commands = black -l79 --check --diff . [testenv:doc-offline-api] +allowlist_externals = + diff + rm commands = - python -c "import app; app.export_swagger_json('docs/swagger.json')" - git diff main -- docs/swagger.json + python -c "import app; app.export_swagger_json('docs/.test_swagger.json')" + diff docs/swagger.json docs/.test_swagger.json +commands_post = + rm docs/.test_swagger.json [testenv:test] commands = - coverage run -m pytest + coverage run -m pytest -vv coverage report [gh-actions]