From 905cb49b77dca8cf5afc46c2ff1c0b0cf2d2b88a Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Mon, 25 Sep 2023 17:56:22 +0200 Subject: [PATCH 01/71] Dockerfile with aws lambda python image for deploying docker image to lambda --- rest/Dockerfile | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 rest/Dockerfile diff --git a/rest/Dockerfile b/rest/Dockerfile new file mode 100644 index 00000000..7068f9be --- /dev/null +++ b/rest/Dockerfile @@ -0,0 +1,24 @@ +# use base aws lambda docker image for python (all the sources listed bellow) +# https://www.cloudtechsimplified.com/run-docker-containers-images-from-ecr-in-aws-lambda-along-with-cicd-python/ +# https://docs.aws.amazon.com/lambda/latest/dg/python-image.html#python-image-instructions +# https://gallery.ecr.aws/lambda/python/ +# https://zebradil.me/post/2018-05-25-pipenv-for-aws-lambda/ +# https://stackoverflow.com/questions/48381918/how-do-i-install-git-using-aws-lambda +# https://stackoverflow.com/a/68787552 +FROM public.ecr.aws/lambda/python:3.9 + +# install pipenv and git that are needed to install dependencies +RUN yum -y install git +RUN pip install pipenv + +# copy the /rest folder +COPY ./ ${LAMBDA_TASK_ROOT} + +# create requirements.txt file and +# install dependencies with pip +RUN pipenv requirements > requirements.txt +RUN pip install -r requirements.txt + +# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile) +# app.app is the same as in zappa_settings.json.template +CMD [ "app.app" ] \ No newline at end of file From 343804ad16c6ae27b32d3520fd849a5537b6645d Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Mon, 2 Oct 2023 14:40:36 +0200 Subject: [PATCH 02/71] try https://ianwhitestone.work/zappa-serverless-docker/ tutorial --- rest/Dockerfile | 18 +- rest/Pipfile | 3 +- rest/Pipfile.lock | 439 +++++++++++++++++++++++++++++----------------- 3 files changed, 295 insertions(+), 165 deletions(-) diff --git a/rest/Dockerfile b/rest/Dockerfile index 7068f9be..3b2dd73a 100644 --- a/rest/Dockerfile +++ b/rest/Dockerfile @@ -1,10 +1,14 @@ -# use base aws lambda docker image for python (all the sources listed bellow) +# resources # https://www.cloudtechsimplified.com/run-docker-containers-images-from-ecr-in-aws-lambda-along-with-cicd-python/ # https://docs.aws.amazon.com/lambda/latest/dg/python-image.html#python-image-instructions # https://gallery.ecr.aws/lambda/python/ # https://zebradil.me/post/2018-05-25-pipenv-for-aws-lambda/ # https://stackoverflow.com/questions/48381918/how-do-i-install-git-using-aws-lambda # https://stackoverflow.com/a/68787552 +# https://github.com/zappa/Zappa#docker-workflows +# https://ianwhitestone.work/zappa-serverless-docker/ + +# use base aws lambda docker image for python (all the sources listed bellow) FROM public.ecr.aws/lambda/python:3.9 # install pipenv and git that are needed to install dependencies @@ -21,4 +25,14 @@ RUN pip install -r requirements.txt # Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile) # app.app is the same as in zappa_settings.json.template -CMD [ "app.app" ] \ No newline at end of file +# CMD [ "app.app" ] + +# https://ianwhitestone.work/zappa-serverless-docker/ +# Grab the zappa handler.py and put it in the working directory +RUN ZAPPA_HANDLER_PATH=$( \ + python -c "from zappa import handler; print (handler.__file__)" \ + ) \ + && echo $ZAPPA_HANDLER_PATH \ + && cp $ZAPPA_HANDLER_PATH ${LAMBDA_TASK_ROOT} + +CMD [ "handler.lambda_handler" ] \ No newline at end of file diff --git a/rest/Pipfile b/rest/Pipfile index 96297d0f..1eb42d14 100644 --- a/rest/Pipfile +++ b/rest/Pipfile @@ -14,6 +14,7 @@ black = "==22.3.0" python_version = "3.9" [packages] +zappa = "==0.54.1" flask = "==2.0.3" pytest = "==7.0.1" boto3 = "==1.21.18" @@ -31,4 +32,4 @@ placebo = "==0.9.0" rioxarray = "*" zarr = "*" netcdf4 = "*" -pip-tools = "==6.1.0" +werkzeug = "==2.3.7" diff --git a/rest/Pipfile.lock b/rest/Pipfile.lock index 51191fd4..ccfbb7d5 100644 --- a/rest/Pipfile.lock +++ b/rest/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "74cc999abd077d3e6f626772ecec2f996c3fc6a0afd10dd32ac71b5e63de92d9" + "sha256": "ef15fcd6688c1ca07629c7593646d428bf0158563a13b96def7280218fd5b68f" }, "pipfile-spec": 6, "requires": { @@ -32,6 +32,14 @@ "markers": "python_version >= '3.7'", "version": "==2.4.0" }, + "argcomplete": { + "hashes": [ + "sha256:d5d1e5efd41435260b8f85673b74ea2e883affcbec9f4230c582689e8e78251b", + "sha256:d97c036d12a752d1079f190bc1521c545b941fda89ad85d15afa909b4d1b9a99" + ], + "markers": "python_version >= '3.6'", + "version": "==3.1.2" + }, "asciitree": { "hashes": [ "sha256:4aa4b9b649f85e3fcb343363d97564aa1fb62e249677f2e18a96765145cc0f6e" @@ -73,72 +81,68 @@ }, "cffi": { "hashes": [ - "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5", - "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef", - "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104", - "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426", - "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405", - "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375", - "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a", - "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e", - "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc", - "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf", - "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185", - "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497", - "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3", - "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35", - "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c", - "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83", - "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21", - "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca", - "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984", - "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac", - "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd", - "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee", - "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a", - "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2", - "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192", - "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7", - "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585", - "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f", - "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e", - "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27", - "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b", - "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e", - "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e", - "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d", - "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c", - "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415", - "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82", - "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02", - "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314", - "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325", - "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c", - "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3", - "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914", - "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045", - "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d", - "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9", - "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5", - "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2", - "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c", - "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3", - "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2", - "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8", - "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d", - "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d", - "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9", - "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162", - "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76", - "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4", - "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e", - "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9", - "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6", - "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b", - "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01", - "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0" - ], - "version": "==1.15.1" + "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc", + "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a", + "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417", + "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab", + "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520", + "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36", + "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743", + "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8", + "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed", + "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684", + "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56", + "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324", + "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d", + "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235", + "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e", + "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088", + "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000", + "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7", + "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e", + "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673", + "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c", + "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe", + "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2", + "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098", + "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8", + "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a", + "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0", + "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b", + "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896", + "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e", + "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9", + "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2", + "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b", + "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6", + "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404", + "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f", + "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0", + "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4", + "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc", + "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936", + "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba", + "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872", + "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb", + "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614", + "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1", + "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d", + "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969", + "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b", + "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4", + "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627", + "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956", + "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357" + ], + "markers": "python_version >= '3.8'", + "version": "==1.16.0" + }, + "cfn-flip": { + "hashes": [ + "sha256:003e02a089c35e1230ffd0e1bcfbbc4b12cc7d2deb2fcc6c4228ac9819307362", + "sha256:faca8e77f0d32fb84cce1db1ef4c18b14a325d31125dae73c13bcc01947d2722" + ], + "version": "==1.3.0" }, "cftime": { "hashes": [ @@ -227,11 +231,17 @@ }, "dataclasses-json": { "hashes": [ - "sha256:5bcd5c1d946a69ef94d8cbb726d87af1256f7ba0898809c7695d299ab9b45122", - "sha256:77cbc80041007c27db90abad79eeed9ea00d4627324b1208266379a6ebdc81e4" + "sha256:1bd8418a61fe3d588bb0079214d7fb71d44937da40742b787256fd53b26b6c80", + "sha256:a53c220c35134ce08211a1057fd0e5bf76dc5331627c6b241cacbc570a89faae" ], "markers": "python_version >= '3.7' and python_version < '4.0'", - "version": "==0.6.0" + "version": "==0.6.1" + }, + "durationpy": { + "hashes": [ + "sha256:5ef9416b527b50d722f34655becfb75e49228eb82f87b855ed1911b3314b5408" + ], + "version": "==0.5" }, "entrypoints": { "hashes": [ @@ -274,6 +284,20 @@ "index": "pypi", "version": "==0.14.0" }, + "future": { + "hashes": [ + "sha256:34a17436ed1e96697a86f9de3d15a3b0be01d8bc8de9c1dffd59fb8234ed5307" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.18.3" + }, + "hjson": { + "hashes": [ + "sha256:55af475a27cf83a7969c808399d7bccdec8fb836a07ddbd574587593b9cdcf75", + "sha256:65713cdcf13214fb554eb8b4ef803419733f4f5e551047c9b711098ab7186b89" + ], + "version": "==3.1.0" + }, "honeycomb-beeline": { "hashes": [ "sha256:16e1b8907934b0b5ff4ab1dcd019814bf1e7498a45bc75fc5446c5b7f072d036", @@ -331,6 +355,13 @@ "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.10.0" }, + "kappa": { + "hashes": [ + "sha256:4b5b372872f25d619e427e04282551048dc975a107385b076b3ffc6406a15833", + "sha256:4d6b7b3accce4a0aaaac92b36237a6304f0f2fffbbe3caea3f7c9f52d12c9989" + ], + "version": "==0.6.0" + }, "libhoney": { "hashes": [ "sha256:04b7358513ff3eb11c07c6d4dd726de45b93a918ffe42748710c0e53a616c04a", @@ -527,11 +558,11 @@ }, "packaging": { "hashes": [ - "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61", - "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f" + "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5", + "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7" ], "markers": "python_version >= '3.7'", - "version": "==23.1" + "version": "==23.2" }, "pandas": { "hashes": [ @@ -564,14 +595,6 @@ "markers": "python_version >= '3.9'", "version": "==2.1.1" }, - "pep517": { - "hashes": [ - "sha256:4ba4446d80aed5b5eac6509ade100bff3e7943a8489de249654a5ae9b33ee35b", - "sha256:ae69927c5c172be1add9203726d4b84cf3ebad1edcd5f71fcdc746e66e829f59" - ], - "markers": "python_version >= '3.6'", - "version": "==0.13.0" - }, "pg-to-evalscript": { "hashes": [ "sha256:298f00fce66b1251e543b961c4c0a36f35369929fe76b2882850f31de8ac2fd5", @@ -648,15 +671,6 @@ "markers": "python_version >= '3.7'", "version": "==23.2.1" }, - "pip-tools": { - "hashes": [ - "sha256:197e3f8839095ccec3ad1ef410e0804c07d9f17dff1c340fb417ca2b63feacc9", - "sha256:400bf77e29cca48c31abc210042932bb52dcc138ef4ea4d52c5db429aa8ae6ee" - ], - "index": "pypi", - "markers": "python_version >= '3.6'", - "version": "==6.1.0" - }, "placebo": { "hashes": [ "sha256:03157f8527bbc2965b71b88f4a139ef8038618b346787f20d63e3c5da541b047" @@ -756,6 +770,14 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.8.2" }, + "python-slugify": { + "hashes": [ + "sha256:70ca6ea68fe63ecc8fa4fcf00ae651fc8a5d02d93dcd12ae6d4fc7ca46c4d395", + "sha256:ce0d46ddb668b3be82f4ed5e503dbc33dd815d83e2eb6824211310d3fb172a27" + ], + "markers": "python_version >= '3.7'", + "version": "==8.0.1" + }, "pytz": { "hashes": [ "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b", @@ -763,6 +785,62 @@ ], "version": "==2023.3.post1" }, + "pyyaml": { + "hashes": [ + "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5", + "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc", + "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df", + "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741", + "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206", + "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27", + "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595", + "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62", + "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98", + "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696", + "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290", + "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9", + "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d", + "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6", + "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867", + "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47", + "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486", + "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6", + "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3", + "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007", + "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938", + "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0", + "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c", + "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735", + "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d", + "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28", + "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4", + "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba", + "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8", + "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5", + "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd", + "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3", + "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0", + "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515", + "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c", + "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c", + "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924", + "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34", + "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43", + "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859", + "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673", + "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54", + "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a", + "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b", + "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab", + "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa", + "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c", + "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585", + "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d", + "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f" + ], + "markers": "python_version >= '3.6'", + "version": "==6.0.1" + }, "rasterio": { "hashes": [ "sha256:0323332ed1bfad522e53a3da45e0d3453e603862c3d2c08d8a639a7be76853fb", @@ -902,13 +980,28 @@ ], "version": "==3.3.0" }, + "text-unidecode": { + "hashes": [ + "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8", + "sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93" + ], + "version": "==1.3" + }, "tifffile": { "hashes": [ - "sha256:03574dc63a333111227c7710b52033def22db0072f9b49fdb25c9eea8249b026", - "sha256:391d49e5e10a0f50f627b9083116f965e37d40bfe4e4713154e5edd57591590c" + "sha256:1de47fa945fddaade256e25ad4f375ae65547f3c1354063aded881c32a64cf89", + "sha256:67e355e4595aab397f8405d04afe1b4ae7c6f62a44e22d933fee1a571a48c7ae" ], "markers": "python_version >= '3.9'", - "version": "==2023.9.18" + "version": "==2023.9.26" + }, + "toml": { + "hashes": [ + "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", + "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.10.2" }, "tomli": { "hashes": [ @@ -926,6 +1019,14 @@ "markers": "python_version >= '3.7'", "version": "==4.66.1" }, + "troposphere": { + "hashes": [ + "sha256:4f8703667a61ded61f061042eb054c061014c9c3ad64957e344019225c954490", + "sha256:7deef2ecb753197c766a143adfef980b9be357eb99644c37a9eed45f9ce53a19" + ], + "markers": "python_full_version >= '3.7.0'", + "version": "==4.4.1" + }, "typing-extensions": { "hashes": [ "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0", @@ -968,9 +1069,18 @@ "sha256:2b8c0e447b4b9dbcc85dd97b6eeb4dcbaf6c8b6c3be0bd654e25553e0a2157d8", "sha256:effc12dba7f3bd72e605ce49807bbe692bd729c3bb122a3b91747a6ae77df528" ], + "index": "pypi", "markers": "python_version >= '3.8'", "version": "==2.3.7" }, + "wheel": { + "hashes": [ + "sha256:0c5ac5ff2afb79ac23ab82bab027a0be7b5dbcf2e54dc50efe4bf507de1f7985", + "sha256:75909db2664838d015e3d9139004ee16711748a52c8f336b52882266540215d8" + ], + "markers": "python_version >= '3.7'", + "version": "==0.41.2" + }, "wrapt": { "hashes": [ "sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0", @@ -1052,13 +1162,28 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==1.15.0" }, + "wsgi-request-logger": { + "hashes": [ + "sha256:445d7ec52799562f812006394d0b4a7064b37084c6ea6bd74ea7a2136c97ed83" + ], + "version": "==0.4.6" + }, "xarray": { "hashes": [ - "sha256:825c6d64202a731a4e49321edd1e9dfabf4be06802f1b8c8a3c00a3ebfc8cedf", - "sha256:eb42b56aea2c7d5db2a7d0c33fb005b78eb5c4421eb747f2ced138c70b5c204e" + "sha256:271955c05dc626dad37791a7807d920aaf9c64cac71d03b45ec7e402cc646603", + "sha256:3fc4a558bd70968040a4e1cefc6ddb3f9a7a86ef6a48e67857156ffe655d3a66" ], "markers": "python_version >= '3.9'", - "version": "==2023.8.0" + "version": "==2023.9.0" + }, + "zappa": { + "hashes": [ + "sha256:5d4e45fee5fe988d0198bd75d6a1df5b89b3b201260b5d9d1d50a06ad735fd76", + "sha256:85d6103b5ae936a181e36011dd48ce9c4f7c0603357684e56149faf27fa92cd9", + "sha256:c77b7017f99a2042f3cce99eef188684e857de9a797e008dead6260c4df61207" + ], + "index": "pypi", + "version": "==0.54.1" }, "zarr": { "hashes": [ @@ -1144,72 +1269,61 @@ }, "cffi": { "hashes": [ - "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5", - "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef", - "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104", - "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426", - "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405", - "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375", - "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a", - "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e", - "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc", - "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf", - "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185", - "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497", - "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3", - "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35", - "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c", - "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83", - "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21", - "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca", - "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984", - "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac", - "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd", - "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee", - "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a", - "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2", - "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192", - "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7", - "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585", - "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f", - "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e", - "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27", - "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b", - "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e", - "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e", - "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d", - "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c", - "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415", - "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82", - "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02", - "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314", - "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325", - "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c", - "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3", - "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914", - "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045", - "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d", - "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9", - "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5", - "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2", - "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c", - "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3", - "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2", - "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8", - "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d", - "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d", - "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9", - "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162", - "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76", - "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4", - "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e", - "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9", - "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6", - "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b", - "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01", - "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0" - ], - "version": "==1.15.1" + "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc", + "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a", + "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417", + "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab", + "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520", + "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36", + "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743", + "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8", + "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed", + "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684", + "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56", + "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324", + "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d", + "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235", + "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e", + "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088", + "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000", + "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7", + "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e", + "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673", + "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c", + "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe", + "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2", + "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098", + "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8", + "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a", + "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0", + "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b", + "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896", + "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e", + "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9", + "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2", + "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b", + "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6", + "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404", + "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f", + "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0", + "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4", + "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc", + "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936", + "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba", + "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872", + "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb", + "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614", + "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1", + "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d", + "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969", + "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b", + "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4", + "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627", + "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956", + "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357" + ], + "markers": "python_version >= '3.8'", + "version": "==1.16.0" }, "cfn-flip": { "hashes": [ @@ -1656,6 +1770,7 @@ "sha256:2b8c0e447b4b9dbcc85dd97b6eeb4dcbaf6c8b6c3be0bd654e25553e0a2157d8", "sha256:effc12dba7f3bd72e605ce49807bbe692bd729c3bb122a3b91747a6ae77df528" ], + "index": "pypi", "markers": "python_version >= '3.8'", "version": "==2.3.7" }, From 078a75437011c1c43993ecd0f6c7cf7fd2047e41 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Thu, 5 Oct 2023 09:48:53 +0200 Subject: [PATCH 03/71] add demo gitlab ci job for deploying docker to lambda --- .gitlab-ci.yml | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7a4e6091..d37442a7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -103,6 +103,58 @@ deploy-testing-rest-lambda: - pipenv run bash -c "DEPLOYMENT_TYPE=testing python dynamodb/dynamodb.py" +deploy-testing-rest-lambda_docker: + stage: deploy + when: manual + only: + variables: + - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/ + image: python:3.9.13 + before_script: + # configure aws access credentials: + - mkdir -p ~/.aws + - echo -e "[default]\nregion=eu-central-1" > ~/.aws/config + - echo -e "[default]\naws_access_key_id=$TESTINGZAPPA_AWS_ACCESS_KEY_ID\naws_secret_access_key=$TESTINGZAPPA_AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials + - pip install pipenv==2021.5.29 + script: + - chmod +x download-process-definitions.sh + - ./download-process-definitions.sh + - cd rest/ + - pipenv install --dev + # create zappa_settings.json on-the-fly: + - cp zappa_settings.json.template zappa_settings.json + - sed -i "s/@@AWS_ACCESS_KEY_ID@@/$TESTINGDATA_AWS_ACCESS_KEY_ID/g" zappa_settings.json + - sed -i "s#@@AWS_SECRET_ACCESS_KEY@@#$TESTINGDATA_AWS_SECRET_ACCESS_KEY#g" zappa_settings.json + - sed -i "s/@@TESTING_SH_CLIENT_ID@@/$TESTING_SH_CLIENT_ID/g" zappa_settings.json + - sed -i "s/@@TESTING_SH_CLIENT_SECRET@@/$TESTING_SH_CLIENT_SECRET/g" zappa_settings.json + - sed -i "s/@@BACKEND_VERSION@@/$CI_COMMIT_TAG/g" zappa_settings.json + - sed -i "s/@@RESULTS_S3_BUCKET_NAME_MAIN@@/$RESULTS_S3_BUCKET_NAME_MAIN/g" zappa_settings.json + - sed -i "s/@@RESULTS_S3_BUCKET_NAME_CREODIAS@@/$RESULTS_S3_BUCKET_NAME_CREODIAS/g" zappa_settings.json + - sed -i "s/@@RESULTS_S3_BUCKET_NAME_USWEST@@/$RESULTS_S3_BUCKET_NAME_USWEST/g" zappa_settings.json + - sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN/g" zappa_settings.json + - sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS/g" zappa_settings.json + - sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST/g" zappa_settings.json + - sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN@@#$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN#g" zappa_settings.json + - sed -i "s/@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS@@/$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS/g" zappa_settings.json + - sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST@@#$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST#g" zappa_settings.json + - sed -i "s#@@USAGE_REPORTING_BASE_URL_TESTING@@#$USAGE_REPORTING_BASE_URL_TESTING#g" zappa_settings.json + - sed -i "s#@@USAGE_REPORTING_AUTH_URL_TESTING@@#$USAGE_REPORTING_AUTH_URL_TESTING#g" zappa_settings.json + - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_ID_TESTING@@#$USAGE_REPORTING_AUTH_CLIENT_ID_TESTING#g" zappa_settings.json + - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING@@#$USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING#g" zappa_settings.json + - sed -i "s#@@LOGGING_LEVEL@@#$LOGGING_LEVEL_TESTING#g" zappa_settings.json + - zappa save-python-settings-file testing + # needs to be run only once!!! + - aws ecr create-repository --repository-name openeo-zappa-docker --image-scanning-configuration scanOnPush=true --region region + # - pipenv run zappa deploy testing || pipenv run zappa update testing + + # HOW TO DO THAT IN DOCKER? + # ensure tables are created: + # - export AWS_ACCESS_KEY_ID="$TESTING_AWS_ACCESS_KEY_ID" + # - export AWS_SECRET_ACCESS_KEY="$TESTING_AWS_SECRET_ACCESS_KEY" + # - pipenv run bash -c "DEPLOYMENT_TYPE=testing python dynamodb/dynamodb.py" + + + deploy-production-rest-lambda: stage: deploy when: manual From fadcb6f84ac18b8c96bb80bf8836fbbca845ce5e Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Thu, 5 Oct 2023 12:27:03 +0200 Subject: [PATCH 04/71] run zappa in pipenv for ci job for docker lambda --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d37442a7..af4be4a0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -142,7 +142,7 @@ deploy-testing-rest-lambda_docker: - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_ID_TESTING@@#$USAGE_REPORTING_AUTH_CLIENT_ID_TESTING#g" zappa_settings.json - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING@@#$USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING#g" zappa_settings.json - sed -i "s#@@LOGGING_LEVEL@@#$LOGGING_LEVEL_TESTING#g" zappa_settings.json - - zappa save-python-settings-file testing + - pipenv run zappa save-python-settings-file testing # needs to be run only once!!! - aws ecr create-repository --repository-name openeo-zappa-docker --image-scanning-configuration scanOnPush=true --region region # - pipenv run zappa deploy testing || pipenv run zappa update testing From d337bf8fc1b174200da96544c5bbd870d2b7c3ba Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Thu, 5 Oct 2023 14:26:51 +0200 Subject: [PATCH 05/71] install awscli for CI job --- rest/Pipfile | 1 + rest/Pipfile.lock | 62 +++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 53 insertions(+), 10 deletions(-) diff --git a/rest/Pipfile b/rest/Pipfile index 1eb42d14..10b8c5c2 100644 --- a/rest/Pipfile +++ b/rest/Pipfile @@ -9,6 +9,7 @@ pylint = "==2.12.2" responses = "==0.21.0" cryptography = "==36.0.1" black = "==22.3.0" +awscli = "*" [requires] python_version = "3.9" diff --git a/rest/Pipfile.lock b/rest/Pipfile.lock index ccfbb7d5..9940af8c 100644 --- a/rest/Pipfile.lock +++ b/rest/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "ef15fcd6688c1ca07629c7593646d428bf0158563a13b96def7280218fd5b68f" + "sha256": "73203f614ce9187603be99b84b2b0507b49e49583d2a1ad0dfff04002fc99427" }, "pipfile-spec": 6, "requires": { @@ -1052,11 +1052,11 @@ }, "urllib3": { "hashes": [ - "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f", - "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14" + "sha256:24d6a242c28d29af46c3fae832c36db3bbebcc533dd1bb549172cd739c82df21", + "sha256:94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==1.26.16" + "version": "==1.26.17" }, "utm": { "hashes": [ @@ -1212,6 +1212,15 @@ "markers": "python_full_version >= '3.6.2'", "version": "==2.9.3" }, + "awscli": { + "hashes": [ + "sha256:3a7d9260ecb44e677f04640fd9959fb4310189e39ef0a42fbb652888843890a3", + "sha256:54772140fa9fe72c36f1214cd8f2a210af420940983d8f663f5cdf4b103b7e58" + ], + "index": "pypi", + "markers": "python_version >= '3.6'", + "version": "==1.22.101" + }, "black": { "hashes": [ "sha256:06f9d8846f2340dfac80ceb20200ea5d1b3f181dd0556b47af4e8e0b24fa0a6b", @@ -1348,6 +1357,14 @@ "markers": "python_version >= '3.7'", "version": "==8.1.7" }, + "colorama": { + "hashes": [ + "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff", + "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==0.4.3" + }, "cryptography": { "hashes": [ "sha256:0a817b961b46894c5ca8a66b599c745b9a3d9f822725221f0e0fe49dc043a3a3", @@ -1375,6 +1392,15 @@ "markers": "python_version >= '3.6'", "version": "==36.0.1" }, + "docutils": { + "hashes": [ + "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0", + "sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827", + "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.15.2" + }, "durationpy": { "hashes": [ "sha256:5ef9416b527b50d722f34655becfb75e49228eb82f87b855ed1911b3314b5408" @@ -1574,11 +1600,19 @@ }, "platformdirs": { "hashes": [ - "sha256:b45696dab2d7cc691a3226759c0d3b00c47c8b6e293d96f6436f733303f77f6d", - "sha256:d7c24979f292f916dc9cbf8648319032f551ea8c49a4c9bf2fb556a02070ec1d" + "sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3", + "sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e" ], "markers": "python_version >= '3.7'", - "version": "==3.10.0" + "version": "==3.11.0" + }, + "pyasn1": { + "hashes": [ + "sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57", + "sha256:97b7290ca68e62a832558ec3976f15cbf911bf5d7c7039d8b861c2a0ece69fde" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==0.5.0" }, "pycparser": { "hashes": [ @@ -1686,6 +1720,14 @@ "markers": "python_version >= '3.7'", "version": "==0.21.0" }, + "rsa": { + "hashes": [ + "sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2", + "sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9" + ], + "markers": "python_version >= '3.5' and python_version < '4'", + "version": "==4.7.2" + }, "s3transfer": { "hashes": [ "sha256:7a6f4c4d1fdb9a2b640244008e142cbc2cd3ae34b386584ef044dd0f27101971", @@ -1759,11 +1801,11 @@ }, "urllib3": { "hashes": [ - "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f", - "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14" + "sha256:24d6a242c28d29af46c3fae832c36db3bbebcc533dd1bb549172cd739c82df21", + "sha256:94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==1.26.16" + "version": "==1.26.17" }, "werkzeug": { "hashes": [ From 0bd41991832286744787ffc3515f50c73980a537 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Thu, 5 Oct 2023 14:40:15 +0200 Subject: [PATCH 06/71] run aws in pipenv --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index af4be4a0..c6dc6263 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -144,7 +144,7 @@ deploy-testing-rest-lambda_docker: - sed -i "s#@@LOGGING_LEVEL@@#$LOGGING_LEVEL_TESTING#g" zappa_settings.json - pipenv run zappa save-python-settings-file testing # needs to be run only once!!! - - aws ecr create-repository --repository-name openeo-zappa-docker --image-scanning-configuration scanOnPush=true --region region + - pipenv run aws ecr create-repository --repository-name openeo-zappa-docker --image-scanning-configuration scanOnPush=true --region region # - pipenv run zappa deploy testing || pipenv run zappa update testing # HOW TO DO THAT IN DOCKER? From 98fb0836d95244129732e1bc034bdf559a5e81c2 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Wed, 11 Oct 2023 10:37:50 +0200 Subject: [PATCH 07/71] add CI job for deploying docker image to lambda --- .gitlab-ci.yml | 28 ++++++++++++++------- rest/zappa_settings.json.template | 41 +++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c6dc6263..33ac8399 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -103,7 +103,7 @@ deploy-testing-rest-lambda: - pipenv run bash -c "DEPLOYMENT_TYPE=testing python dynamodb/dynamodb.py" -deploy-testing-rest-lambda_docker: +deploy-testing-rest-lambda-docker: stage: deploy when: manual only: @@ -142,16 +142,26 @@ deploy-testing-rest-lambda_docker: - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_ID_TESTING@@#$USAGE_REPORTING_AUTH_CLIENT_ID_TESTING#g" zappa_settings.json - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING@@#$USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING#g" zappa_settings.json - sed -i "s#@@LOGGING_LEVEL@@#$LOGGING_LEVEL_TESTING#g" zappa_settings.json + # generate zappa_settings.py for docker image - pipenv run zappa save-python-settings-file testing - # needs to be run only once!!! - - pipenv run aws ecr create-repository --repository-name openeo-zappa-docker --image-scanning-configuration scanOnPush=true --region region - # - pipenv run zappa deploy testing || pipenv run zappa update testing - - # HOW TO DO THAT IN DOCKER? + # build docker image + - docker build -t openeo-sh-zappa:latest . + # login for AWS ECR docker + - aws ecr get-login-password --region eu-central-1 | docker login --username AWS --password-stdin 127163565383.dkr.ecr.eu-central-1.amazonaws.com + # re-tag the built image + - docker tag openeo-sh-zappa:latest 127163565383.dkr.ecr.eu-central-1.amazonaws.com/openeo-sh-zappa:latest + # push docker image to AWS ECR + - docker push 127163565383.dkr.ecr.eu-central-1.amazonaws.com/openeo-sh-zappa:latest + # deploy lambda with new docker image + - pipenv run zappa deploy testing-docker -d 127163565383.dkr.ecr.eu-central-1.amazonaws.com/openeo-sh-zappa:latest || pipenv run zappa update testing-docker -d 127163565383.dkr.ecr.eu-central-1.amazonaws.com/openeo-sh-zappa:latest # ensure tables are created: - # - export AWS_ACCESS_KEY_ID="$TESTING_AWS_ACCESS_KEY_ID" - # - export AWS_SECRET_ACCESS_KEY="$TESTING_AWS_SECRET_ACCESS_KEY" - # - pipenv run bash -c "DEPLOYMENT_TYPE=testing python dynamodb/dynamodb.py" + - export AWS_ACCESS_KEY_ID="$TESTING_AWS_ACCESS_KEY_ID" + - export AWS_SECRET_ACCESS_KEY="$TESTING_AWS_SECRET_ACCESS_KEY" + - pipenv run bash -c "DEPLOYMENT_TYPE=testing python dynamodb/dynamodb.py" + + + # needs to be run only once!!! + # - pipenv run aws ecr create-repository --repository-name openeo-zappa-docker --image-scanning-configuration scanOnPush=true --region region diff --git a/rest/zappa_settings.json.template b/rest/zappa_settings.json.template index 43c52302..9cd629c3 100644 --- a/rest/zappa_settings.json.template +++ b/rest/zappa_settings.json.template @@ -40,6 +40,47 @@ "project": "openEO" } }, + "testing-docker": { + "app_function": "app.app", + "aws_region": "eu-central-1", + "lambda_description": "Testing deploy of Sentinel Hub openEO backend", + "environment_variables": { + "DEPLOYMENT_TYPE": "testing", + "AWS_ACCESS_KEY_ID": "@@AWS_ACCESS_KEY_ID@@", + "AWS_SECRET_ACCESS_KEY": "@@AWS_SECRET_ACCESS_KEY@@", + "SH_CLIENT_ID": "@@TESTING_SH_CLIENT_ID@@", + "SH_CLIENT_SECRET": "@@TESTING_SH_CLIENT_SECRET@@", + "BACKEND_VERSION": "@@BACKEND_VERSION@@", + "RESULTS_S3_BUCKET_NAME_MAIN": "@@RESULTS_S3_BUCKET_NAME_MAIN@@", + "RESULTS_S3_BUCKET_NAME_CREODIAS": "@@RESULTS_S3_BUCKET_NAME_CREODIAS@@", + "RESULTS_S3_BUCKET_NAME_USWEST": "@@RESULTS_S3_BUCKET_NAME_USWEST@@", + "RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN": "@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN@@", + "RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS": "@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS@@", + "RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST": "@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST@@", + "RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN": "@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN@@", + "RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS": "@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS@@", + "RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST": "@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST@@", + "USAGE_REPORTING_BASE_URL": "@@USAGE_REPORTING_BASE_URL_TESTING@@", + "USAGE_REPORTING_AUTH_URL": "@@USAGE_REPORTING_AUTH_URL_TESTING@@", + "USAGE_REPORTING_AUTH_CLIENT_ID": "@@USAGE_REPORTING_AUTH_CLIENT_ID_TESTING@@", + "USAGE_REPORTING_AUTH_CLIENT_SECRET": "@@USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING@@", + "LOGGING_LEVEL": "@@LOGGING_LEVEL@@" + }, + "memory_size": 512, + "timeout_seconds": 30, + "profile_name": "default", + "project_name": "openeo", + "runtime": "python3.9", + "manage_roles": false, + "role_arn": "arn:aws:iam::127163565383:role/openEO-dev-role", + "s3_bucket": "openeodevdeploy", + "tags": { + "cost:ou": "frontend", + "cost:contact": "dthiex", + "env": "dev", + "project": "openEO" + } + }, "production": { "app_function": "app.app", "aws_region": "eu-central-1", From c0cc682191a6504e267d86816e12d87ffc17ae2f Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Wed, 11 Oct 2023 11:08:22 +0200 Subject: [PATCH 08/71] use different docker image for CI job for deploying docker image to lambda --- .gitlab-ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 33ac8399..95bcc4ac 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -143,7 +143,7 @@ deploy-testing-rest-lambda-docker: - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING@@#$USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING#g" zappa_settings.json - sed -i "s#@@LOGGING_LEVEL@@#$LOGGING_LEVEL_TESTING#g" zappa_settings.json # generate zappa_settings.py for docker image - - pipenv run zappa save-python-settings-file testing + - pipenv run zappa save-python-settings-file testing-docker # build docker image - docker build -t openeo-sh-zappa:latest . # login for AWS ECR docker @@ -171,12 +171,14 @@ deploy-production-rest-lambda: only: variables: - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]$/ - image: python:3.9.13 + # image: python:3.9.13 before_script: # configure aws access credentials: - mkdir -p ~/.aws - echo -e "[default]\nregion=eu-central-1" > ~/.aws/config - echo -e "[default]\naws_access_key_id=$PRODZAPPA_AWS_ACCESS_KEY_ID\naws_secret_access_key=$PRODZAPPA_AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials + - python --version + - pip --version - pip install pipenv==2021.5.29 script: - chmod +x download-process-definitions.sh From 62941d0371c58dd1cc7acebb169072b650d14eac Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Wed, 11 Oct 2023 11:20:57 +0200 Subject: [PATCH 09/71] use different docker image for CI job for deploying docker image to lambda --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 95bcc4ac..e1cf2e78 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -109,7 +109,7 @@ deploy-testing-rest-lambda-docker: only: variables: - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/ - image: python:3.9.13 + # image: python:3.9.13 before_script: # configure aws access credentials: - mkdir -p ~/.aws @@ -120,6 +120,8 @@ deploy-testing-rest-lambda-docker: - chmod +x download-process-definitions.sh - ./download-process-definitions.sh - cd rest/ + - python --version + - pip --version - pipenv install --dev # create zappa_settings.json on-the-fly: - cp zappa_settings.json.template zappa_settings.json @@ -171,14 +173,12 @@ deploy-production-rest-lambda: only: variables: - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]$/ - # image: python:3.9.13 + image: python:3.9.13 before_script: # configure aws access credentials: - mkdir -p ~/.aws - echo -e "[default]\nregion=eu-central-1" > ~/.aws/config - echo -e "[default]\naws_access_key_id=$PRODZAPPA_AWS_ACCESS_KEY_ID\naws_secret_access_key=$PRODZAPPA_AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials - - python --version - - pip --version - pip install pipenv==2021.5.29 script: - chmod +x download-process-definitions.sh From a7290a68a705ff30134b9065952448743e9d92bc Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Wed, 11 Oct 2023 12:13:21 +0200 Subject: [PATCH 10/71] install pip for deploying docker to lambda --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e1cf2e78..45150579 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -111,6 +111,9 @@ deploy-testing-rest-lambda-docker: - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/ # image: python:3.9.13 before_script: + - python -m ensurepip --upgrade || python get-pip.py + - python --version + - pip --version # configure aws access credentials: - mkdir -p ~/.aws - echo -e "[default]\nregion=eu-central-1" > ~/.aws/config @@ -120,8 +123,6 @@ deploy-testing-rest-lambda-docker: - chmod +x download-process-definitions.sh - ./download-process-definitions.sh - cd rest/ - - python --version - - pip --version - pipenv install --dev # create zappa_settings.json on-the-fly: - cp zappa_settings.json.template zappa_settings.json From 229055201bca9926d170d1bedc5bf2298d6409ed Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Wed, 11 Oct 2023 12:23:02 +0200 Subject: [PATCH 11/71] install python, pip for deploying docker to lambda --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 45150579..009617e3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -111,7 +111,7 @@ deploy-testing-rest-lambda-docker: - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/ # image: python:3.9.13 before_script: - - python -m ensurepip --upgrade || python get-pip.py + - apk add --update python python-dev py-pip - python --version - pip --version # configure aws access credentials: From 2e24ce4ca899544dc9148d1a2342e890e2bec1a6 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Wed, 11 Oct 2023 12:50:33 +0200 Subject: [PATCH 12/71] install python3, pip3 for deploying docker to lambda --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 009617e3..49f34001 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,7 +27,7 @@ integration tests: - docker-compose -f docker-compose.yml -f docker-compose.pytest.yml up --exit-code-from pytest after_script: - docker-compose -f docker-compose.yml -f docker-compose.pytest.yml down -v - + allow_failure: true ## GITHUB: publish_github: @@ -111,7 +111,7 @@ deploy-testing-rest-lambda-docker: - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/ # image: python:3.9.13 before_script: - - apk add --update python python-dev py-pip + - apk add --update python3 python3-dev py3-pip - python --version - pip --version # configure aws access credentials: @@ -120,6 +120,7 @@ deploy-testing-rest-lambda-docker: - echo -e "[default]\naws_access_key_id=$TESTINGZAPPA_AWS_ACCESS_KEY_ID\naws_secret_access_key=$TESTINGZAPPA_AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials - pip install pipenv==2021.5.29 script: + - ls - chmod +x download-process-definitions.sh - ./download-process-definitions.sh - cd rest/ From 6af03780a9272c0f6633986e9a52b99757b18c3c Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Wed, 11 Oct 2023 13:05:20 +0200 Subject: [PATCH 13/71] use correct versions of python and pip in ci job for deploying docker image to lambda --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 49f34001..5654a65f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -112,6 +112,8 @@ deploy-testing-rest-lambda-docker: # image: python:3.9.13 before_script: - apk add --update python3 python3-dev py3-pip + - alias python="python3" + - alias pip="pip3" - python --version - pip --version # configure aws access credentials: From 27351702c8d33f001683db606446cbfa38a1658a Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Wed, 11 Oct 2023 13:20:37 +0200 Subject: [PATCH 14/71] install git, update bash for ci job for deploying docker on lambda --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5654a65f..7b5a329e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -111,7 +111,7 @@ deploy-testing-rest-lambda-docker: - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/ # image: python:3.9.13 before_script: - - apk add --update python3 python3-dev py3-pip + - apk add --update python3 python3-dev py3-pip git bash - alias python="python3" - alias pip="pip3" - python --version @@ -124,7 +124,7 @@ deploy-testing-rest-lambda-docker: script: - ls - chmod +x download-process-definitions.sh - - ./download-process-definitions.sh + - bash download-process-definitions.sh - cd rest/ - pipenv install --dev # create zappa_settings.json on-the-fly: From 4325e8e02feacf8baa0dce05bc48c4bda2a1930a Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Wed, 11 Oct 2023 13:56:23 +0200 Subject: [PATCH 15/71] install py3-env --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7b5a329e..93ce1231 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -111,7 +111,7 @@ deploy-testing-rest-lambda-docker: - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/ # image: python:3.9.13 before_script: - - apk add --update python3 python3-dev py3-pip git bash + - apk add --update python3 python3-dev py3-pip py3-virtualenv git bash - alias python="python3" - alias pip="pip3" - python --version From a5a4a2efc319a663708cd6be39d2376dc9c352f8 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Wed, 11 Oct 2023 14:53:15 +0200 Subject: [PATCH 16/71] split ci jobs for deploying docker to lambda --- .gitlab-ci.yml | 67 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 52 insertions(+), 15 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 93ce1231..a366eb01 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,6 +7,8 @@ services: stages: - test + - build-python + - build-docker - deploy linting: @@ -103,28 +105,22 @@ deploy-testing-rest-lambda: - pipenv run bash -c "DEPLOYMENT_TYPE=testing python dynamodb/dynamodb.py" -deploy-testing-rest-lambda-docker: - stage: deploy +build-python-testing-rest-lambda-docker: + stage: build-python when: manual only: variables: - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/ - # image: python:3.9.13 + image: python:3.9.13 before_script: - - apk add --update python3 python3-dev py3-pip py3-virtualenv git bash - - alias python="python3" - - alias pip="pip3" - - python --version - - pip --version # configure aws access credentials: - mkdir -p ~/.aws - echo -e "[default]\nregion=eu-central-1" > ~/.aws/config - echo -e "[default]\naws_access_key_id=$TESTINGZAPPA_AWS_ACCESS_KEY_ID\naws_secret_access_key=$TESTINGZAPPA_AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials - pip install pipenv==2021.5.29 script: - - ls - chmod +x download-process-definitions.sh - - bash download-process-definitions.sh + - ./download-process-definitions.sh - cd rest/ - pipenv install --dev # create zappa_settings.json on-the-fly: @@ -150,6 +146,30 @@ deploy-testing-rest-lambda-docker: - sed -i "s#@@LOGGING_LEVEL@@#$LOGGING_LEVEL_TESTING#g" zappa_settings.json # generate zappa_settings.py for docker image - pipenv run zappa save-python-settings-file testing-docker + + # needs to be run only once!!! + # - pipenv run aws ecr create-repository --repository-name openeo-zappa-docker --image-scanning-configuration scanOnPush=true --region region + + +build-docker-testing-rest-lambda-docker: + stage: build-docker + when: manual + only: + variables: + - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/ + dependencies: + - build-python-testing-rest-lambda-docker + # image: python:3.9.13 + before_script: + - apk add --no-cache aws-cli + # configure aws access credentials: + - mkdir -p ~/.aws + - echo -e "[default]\nregion=eu-central-1" > ~/.aws/config + - echo -e "[default]\naws_access_key_id=$TESTINGZAPPA_AWS_ACCESS_KEY_ID\naws_secret_access_key=$TESTINGZAPPA_AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials + script: + - chmod +x download-process-definitions.sh + - ./download-process-definitions.sh + - cd rest/ # build docker image - docker build -t openeo-sh-zappa:latest . # login for AWS ECR docker @@ -158,6 +178,28 @@ deploy-testing-rest-lambda-docker: - docker tag openeo-sh-zappa:latest 127163565383.dkr.ecr.eu-central-1.amazonaws.com/openeo-sh-zappa:latest # push docker image to AWS ECR - docker push 127163565383.dkr.ecr.eu-central-1.amazonaws.com/openeo-sh-zappa:latest + + +deploy-testing-rest-lambda-docker: + stage: deploy + when: manual + only: + variables: + - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/ + dependencies: + - build-docker-testing-rest-lambda-docker + image: python:3.9.13 + before_script: + # configure aws access credentials: + - mkdir -p ~/.aws + - echo -e "[default]\nregion=eu-central-1" > ~/.aws/config + - echo -e "[default]\naws_access_key_id=$TESTINGZAPPA_AWS_ACCESS_KEY_ID\naws_secret_access_key=$TESTINGZAPPA_AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials + - pip install pipenv==2021.5.29 + script: + - chmod +x download-process-definitions.sh + - ./download-process-definitions.sh + - cd rest/ + - pipenv install --dev # deploy lambda with new docker image - pipenv run zappa deploy testing-docker -d 127163565383.dkr.ecr.eu-central-1.amazonaws.com/openeo-sh-zappa:latest || pipenv run zappa update testing-docker -d 127163565383.dkr.ecr.eu-central-1.amazonaws.com/openeo-sh-zappa:latest # ensure tables are created: @@ -166,11 +208,6 @@ deploy-testing-rest-lambda-docker: - pipenv run bash -c "DEPLOYMENT_TYPE=testing python dynamodb/dynamodb.py" - # needs to be run only once!!! - # - pipenv run aws ecr create-repository --repository-name openeo-zappa-docker --image-scanning-configuration scanOnPush=true --region region - - - deploy-production-rest-lambda: stage: deploy when: manual From b2d263d561c29847e271902672f0730b0bcfc36b Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Wed, 11 Oct 2023 15:34:39 +0200 Subject: [PATCH 17/71] install aws cli in CI job the Nth way --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a366eb01..eb075480 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -161,7 +161,8 @@ build-docker-testing-rest-lambda-docker: - build-python-testing-rest-lambda-docker # image: python:3.9.13 before_script: - - apk add --no-cache aws-cli + - apk add --update python python-dev py-pip git + - pip install awscli --no-build-isolation # configure aws access credentials: - mkdir -p ~/.aws - echo -e "[default]\nregion=eu-central-1" > ~/.aws/config From 31c35e9a71575c191b9d31240b687e90955e9ad5 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Wed, 11 Oct 2023 15:43:14 +0200 Subject: [PATCH 18/71] various improvements for ci job deploying docker to lambda --- .gitlab-ci.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eb075480..0f9bbb22 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -153,12 +153,13 @@ build-python-testing-rest-lambda-docker: build-docker-testing-rest-lambda-docker: stage: build-docker - when: manual - only: - variables: - - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/ + # when: manual + # only: + # variables: + # - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/ dependencies: - - build-python-testing-rest-lambda-docker + - build-python-testing-rest-lambda-docker + needs: ["build-python-testing-rest-lambda-docker"] # image: python:3.9.13 before_script: - apk add --update python python-dev py-pip git @@ -168,8 +169,6 @@ build-docker-testing-rest-lambda-docker: - echo -e "[default]\nregion=eu-central-1" > ~/.aws/config - echo -e "[default]\naws_access_key_id=$TESTINGZAPPA_AWS_ACCESS_KEY_ID\naws_secret_access_key=$TESTINGZAPPA_AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials script: - - chmod +x download-process-definitions.sh - - ./download-process-definitions.sh - cd rest/ # build docker image - docker build -t openeo-sh-zappa:latest . @@ -183,12 +182,13 @@ build-docker-testing-rest-lambda-docker: deploy-testing-rest-lambda-docker: stage: deploy - when: manual - only: - variables: - - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/ + # when: manual + # only: + # variables: + # - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/ dependencies: - - build-docker-testing-rest-lambda-docker + - build-docker-testing-rest-lambda-docker + needs: ["build-docker-testing-rest-lambda-docker"] image: python:3.9.13 before_script: # configure aws access credentials: From a2ccc4fc9f828ae8757d17d4e19c8d011f5caa5c Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Wed, 11 Oct 2023 15:45:04 +0200 Subject: [PATCH 19/71] various improvements for ci job deploying docker to lambda 3 --- .gitlab-ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0f9bbb22..38aedd14 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -154,9 +154,9 @@ build-python-testing-rest-lambda-docker: build-docker-testing-rest-lambda-docker: stage: build-docker # when: manual - # only: - # variables: - # - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/ + only: + variables: + - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/ dependencies: - build-python-testing-rest-lambda-docker needs: ["build-python-testing-rest-lambda-docker"] @@ -183,9 +183,9 @@ build-docker-testing-rest-lambda-docker: deploy-testing-rest-lambda-docker: stage: deploy # when: manual - # only: - # variables: - # - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/ + only: + variables: + - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/ dependencies: - build-docker-testing-rest-lambda-docker needs: ["build-docker-testing-rest-lambda-docker"] From dbca95d9c137ed263dc29c793046e10ba111585f Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Wed, 11 Oct 2023 15:58:52 +0200 Subject: [PATCH 20/71] various improvements for ci job deploying docker to lambda 4 --- .gitlab-ci.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 38aedd14..3650a18b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -201,6 +201,27 @@ deploy-testing-rest-lambda-docker: - ./download-process-definitions.sh - cd rest/ - pipenv install --dev + # create zappa_settings.json on-the-fly: + - cp zappa_settings.json.template zappa_settings.json + - sed -i "s/@@AWS_ACCESS_KEY_ID@@/$TESTINGDATA_AWS_ACCESS_KEY_ID/g" zappa_settings.json + - sed -i "s#@@AWS_SECRET_ACCESS_KEY@@#$TESTINGDATA_AWS_SECRET_ACCESS_KEY#g" zappa_settings.json + - sed -i "s/@@TESTING_SH_CLIENT_ID@@/$TESTING_SH_CLIENT_ID/g" zappa_settings.json + - sed -i "s/@@TESTING_SH_CLIENT_SECRET@@/$TESTING_SH_CLIENT_SECRET/g" zappa_settings.json + - sed -i "s/@@BACKEND_VERSION@@/$CI_COMMIT_TAG/g" zappa_settings.json + - sed -i "s/@@RESULTS_S3_BUCKET_NAME_MAIN@@/$RESULTS_S3_BUCKET_NAME_MAIN/g" zappa_settings.json + - sed -i "s/@@RESULTS_S3_BUCKET_NAME_CREODIAS@@/$RESULTS_S3_BUCKET_NAME_CREODIAS/g" zappa_settings.json + - sed -i "s/@@RESULTS_S3_BUCKET_NAME_USWEST@@/$RESULTS_S3_BUCKET_NAME_USWEST/g" zappa_settings.json + - sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN/g" zappa_settings.json + - sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS/g" zappa_settings.json + - sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST/g" zappa_settings.json + - sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN@@#$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN#g" zappa_settings.json + - sed -i "s/@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS@@/$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS/g" zappa_settings.json + - sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST@@#$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST#g" zappa_settings.json + - sed -i "s#@@USAGE_REPORTING_BASE_URL_TESTING@@#$USAGE_REPORTING_BASE_URL_TESTING#g" zappa_settings.json + - sed -i "s#@@USAGE_REPORTING_AUTH_URL_TESTING@@#$USAGE_REPORTING_AUTH_URL_TESTING#g" zappa_settings.json + - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_ID_TESTING@@#$USAGE_REPORTING_AUTH_CLIENT_ID_TESTING#g" zappa_settings.json + - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING@@#$USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING#g" zappa_settings.json + - sed -i "s#@@LOGGING_LEVEL@@#$LOGGING_LEVEL_TESTING#g" zappa_settings.json # deploy lambda with new docker image - pipenv run zappa deploy testing-docker -d 127163565383.dkr.ecr.eu-central-1.amazonaws.com/openeo-sh-zappa:latest || pipenv run zappa update testing-docker -d 127163565383.dkr.ecr.eu-central-1.amazonaws.com/openeo-sh-zappa:latest # ensure tables are created: From 7827b67aeb7ff9889e513377e095a9eaa1ab8a09 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Thu, 12 Oct 2023 14:44:52 +0200 Subject: [PATCH 21/71] add zappa settings for deploying docker to zappa for production deploy --- rest/zappa_settings.json.template | 43 ++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/rest/zappa_settings.json.template b/rest/zappa_settings.json.template index 9cd629c3..5ba3bee8 100644 --- a/rest/zappa_settings.json.template +++ b/rest/zappa_settings.json.template @@ -66,7 +66,7 @@ "USAGE_REPORTING_AUTH_CLIENT_SECRET": "@@USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING@@", "LOGGING_LEVEL": "@@LOGGING_LEVEL@@" }, - "memory_size": 512, + "memory_size": 4096, "timeout_seconds": 30, "profile_name": "default", "project_name": "openeo", @@ -122,4 +122,45 @@ "project": "openEO" } } + "production-docker": { + "app_function": "app.app", + "aws_region": "eu-central-1", + "lambda_description": "Production deploy of Sentinel Hub openEO backend", + "environment_variables": { + "DEPLOYMENT_TYPE": "production", + "AWS_ACCESS_KEY_ID": "@@AWS_ACCESS_KEY_ID@@", + "AWS_SECRET_ACCESS_KEY": "@@AWS_SECRET_ACCESS_KEY@@", + "SH_CLIENT_ID": "@@PRODUCTION_SH_CLIENT_ID@@", + "SH_CLIENT_SECRET": "@@PRODUCTION_SH_CLIENT_SECRET@@", + "BACKEND_VERSION": "@@BACKEND_VERSION@@", + "RESULTS_S3_BUCKET_NAME_MAIN": "@@RESULTS_S3_BUCKET_NAME_MAIN@@", + "RESULTS_S3_BUCKET_NAME_CREODIAS": "@@RESULTS_S3_BUCKET_NAME_CREODIAS@@", + "RESULTS_S3_BUCKET_NAME_USWEST": "@@RESULTS_S3_BUCKET_NAME_USWEST@@", + "RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN": "@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN@@", + "RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS": "@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS@@", + "RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST": "@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST@@", + "RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN": "@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN@@", + "RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS": "@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS@@", + "RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST": "@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST@@", + "USAGE_REPORTING_BASE_URL": "@@USAGE_REPORTING_BASE_URL_PRODUCTION@@", + "USAGE_REPORTING_AUTH_URL": "@@USAGE_REPORTING_AUTH_URL_PRODUCTION@@", + "USAGE_REPORTING_AUTH_CLIENT_ID": "@@USAGE_REPORTING_AUTH_CLIENT_ID_PRODUCTION@@", + "USAGE_REPORTING_AUTH_CLIENT_SECRET": "@@USAGE_REPORTING_AUTH_CLIENT_SECRET_PRODUCTION@@", + "LOGGING_LEVEL": "@@LOGGING_LEVEL@@" + }, + "memory_size": 4096, + "timeout_seconds": 30, + "profile_name": "default", + "project_name": "openeo", + "runtime": "python3.9", + "manage_roles": false, + "role_arn": "arn:aws:iam::707953651112:role/openEO-prod-role", + "s3_bucket": "openeoproddeploy", + "tags": { + "cost:ou": "frontend", + "cost:contact": "dthiex", + "env": "prod", + "project": "openEO" + } + } } \ No newline at end of file From efc2d056786f5f1d374b6d6ca38fae2f9fbd4f80 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Fri, 13 Oct 2023 14:27:41 +0200 Subject: [PATCH 22/71] tag docker images with git tag version before pushing to AWS ECR --- .gitlab-ci.yml | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3650a18b..9ed43263 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -146,6 +146,7 @@ build-python-testing-rest-lambda-docker: - sed -i "s#@@LOGGING_LEVEL@@#$LOGGING_LEVEL_TESTING#g" zappa_settings.json # generate zappa_settings.py for docker image - pipenv run zappa save-python-settings-file testing-docker + - ls # needs to be run only once!!! # - pipenv run aws ecr create-repository --repository-name openeo-zappa-docker --image-scanning-configuration scanOnPush=true --region region @@ -170,14 +171,19 @@ build-docker-testing-rest-lambda-docker: - echo -e "[default]\naws_access_key_id=$TESTINGZAPPA_AWS_ACCESS_KEY_ID\naws_secret_access_key=$TESTINGZAPPA_AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials script: - cd rest/ + - ls # build docker image - - docker build -t openeo-sh-zappa:latest . + # - docker build -t openeo-sh-zappa:latest . + - docker build -t "$CI_REGISTRY/$CI_REGISTRY_IMAGE:$CI_COMMIT_TAG" -t "$CI_REGISTRY/$CI_REGISTRY_IMAGE:latest" --build-arg VERSION=$CI_COMMIT_TAG --build-arg VCS_REF=$CI_COMMIT_SHA --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') . # login for AWS ECR docker - - aws ecr get-login-password --region eu-central-1 | docker login --username AWS --password-stdin 127163565383.dkr.ecr.eu-central-1.amazonaws.com + - aws ecr get-login-password --region eu-central-1 | docker login --username AWS --password-stdin "$CI_REGISTRY" # re-tag the built image - - docker tag openeo-sh-zappa:latest 127163565383.dkr.ecr.eu-central-1.amazonaws.com/openeo-sh-zappa:latest + # - docker tag openeo-sh-zappa:latest 127163565383.dkr.ecr.eu-central-1.amazonaws.com/openeo-sh-zappa:latest # push docker image to AWS ECR - - docker push 127163565383.dkr.ecr.eu-central-1.amazonaws.com/openeo-sh-zappa:latest + # - docker push 127163565383.dkr.ecr.eu-central-1.amazonaws.com/openeo-sh-zappa:latest + - docker push "$CI_REGISTRY/$CI_REGISTRY_IMAGE:$CI_COMMIT_TAG" + - docker push "$CI_REGISTRY/$CI_REGISTRY_IMAGE:latest" + deploy-testing-rest-lambda-docker: @@ -190,6 +196,8 @@ deploy-testing-rest-lambda-docker: - build-docker-testing-rest-lambda-docker needs: ["build-docker-testing-rest-lambda-docker"] image: python:3.9.13 + variables: + DEPLOYMENT_NAME: "testing-docker" before_script: # configure aws access credentials: - mkdir -p ~/.aws @@ -200,6 +208,7 @@ deploy-testing-rest-lambda-docker: - chmod +x download-process-definitions.sh - ./download-process-definitions.sh - cd rest/ + - ls - pipenv install --dev # create zappa_settings.json on-the-fly: - cp zappa_settings.json.template zappa_settings.json @@ -223,11 +232,11 @@ deploy-testing-rest-lambda-docker: - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING@@#$USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING#g" zappa_settings.json - sed -i "s#@@LOGGING_LEVEL@@#$LOGGING_LEVEL_TESTING#g" zappa_settings.json # deploy lambda with new docker image - - pipenv run zappa deploy testing-docker -d 127163565383.dkr.ecr.eu-central-1.amazonaws.com/openeo-sh-zappa:latest || pipenv run zappa update testing-docker -d 127163565383.dkr.ecr.eu-central-1.amazonaws.com/openeo-sh-zappa:latest + - pipenv run zappa deploy "$DEPLOYMENT_NAME"-d "$CI_REGISTRY/$CI_REGISTRY_IMAGE:$CI_COMMIT_TAG" || pipenv run zappa update testing-docker -d "$CI_REGISTRY/$CI_REGISTRY_IMAGE:$CI_COMMIT_TAG" # ensure tables are created: - - export AWS_ACCESS_KEY_ID="$TESTING_AWS_ACCESS_KEY_ID" - - export AWS_SECRET_ACCESS_KEY="$TESTING_AWS_SECRET_ACCESS_KEY" - - pipenv run bash -c "DEPLOYMENT_TYPE=testing python dynamodb/dynamodb.py" + # - export AWS_ACCESS_KEY_ID="$TESTING_AWS_ACCESS_KEY_ID" + # - export AWS_SECRET_ACCESS_KEY="$TESTING_AWS_SECRET_ACCESS_KEY" + # - pipenv run bash -c "DEPLOYMENT_TYPE=testing python dynamodb/dynamodb.py" deploy-production-rest-lambda: From 6dd282d8b605b53850c4dcb9262464750cf41494 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Fri, 13 Oct 2023 15:51:51 +0200 Subject: [PATCH 23/71] keep artifacts for CI jobs deploying docker to lambda --- .gitlab-ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9ed43263..6d112954 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -147,6 +147,9 @@ build-python-testing-rest-lambda-docker: # generate zappa_settings.py for docker image - pipenv run zappa save-python-settings-file testing-docker - ls + artifacts: + untracked: true + expire_in: 1 day # needs to be run only once!!! # - pipenv run aws ecr create-repository --repository-name openeo-zappa-docker --image-scanning-configuration scanOnPush=true --region region @@ -183,7 +186,9 @@ build-docker-testing-rest-lambda-docker: # - docker push 127163565383.dkr.ecr.eu-central-1.amazonaws.com/openeo-sh-zappa:latest - docker push "$CI_REGISTRY/$CI_REGISTRY_IMAGE:$CI_COMMIT_TAG" - docker push "$CI_REGISTRY/$CI_REGISTRY_IMAGE:latest" - + artifacts: + untracked: true + expire_in: 1 day deploy-testing-rest-lambda-docker: From 11853664069ff98a6c6148b228d9594ed80c0245 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Fri, 13 Oct 2023 16:16:09 +0200 Subject: [PATCH 24/71] deploy latest tagged docker image instead of fixed version --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6d112954..042140b7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -237,7 +237,7 @@ deploy-testing-rest-lambda-docker: - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING@@#$USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING#g" zappa_settings.json - sed -i "s#@@LOGGING_LEVEL@@#$LOGGING_LEVEL_TESTING#g" zappa_settings.json # deploy lambda with new docker image - - pipenv run zappa deploy "$DEPLOYMENT_NAME"-d "$CI_REGISTRY/$CI_REGISTRY_IMAGE:$CI_COMMIT_TAG" || pipenv run zappa update testing-docker -d "$CI_REGISTRY/$CI_REGISTRY_IMAGE:$CI_COMMIT_TAG" + - pipenv run zappa deploy "$DEPLOYMENT_NAME" -d "$CI_REGISTRY/$CI_REGISTRY_IMAGE:latest" || pipenv run zappa update "$DEPLOYMENT_NAME" -d "$CI_REGISTRY/$CI_REGISTRY_IMAGE:latest" # ensure tables are created: # - export AWS_ACCESS_KEY_ID="$TESTING_AWS_ACCESS_KEY_ID" # - export AWS_SECRET_ACCESS_KEY="$TESTING_AWS_SECRET_ACCESS_KEY" From 7cb789f16f9c44512639942aeffefba4d02c0d4e Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Fri, 13 Oct 2023 16:41:15 +0200 Subject: [PATCH 25/71] deploy docker to lambda CI job cleanup --- .gitlab-ci.yml | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 042140b7..8d7b78dd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -144,29 +144,24 @@ build-python-testing-rest-lambda-docker: - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_ID_TESTING@@#$USAGE_REPORTING_AUTH_CLIENT_ID_TESTING#g" zappa_settings.json - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING@@#$USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING#g" zappa_settings.json - sed -i "s#@@LOGGING_LEVEL@@#$LOGGING_LEVEL_TESTING#g" zappa_settings.json - # generate zappa_settings.py for docker image + # generate zappa_settings.py for docker image from zappa_settings.json - pipenv run zappa save-python-settings-file testing-docker - - ls + # save downloaded process definitions and zappa configuration + # for CI job that creates docker image artifacts: untracked: true expire_in: 1 day - # needs to be run only once!!! - # - pipenv run aws ecr create-repository --repository-name openeo-zappa-docker --image-scanning-configuration scanOnPush=true --region region - - build-docker-testing-rest-lambda-docker: stage: build-docker - # when: manual only: variables: - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/ dependencies: - build-python-testing-rest-lambda-docker needs: ["build-python-testing-rest-lambda-docker"] - # image: python:3.9.13 before_script: - - apk add --update python python-dev py-pip git + - apk add --update python python-dev py-pip - pip install awscli --no-build-isolation # configure aws access credentials: - mkdir -p ~/.aws @@ -174,26 +169,18 @@ build-docker-testing-rest-lambda-docker: - echo -e "[default]\naws_access_key_id=$TESTINGZAPPA_AWS_ACCESS_KEY_ID\naws_secret_access_key=$TESTINGZAPPA_AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials script: - cd rest/ - - ls - # build docker image - # - docker build -t openeo-sh-zappa:latest . + # build docker image with correct tags - docker build -t "$CI_REGISTRY/$CI_REGISTRY_IMAGE:$CI_COMMIT_TAG" -t "$CI_REGISTRY/$CI_REGISTRY_IMAGE:latest" --build-arg VERSION=$CI_COMMIT_TAG --build-arg VCS_REF=$CI_COMMIT_SHA --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') . # login for AWS ECR docker - aws ecr get-login-password --region eu-central-1 | docker login --username AWS --password-stdin "$CI_REGISTRY" - # re-tag the built image - # - docker tag openeo-sh-zappa:latest 127163565383.dkr.ecr.eu-central-1.amazonaws.com/openeo-sh-zappa:latest - # push docker image to AWS ECR - # - docker push 127163565383.dkr.ecr.eu-central-1.amazonaws.com/openeo-sh-zappa:latest + # create a repository - needs to be executed only once! + # - aws ecr create-repository --repository-name openeo-sh-zappa --image-scanning-configuration scanOnPush=true --region eu-central-1 + # push docker images (versioned with CI tag and "latest") to AWS ECR with docker - docker push "$CI_REGISTRY/$CI_REGISTRY_IMAGE:$CI_COMMIT_TAG" - docker push "$CI_REGISTRY/$CI_REGISTRY_IMAGE:latest" - artifacts: - untracked: true - expire_in: 1 day - deploy-testing-rest-lambda-docker: stage: deploy - # when: manual only: variables: - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/ From 188a2dc7e6d6a7297c10ce0b08a2667ab5ec00ad Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Mon, 16 Oct 2023 14:41:14 +0200 Subject: [PATCH 26/71] better stage names in gitlab-ci.yml --- .gitlab-ci.yml | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8d7b78dd..18e033df 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,9 +7,10 @@ services: stages: - test - - build-python - - build-docker - deploy + - deploy-generate-zappa-settings + - deploy-create-docker-image + - deploy-update-lambda linting: stage: test @@ -59,6 +60,7 @@ publish_github: - if git remote | grep github > /dev/null; then git remote rm github; fi +# DEPLOY TESTING: uploading .zip file on lambda deploy-testing-rest-lambda: stage: deploy when: manual @@ -104,9 +106,14 @@ deploy-testing-rest-lambda: - export AWS_SECRET_ACCESS_KEY="$TESTING_AWS_SECRET_ACCESS_KEY" - pipenv run bash -c "DEPLOYMENT_TYPE=testing python dynamodb/dynamodb.py" +# DEPLOY TESTING: create docker image, upload it to AWS ECR and use it in AWS Lambda +# - step 1: generate zappa settings for the app inside the docker image +# - step 2: create docker image and push it to AWS ECR +# - step 3: update lambda function to use the new docker image -build-python-testing-rest-lambda-docker: - stage: build-python +# generate zappa settings for the app inside the docker image +deploy-testing-generate-zappa-settings: + stage: deploy-generate-zappa-settings when: manual only: variables: @@ -152,14 +159,15 @@ build-python-testing-rest-lambda-docker: untracked: true expire_in: 1 day -build-docker-testing-rest-lambda-docker: - stage: build-docker +# create docker image and push it to AWS ECR +deploy-testing-create-docker-image: + stage: deploy-create-docker-image only: variables: - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/ dependencies: - - build-python-testing-rest-lambda-docker - needs: ["build-python-testing-rest-lambda-docker"] + - deploy-testing-generate-zappa-settings + needs: ["deploy-testing-generate-zappa-settings"] before_script: - apk add --update python python-dev py-pip - pip install awscli --no-build-isolation @@ -179,14 +187,15 @@ build-docker-testing-rest-lambda-docker: - docker push "$CI_REGISTRY/$CI_REGISTRY_IMAGE:$CI_COMMIT_TAG" - docker push "$CI_REGISTRY/$CI_REGISTRY_IMAGE:latest" +# update lambda function to use the new docker image deploy-testing-rest-lambda-docker: - stage: deploy + stage: deploy-update-lambda only: variables: - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/ dependencies: - - build-docker-testing-rest-lambda-docker - needs: ["build-docker-testing-rest-lambda-docker"] + - deploy-testing-create-docker-image + needs: ["deploy-testing-create-docker-image"] image: python:3.9.13 variables: DEPLOYMENT_NAME: "testing-docker" @@ -230,7 +239,7 @@ deploy-testing-rest-lambda-docker: # - export AWS_SECRET_ACCESS_KEY="$TESTING_AWS_SECRET_ACCESS_KEY" # - pipenv run bash -c "DEPLOYMENT_TYPE=testing python dynamodb/dynamodb.py" - +# DEPLOY PRODUCTION: uploading .zip file on lambda deploy-production-rest-lambda: stage: deploy when: manual @@ -275,3 +284,6 @@ deploy-production-rest-lambda: - export AWS_ACCESS_KEY_ID="$PRODDATA_AWS_ACCESS_KEY_ID" - export AWS_SECRET_ACCESS_KEY="$PRODDATA_AWS_SECRET_ACCESS_KEY" - pipenv run bash -c "DEPLOYMENT_TYPE=production python dynamodb/dynamodb.py" + + +### DEPLOY PRODUCTION: DOCKER ON LAMBDA \ No newline at end of file From dcbd91dbd1e7389488f28744fe2524df501db0b5 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Mon, 16 Oct 2023 14:58:44 +0200 Subject: [PATCH 27/71] add CI job for deploying docker img to prod lambda --- .gitlab-ci.yml | 146 ++++++++++++++++++++++++++++-- rest/zappa_settings.json.template | 2 +- 2 files changed, 141 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 18e033df..89b64a73 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -60,6 +60,7 @@ publish_github: - if git remote | grep github > /dev/null; then git remote rm github; fi +# BEFORE 2023-10-16 # DEPLOY TESTING: uploading .zip file on lambda deploy-testing-rest-lambda: stage: deploy @@ -106,6 +107,7 @@ deploy-testing-rest-lambda: - export AWS_SECRET_ACCESS_KEY="$TESTING_AWS_SECRET_ACCESS_KEY" - pipenv run bash -c "DEPLOYMENT_TYPE=testing python dynamodb/dynamodb.py" +# AFTER 2023-10-16 # DEPLOY TESTING: create docker image, upload it to AWS ECR and use it in AWS Lambda # - step 1: generate zappa settings for the app inside the docker image # - step 2: create docker image and push it to AWS ECR @@ -178,14 +180,14 @@ deploy-testing-create-docker-image: script: - cd rest/ # build docker image with correct tags - - docker build -t "$CI_REGISTRY/$CI_REGISTRY_IMAGE:$CI_COMMIT_TAG" -t "$CI_REGISTRY/$CI_REGISTRY_IMAGE:latest" --build-arg VERSION=$CI_COMMIT_TAG --build-arg VCS_REF=$CI_COMMIT_SHA --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') . + - docker build -t "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:$CI_COMMIT_TAG" -t "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest" --build-arg VERSION=$CI_COMMIT_TAG --build-arg VCS_REF=$CI_COMMIT_SHA --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') . # login for AWS ECR docker - - aws ecr get-login-password --region eu-central-1 | docker login --username AWS --password-stdin "$CI_REGISTRY" + - aws ecr get-login-password --region eu-central-1 | docker login --username AWS --password-stdin "$TESTING_ECR_REGISTRY" # create a repository - needs to be executed only once! # - aws ecr create-repository --repository-name openeo-sh-zappa --image-scanning-configuration scanOnPush=true --region eu-central-1 # push docker images (versioned with CI tag and "latest") to AWS ECR with docker - - docker push "$CI_REGISTRY/$CI_REGISTRY_IMAGE:$CI_COMMIT_TAG" - - docker push "$CI_REGISTRY/$CI_REGISTRY_IMAGE:latest" + - docker push "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:$CI_COMMIT_TAG" + - docker push "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest" # update lambda function to use the new docker image deploy-testing-rest-lambda-docker: @@ -233,12 +235,13 @@ deploy-testing-rest-lambda-docker: - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING@@#$USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING#g" zappa_settings.json - sed -i "s#@@LOGGING_LEVEL@@#$LOGGING_LEVEL_TESTING#g" zappa_settings.json # deploy lambda with new docker image - - pipenv run zappa deploy "$DEPLOYMENT_NAME" -d "$CI_REGISTRY/$CI_REGISTRY_IMAGE:latest" || pipenv run zappa update "$DEPLOYMENT_NAME" -d "$CI_REGISTRY/$CI_REGISTRY_IMAGE:latest" + - pipenv run zappa deploy "$DEPLOYMENT_NAME" -d "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest" || pipenv run zappa update "$DEPLOYMENT_NAME" -d "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest" # ensure tables are created: # - export AWS_ACCESS_KEY_ID="$TESTING_AWS_ACCESS_KEY_ID" # - export AWS_SECRET_ACCESS_KEY="$TESTING_AWS_SECRET_ACCESS_KEY" # - pipenv run bash -c "DEPLOYMENT_TYPE=testing python dynamodb/dynamodb.py" +# BEFORE 2023-10-16 # DEPLOY PRODUCTION: uploading .zip file on lambda deploy-production-rest-lambda: stage: deploy @@ -285,5 +288,136 @@ deploy-production-rest-lambda: - export AWS_SECRET_ACCESS_KEY="$PRODDATA_AWS_SECRET_ACCESS_KEY" - pipenv run bash -c "DEPLOYMENT_TYPE=production python dynamodb/dynamodb.py" +# AFTER 2023-10-16 +# DEPLOY PRODUCTION: DOCKER ON LAMBDA +# - step 1: generate zappa settings for the app inside the docker image +# - step 2: create docker image and push it to AWS ECR +# - step 3: update lambda function to use the new docker image + +# generate zappa settings for the app inside the docker image +deploy-production-generate-zappa-settings: + stage: deploy-generate-zappa-settings + when: manual + only: + variables: + - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/ + image: python:3.9.13 + before_script: + # configure aws access credentials: + - mkdir -p ~/.aws + - echo -e "[default]\nregion=eu-central-1" > ~/.aws/config + - echo -e "[default]\naws_access_key_id=$PRODZAPPA_AWS_ACCESS_KEY_ID\naws_secret_access_key=$PRODZAPPA_AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials + - pip install pipenv==2021.5.29 + script: + - chmod +x download-process-definitions.sh + - ./download-process-definitions.sh + - cd rest/ + - pipenv install --dev + # create zappa_settings.json on-the-fly: + - cp zappa_settings.json.template zappa_settings.json + - sed -i "s/@@AWS_ACCESS_KEY_ID@@/$PRODDATA_AWS_ACCESS_KEY_ID/g" zappa_settings.json + - sed -i "s#@@AWS_SECRET_ACCESS_KEY@@#$PRODDATA_AWS_SECRET_ACCESS_KEY#g" zappa_settings.json + - sed -i "s/@@PRODUCTION_SH_CLIENT_ID@@/$PRODUCTION_SH_CLIENT_ID/g" zappa_settings.json + - sed -i "s/@@PRODUCTION_SH_CLIENT_SECRET@@/$PRODUCTION_SH_CLIENT_SECRET/g" zappa_settings.json + - sed -i "s/@@BACKEND_VERSION@@/$CI_COMMIT_TAG/g" zappa_settings.json + - sed -i "s/@@RESULTS_S3_BUCKET_NAME_MAIN@@/$RESULTS_S3_BUCKET_NAME_MAIN_PRODUCTION/g" zappa_settings.json + - sed -i "s/@@RESULTS_S3_BUCKET_NAME_CREODIAS@@/$RESULTS_S3_BUCKET_NAME_CREODIAS_PRODUCTION/g" zappa_settings.json + - sed -i "s/@@RESULTS_S3_BUCKET_NAME_USWEST@@/$RESULTS_S3_BUCKET_NAME_USWEST_PRODUCTION/g" zappa_settings.json + - sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN_PRODUCTION/g" zappa_settings.json + - sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS_PRODUCTION/g" zappa_settings.json + - sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST_PRODUCTION/g" zappa_settings.json + - sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN@@#$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN_PRODUCTION#g" zappa_settings.json + - sed -i "s/@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS@@/$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS_PRODUCTION/g" zappa_settings.json + - sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST@@#$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST_PRODUCTION#g" zappa_settings.json + - sed -i "s#@@USAGE_REPORTING_BASE_URL_PRODUCTION@@#$USAGE_REPORTING_BASE_URL_PRODUCTION#g" zappa_settings.json + - sed -i "s#@@USAGE_REPORTING_AUTH_URL_PRODUCTION@@#$USAGE_REPORTING_AUTH_URL_PRODUCTION#g" zappa_settings.json + - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_ID_PRODUCTION@@#$USAGE_REPORTING_AUTH_CLIENT_ID_PRODUCTION#g" zappa_settings.json + - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_PRODUCTION@@#$USAGE_REPORTING_AUTH_CLIENT_SECRET_PRODUCTION#g" zappa_settings.json + - sed -i "s#@@LOGGING_LEVEL@@#$LOGGING_LEVEL_PRODUCTION#g" zappa_settings.json + # generate zappa_settings.py for docker image from zappa_settings.json + - pipenv run zappa save-python-settings-file testing-docker + # save downloaded process definitions and zappa configuration + # for CI job that creates docker image + artifacts: + untracked: true + expire_in: 1 day + +# create docker image and push it to AWS ECR +deploy-production-create-docker-image: + stage: deploy-create-docker-image + only: + variables: + - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/ + dependencies: + - deploy-production-generate-zappa-settings + needs: ["deploy-production-generate-zappa-settings"] + before_script: + - apk add --update python python-dev py-pip + - pip install awscli --no-build-isolation + # configure aws access credentials: + - mkdir -p ~/.aws + - echo -e "[default]\nregion=eu-central-1" > ~/.aws/config + - echo -e "[default]\naws_access_key_id=$TESTINGZAPPA_AWS_ACCESS_KEY_ID\naws_secret_access_key=$TESTINGZAPPA_AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials + script: + - cd rest/ + # build docker image with correct tags + - docker build -t "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:$CI_COMMIT_TAG" -t "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:latest" --build-arg VERSION=$CI_COMMIT_TAG --build-arg VCS_REF=$CI_COMMIT_SHA --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') . + # login for AWS ECR docker + - aws ecr get-login-password --region eu-central-1 | docker login --username AWS --password-stdin "$PROD_ECR_REGISTRY" + # create a repository - needs to be executed only once! + # - aws ecr create-repository --repository-name openeo-sh-zappa --image-scanning-configuration scanOnPush=true --region eu-central-1 + # push docker images (versioned with CI tag and "latest") to AWS ECR with docker + - docker push "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:$CI_COMMIT_TAG" + - docker push "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:latest" -### DEPLOY PRODUCTION: DOCKER ON LAMBDA \ No newline at end of file +# update lambda function to use the new docker image +deploy-production-rest-lambda-docker: + stage: deploy-update-lambda + only: + variables: + - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/ + dependencies: + - deploy-production-create-docker-image + needs: ["deploy-production-create-docker-image"] + image: python:3.9.13 + variables: + DEPLOYMENT_NAME: "production-docker" + before_script: + # configure aws access credentials: + - mkdir -p ~/.aws + - echo -e "[default]\nregion=eu-central-1" > ~/.aws/config + - echo -e "[default]\naws_access_key_id=$TESTINGZAPPA_AWS_ACCESS_KEY_ID\naws_secret_access_key=$TESTINGZAPPA_AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials + - pip install pipenv==2021.5.29 + script: + - chmod +x download-process-definitions.sh + - ./download-process-definitions.sh + - cd rest/ + - ls + - pipenv install --dev + # create zappa_settings.json on-the-fly: + - cp zappa_settings.json.template zappa_settings.json + - sed -i "s/@@AWS_ACCESS_KEY_ID@@/$PRODDATA_AWS_ACCESS_KEY_ID/g" zappa_settings.json + - sed -i "s#@@AWS_SECRET_ACCESS_KEY@@#$PRODDATA_AWS_SECRET_ACCESS_KEY#g" zappa_settings.json + - sed -i "s/@@PRODUCTION_SH_CLIENT_ID@@/$PRODUCTION_SH_CLIENT_ID/g" zappa_settings.json + - sed -i "s/@@PRODUCTION_SH_CLIENT_SECRET@@/$PRODUCTION_SH_CLIENT_SECRET/g" zappa_settings.json + - sed -i "s/@@BACKEND_VERSION@@/$CI_COMMIT_TAG/g" zappa_settings.json + - sed -i "s/@@RESULTS_S3_BUCKET_NAME_MAIN@@/$RESULTS_S3_BUCKET_NAME_MAIN_PRODUCTION/g" zappa_settings.json + - sed -i "s/@@RESULTS_S3_BUCKET_NAME_CREODIAS@@/$RESULTS_S3_BUCKET_NAME_CREODIAS_PRODUCTION/g" zappa_settings.json + - sed -i "s/@@RESULTS_S3_BUCKET_NAME_USWEST@@/$RESULTS_S3_BUCKET_NAME_USWEST_PRODUCTION/g" zappa_settings.json + - sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN_PRODUCTION/g" zappa_settings.json + - sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS_PRODUCTION/g" zappa_settings.json + - sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST_PRODUCTION/g" zappa_settings.json + - sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN@@#$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN_PRODUCTION#g" zappa_settings.json + - sed -i "s/@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS@@/$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS_PRODUCTION/g" zappa_settings.json + - sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST@@#$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST_PRODUCTION#g" zappa_settings.json + - sed -i "s#@@USAGE_REPORTING_BASE_URL_PRODUCTION@@#$USAGE_REPORTING_BASE_URL_PRODUCTION#g" zappa_settings.json + - sed -i "s#@@USAGE_REPORTING_AUTH_URL_PRODUCTION@@#$USAGE_REPORTING_AUTH_URL_PRODUCTION#g" zappa_settings.json + - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_ID_PRODUCTION@@#$USAGE_REPORTING_AUTH_CLIENT_ID_PRODUCTION#g" zappa_settings.json + - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_PRODUCTION@@#$USAGE_REPORTING_AUTH_CLIENT_SECRET_PRODUCTION#g" zappa_settings.json + - sed -i "s#@@LOGGING_LEVEL@@#$LOGGING_LEVEL_PRODUCTION#g" zappa_settings.json + # deploy lambda with new docker image + - pipenv run zappa deploy "$DEPLOYMENT_NAME" -d "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:latest" || pipenv run zappa update "$DEPLOYMENT_NAME" -d "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:latest" + # ensure tables are created: + # - export AWS_ACCESS_KEY_ID="$PRODDATA_AWS_ACCESS_KEY_ID" + # - export AWS_SECRET_ACCESS_KEY="$PRODDATA_AWS_SECRET_ACCESS_KEY" + # - pipenv run bash -c "DEPLOYMENT_TYPE=production python dynamodb/dynamodb.py" diff --git a/rest/zappa_settings.json.template b/rest/zappa_settings.json.template index 5ba3bee8..c2684134 100644 --- a/rest/zappa_settings.json.template +++ b/rest/zappa_settings.json.template @@ -121,7 +121,7 @@ "env": "prod", "project": "openEO" } - } + }, "production-docker": { "app_function": "app.app", "aws_region": "eu-central-1", From dec6f4e295ed2099305f5a3242777ceeb11ce0db Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Mon, 16 Oct 2023 15:01:01 +0200 Subject: [PATCH 28/71] fix comment in .gitlab-ci.yml --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 89b64a73..5f789a31 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -289,7 +289,7 @@ deploy-production-rest-lambda: - pipenv run bash -c "DEPLOYMENT_TYPE=production python dynamodb/dynamodb.py" # AFTER 2023-10-16 -# DEPLOY PRODUCTION: DOCKER ON LAMBDA +# DEPLOY PRODUCTION: create docker image, upload it to AWS ECR and use it in AWS Lambda # - step 1: generate zappa settings for the app inside the docker image # - step 2: create docker image and push it to AWS ECR # - step 3: update lambda function to use the new docker image From 49ab05be1e2ecf07b957c69ed3faf140f459608c Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Mon, 16 Oct 2023 15:30:12 +0200 Subject: [PATCH 29/71] fix CI tag regex for when the CI job can be triggered --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5f789a31..7fb01881 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -248,7 +248,7 @@ deploy-production-rest-lambda: when: manual only: variables: - - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]$/ + - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+$/ image: python:3.9.13 before_script: # configure aws access credentials: @@ -300,7 +300,7 @@ deploy-production-generate-zappa-settings: when: manual only: variables: - - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/ + - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+$/ image: python:3.9.13 before_script: # configure aws access credentials: @@ -347,7 +347,7 @@ deploy-production-create-docker-image: stage: deploy-create-docker-image only: variables: - - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/ + - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+$/ dependencies: - deploy-production-generate-zappa-settings needs: ["deploy-production-generate-zappa-settings"] @@ -375,7 +375,7 @@ deploy-production-rest-lambda-docker: stage: deploy-update-lambda only: variables: - - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/ + - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+$/ dependencies: - deploy-production-create-docker-image needs: ["deploy-production-create-docker-image"] From 2fdfc8f526eba4c4c841e85602e4f7b095e5576c Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Mon, 16 Oct 2023 15:55:11 +0200 Subject: [PATCH 30/71] keep artifacts to avoid repeating generation of zappa settings --- .gitlab-ci.yml | 70 ++++++++++++-------------------------------------- 1 file changed, 16 insertions(+), 54 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7fb01881..a101cb94 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -113,7 +113,7 @@ deploy-testing-rest-lambda: # - step 2: create docker image and push it to AWS ECR # - step 3: update lambda function to use the new docker image -# generate zappa settings for the app inside the docker image +# step 1: generate zappa settings for the app inside the docker image deploy-testing-generate-zappa-settings: stage: deploy-generate-zappa-settings when: manual @@ -155,13 +155,12 @@ deploy-testing-generate-zappa-settings: - sed -i "s#@@LOGGING_LEVEL@@#$LOGGING_LEVEL_TESTING#g" zappa_settings.json # generate zappa_settings.py for docker image from zappa_settings.json - pipenv run zappa save-python-settings-file testing-docker - # save downloaded process definitions and zappa configuration - # for CI job that creates docker image + # keep downloaded process definitions and zappa configuration for the following CI jobs artifacts: untracked: true expire_in: 1 day -# create docker image and push it to AWS ECR +# step 2: create docker image and push it to AWS ECR deploy-testing-create-docker-image: stage: deploy-create-docker-image only: @@ -188,8 +187,12 @@ deploy-testing-create-docker-image: # push docker images (versioned with CI tag and "latest") to AWS ECR with docker - docker push "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:$CI_COMMIT_TAG" - docker push "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest" + # keep downloaded process definitions and zappa configuration for the following CI jobs + artifacts: + untracked: true + expire_in: 1 day -# update lambda function to use the new docker image +# step 3: update lambda function to use the new docker image deploy-testing-rest-lambda-docker: stage: deploy-update-lambda only: @@ -211,29 +214,7 @@ deploy-testing-rest-lambda-docker: - chmod +x download-process-definitions.sh - ./download-process-definitions.sh - cd rest/ - - ls - pipenv install --dev - # create zappa_settings.json on-the-fly: - - cp zappa_settings.json.template zappa_settings.json - - sed -i "s/@@AWS_ACCESS_KEY_ID@@/$TESTINGDATA_AWS_ACCESS_KEY_ID/g" zappa_settings.json - - sed -i "s#@@AWS_SECRET_ACCESS_KEY@@#$TESTINGDATA_AWS_SECRET_ACCESS_KEY#g" zappa_settings.json - - sed -i "s/@@TESTING_SH_CLIENT_ID@@/$TESTING_SH_CLIENT_ID/g" zappa_settings.json - - sed -i "s/@@TESTING_SH_CLIENT_SECRET@@/$TESTING_SH_CLIENT_SECRET/g" zappa_settings.json - - sed -i "s/@@BACKEND_VERSION@@/$CI_COMMIT_TAG/g" zappa_settings.json - - sed -i "s/@@RESULTS_S3_BUCKET_NAME_MAIN@@/$RESULTS_S3_BUCKET_NAME_MAIN/g" zappa_settings.json - - sed -i "s/@@RESULTS_S3_BUCKET_NAME_CREODIAS@@/$RESULTS_S3_BUCKET_NAME_CREODIAS/g" zappa_settings.json - - sed -i "s/@@RESULTS_S3_BUCKET_NAME_USWEST@@/$RESULTS_S3_BUCKET_NAME_USWEST/g" zappa_settings.json - - sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN/g" zappa_settings.json - - sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS/g" zappa_settings.json - - sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST/g" zappa_settings.json - - sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN@@#$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN#g" zappa_settings.json - - sed -i "s/@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS@@/$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS/g" zappa_settings.json - - sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST@@#$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST#g" zappa_settings.json - - sed -i "s#@@USAGE_REPORTING_BASE_URL_TESTING@@#$USAGE_REPORTING_BASE_URL_TESTING#g" zappa_settings.json - - sed -i "s#@@USAGE_REPORTING_AUTH_URL_TESTING@@#$USAGE_REPORTING_AUTH_URL_TESTING#g" zappa_settings.json - - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_ID_TESTING@@#$USAGE_REPORTING_AUTH_CLIENT_ID_TESTING#g" zappa_settings.json - - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING@@#$USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING#g" zappa_settings.json - - sed -i "s#@@LOGGING_LEVEL@@#$LOGGING_LEVEL_TESTING#g" zappa_settings.json # deploy lambda with new docker image - pipenv run zappa deploy "$DEPLOYMENT_NAME" -d "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest" || pipenv run zappa update "$DEPLOYMENT_NAME" -d "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest" # ensure tables are created: @@ -294,7 +275,7 @@ deploy-production-rest-lambda: # - step 2: create docker image and push it to AWS ECR # - step 3: update lambda function to use the new docker image -# generate zappa settings for the app inside the docker image +# step 1: generate zappa settings for the app inside the docker image deploy-production-generate-zappa-settings: stage: deploy-generate-zappa-settings when: manual @@ -336,13 +317,12 @@ deploy-production-generate-zappa-settings: - sed -i "s#@@LOGGING_LEVEL@@#$LOGGING_LEVEL_PRODUCTION#g" zappa_settings.json # generate zappa_settings.py for docker image from zappa_settings.json - pipenv run zappa save-python-settings-file testing-docker - # save downloaded process definitions and zappa configuration - # for CI job that creates docker image + # keep downloaded process definitions and zappa configuration for the following CI jobs artifacts: untracked: true expire_in: 1 day -# create docker image and push it to AWS ECR +# step 2: create docker image and push it to AWS ECR deploy-production-create-docker-image: stage: deploy-create-docker-image only: @@ -369,8 +349,12 @@ deploy-production-create-docker-image: # push docker images (versioned with CI tag and "latest") to AWS ECR with docker - docker push "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:$CI_COMMIT_TAG" - docker push "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:latest" + # keep downloaded process definitions and zappa configuration for the following CI jobs + artifacts: + untracked: true + expire_in: 1 day -# update lambda function to use the new docker image +# step 3: update lambda function to use the new docker image deploy-production-rest-lambda-docker: stage: deploy-update-lambda only: @@ -392,29 +376,7 @@ deploy-production-rest-lambda-docker: - chmod +x download-process-definitions.sh - ./download-process-definitions.sh - cd rest/ - - ls - pipenv install --dev - # create zappa_settings.json on-the-fly: - - cp zappa_settings.json.template zappa_settings.json - - sed -i "s/@@AWS_ACCESS_KEY_ID@@/$PRODDATA_AWS_ACCESS_KEY_ID/g" zappa_settings.json - - sed -i "s#@@AWS_SECRET_ACCESS_KEY@@#$PRODDATA_AWS_SECRET_ACCESS_KEY#g" zappa_settings.json - - sed -i "s/@@PRODUCTION_SH_CLIENT_ID@@/$PRODUCTION_SH_CLIENT_ID/g" zappa_settings.json - - sed -i "s/@@PRODUCTION_SH_CLIENT_SECRET@@/$PRODUCTION_SH_CLIENT_SECRET/g" zappa_settings.json - - sed -i "s/@@BACKEND_VERSION@@/$CI_COMMIT_TAG/g" zappa_settings.json - - sed -i "s/@@RESULTS_S3_BUCKET_NAME_MAIN@@/$RESULTS_S3_BUCKET_NAME_MAIN_PRODUCTION/g" zappa_settings.json - - sed -i "s/@@RESULTS_S3_BUCKET_NAME_CREODIAS@@/$RESULTS_S3_BUCKET_NAME_CREODIAS_PRODUCTION/g" zappa_settings.json - - sed -i "s/@@RESULTS_S3_BUCKET_NAME_USWEST@@/$RESULTS_S3_BUCKET_NAME_USWEST_PRODUCTION/g" zappa_settings.json - - sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN_PRODUCTION/g" zappa_settings.json - - sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS_PRODUCTION/g" zappa_settings.json - - sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST_PRODUCTION/g" zappa_settings.json - - sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN@@#$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN_PRODUCTION#g" zappa_settings.json - - sed -i "s/@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS@@/$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS_PRODUCTION/g" zappa_settings.json - - sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST@@#$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST_PRODUCTION#g" zappa_settings.json - - sed -i "s#@@USAGE_REPORTING_BASE_URL_PRODUCTION@@#$USAGE_REPORTING_BASE_URL_PRODUCTION#g" zappa_settings.json - - sed -i "s#@@USAGE_REPORTING_AUTH_URL_PRODUCTION@@#$USAGE_REPORTING_AUTH_URL_PRODUCTION#g" zappa_settings.json - - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_ID_PRODUCTION@@#$USAGE_REPORTING_AUTH_CLIENT_ID_PRODUCTION#g" zappa_settings.json - - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_PRODUCTION@@#$USAGE_REPORTING_AUTH_CLIENT_SECRET_PRODUCTION#g" zappa_settings.json - - sed -i "s#@@LOGGING_LEVEL@@#$LOGGING_LEVEL_PRODUCTION#g" zappa_settings.json # deploy lambda with new docker image - pipenv run zappa deploy "$DEPLOYMENT_NAME" -d "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:latest" || pipenv run zappa update "$DEPLOYMENT_NAME" -d "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:latest" # ensure tables are created: From 29fdcf125a00deaa36ea78b7edc680cb22c73b38 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Tue, 17 Oct 2023 16:30:18 +0200 Subject: [PATCH 31/71] don't use hardcoded strings in ci jobs --- .gitlab-ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a101cb94..f849d2c7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -121,6 +121,8 @@ deploy-testing-generate-zappa-settings: variables: - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/ image: python:3.9.13 + variables: + DEPLOYMENT_NAME: "testing-docker" before_script: # configure aws access credentials: - mkdir -p ~/.aws @@ -154,7 +156,7 @@ deploy-testing-generate-zappa-settings: - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING@@#$USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING#g" zappa_settings.json - sed -i "s#@@LOGGING_LEVEL@@#$LOGGING_LEVEL_TESTING#g" zappa_settings.json # generate zappa_settings.py for docker image from zappa_settings.json - - pipenv run zappa save-python-settings-file testing-docker + - pipenv run zappa save-python-settings-file "$DEPLOYMENT_NAME" # keep downloaded process definitions and zappa configuration for the following CI jobs artifacts: untracked: true @@ -283,6 +285,8 @@ deploy-production-generate-zappa-settings: variables: - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+$/ image: python:3.9.13 + variables: + DEPLOYMENT_NAME: "testing-docker" before_script: # configure aws access credentials: - mkdir -p ~/.aws @@ -316,7 +320,7 @@ deploy-production-generate-zappa-settings: - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_PRODUCTION@@#$USAGE_REPORTING_AUTH_CLIENT_SECRET_PRODUCTION#g" zappa_settings.json - sed -i "s#@@LOGGING_LEVEL@@#$LOGGING_LEVEL_PRODUCTION#g" zappa_settings.json # generate zappa_settings.py for docker image from zappa_settings.json - - pipenv run zappa save-python-settings-file testing-docker + - pipenv run zappa save-python-settings-file "$DEPLOYMENT_NAME" # keep downloaded process definitions and zappa configuration for the following CI jobs artifacts: untracked: true From b2398a550359e2c94565c0a034e8780ba64a59f4 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Wed, 18 Oct 2023 11:26:56 +0200 Subject: [PATCH 32/71] add linting github action --- .github/workflows/on_push.yaml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/workflows/on_push.yaml diff --git a/.github/workflows/on_push.yaml b/.github/workflows/on_push.yaml new file mode 100644 index 00000000..75d61466 --- /dev/null +++ b/.github/workflows/on_push.yaml @@ -0,0 +1,26 @@ +name: Lint +on: [push, pull_request] + +jobs: + lint: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Display Python version + run: python -c "import sys; print(sys.version)" + + - name: Install black + run: python -m pip install --upgrade black==22.3.0 + + - name: make sure that the *same* version is used in Pipfiles to avoid incompatibilities + run: grep 'black = "==22.3.0"' rest/Pipfile + + - name: check the files for correct formatting + run: black -l 120 --check . || exit 1 \ No newline at end of file From a99f6a9a38bf8903e978c226abd9d9541a5060d1 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Wed, 18 Oct 2023 11:30:11 +0200 Subject: [PATCH 33/71] cleanup lint github action --- .github/workflows/on_push.yaml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/on_push.yaml b/.github/workflows/on_push.yaml index 75d61466..0fc7d007 100644 --- a/.github/workflows/on_push.yaml +++ b/.github/workflows/on_push.yaml @@ -3,24 +3,20 @@ on: [push, pull_request] jobs: lint: - runs-on: ubuntu-latest - steps: - uses: actions/checkout@v4 + - name: Set up Python uses: actions/setup-python@v4 with: python-version: '3.9' - - name: Display Python version - run: python -c "import sys; print(sys.version)" - - name: Install black run: python -m pip install --upgrade black==22.3.0 - + - name: make sure that the *same* version is used in Pipfiles to avoid incompatibilities run: grep 'black = "==22.3.0"' rest/Pipfile - + - name: check the files for correct formatting run: black -l 120 --check . || exit 1 \ No newline at end of file From 11af680980f500cb7c79d9f9a043a49e15dc83fe Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Wed, 18 Oct 2023 12:04:58 +0200 Subject: [PATCH 34/71] add integration-tests github action --- .github/workflows/on_push.yaml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/on_push.yaml b/.github/workflows/on_push.yaml index 0fc7d007..7956485a 100644 --- a/.github/workflows/on_push.yaml +++ b/.github/workflows/on_push.yaml @@ -1,4 +1,4 @@ -name: Lint +name: Lint and test on: [push, pull_request] jobs: @@ -19,4 +19,18 @@ jobs: run: grep 'black = "==22.3.0"' rest/Pipfile - name: check the files for correct formatting - run: black -l 120 --check . || exit 1 \ No newline at end of file + run: black -l 120 --check . || exit 1 + + integration-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Docker-compose build + run: docker-compose -f docker-compose.yml -f docker-compose.pytest.yml build + + - name: Docker-compose up and run pytest + run: docker-compose -f docker-compose.yml -f docker-compose.pytest.yml up --exit-code-from pytest + + - name: Docker-compose cleanup + run: docker-compose -f docker-compose.yml -f docker-compose.pytest.yml down -v \ No newline at end of file From ae1b78ae226889f810b96e12df02edb281d658a5 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Wed, 18 Oct 2023 12:39:26 +0200 Subject: [PATCH 35/71] integration-tests action job cleanup even if tests fail --- .github/workflows/on_push.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/on_push.yaml b/.github/workflows/on_push.yaml index 7956485a..c5446f76 100644 --- a/.github/workflows/on_push.yaml +++ b/.github/workflows/on_push.yaml @@ -33,4 +33,5 @@ jobs: run: docker-compose -f docker-compose.yml -f docker-compose.pytest.yml up --exit-code-from pytest - name: Docker-compose cleanup + if: always() run: docker-compose -f docker-compose.yml -f docker-compose.pytest.yml down -v \ No newline at end of file From 219ab7562a8139d8ec631df692302c5fd6bbd699 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Wed, 18 Oct 2023 13:03:42 +0200 Subject: [PATCH 36/71] add env variables for integration-tests github action --- .github/workflows/on_push.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/on_push.yaml b/.github/workflows/on_push.yaml index c5446f76..c897b3c0 100644 --- a/.github/workflows/on_push.yaml +++ b/.github/workflows/on_push.yaml @@ -23,6 +23,24 @@ jobs: integration-tests: runs-on: ubuntu-latest + + env: + SH_CLIENT_ID: ${{ secrets.TESTS_SH_CLIENT_ID }} + SH_CLIENT_SECRET: ${{ secrets.TESTS_SH_CLIENT_SECRET }} + RESULTS_S3_BUCKET_NAME_MAIN: ${{ secrets.RESULTS_S3_BUCKET_NAME_MAIN }} + RESULTS_S3_BUCKET_NAME_CREODIAS: ${{ secrets.RESULTS_S3_BUCKET_NAME_CREODIAS }} + RESULTS_S3_BUCKET_NAME_USWEST: ${{ secrets.RESULTS_S3_BUCKET_NAME_USWEST }} + RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN: ${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN }} + RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS: ${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS }} + RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST: ${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST }} + RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN }} + RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS }} + RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST }} + USAGE_REPORTING_BASE_URL: ${{ secrets.USAGE_REPORTING_BASE_URL_TESTING }} + USAGE_REPORTING_AUTH_URL: ${{ secrets.USAGE_REPORTING_AUTH_URL_TESTING }} + USAGE_REPORTING_AUTH_CLIENT_ID: ${{ secrets.USAGE_REPORTING_AUTH_CLIENT_ID_TESTING }} + USAGE_REPORTING_AUTH_CLIENT_SECRET: ${{ secrets.USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING }} + steps: - uses: actions/checkout@v4 From 1ea4ed9db5d454707f1ff3b7200b2dae4318ff50 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Wed, 18 Oct 2023 13:26:34 +0200 Subject: [PATCH 37/71] fix wrongly named env vars in integration-tests github action --- .github/workflows/on_push.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/on_push.yaml b/.github/workflows/on_push.yaml index c897b3c0..5f13f5ff 100644 --- a/.github/workflows/on_push.yaml +++ b/.github/workflows/on_push.yaml @@ -25,8 +25,8 @@ jobs: runs-on: ubuntu-latest env: - SH_CLIENT_ID: ${{ secrets.TESTS_SH_CLIENT_ID }} - SH_CLIENT_SECRET: ${{ secrets.TESTS_SH_CLIENT_SECRET }} + TESTS_SH_CLIENT_ID: ${{ secrets.TESTS_SH_CLIENT_ID }} + TESTS_SH_CLIENT_SECRET: ${{ secrets.TESTS_SH_CLIENT_SECRET }} RESULTS_S3_BUCKET_NAME_MAIN: ${{ secrets.RESULTS_S3_BUCKET_NAME_MAIN }} RESULTS_S3_BUCKET_NAME_CREODIAS: ${{ secrets.RESULTS_S3_BUCKET_NAME_CREODIAS }} RESULTS_S3_BUCKET_NAME_USWEST: ${{ secrets.RESULTS_S3_BUCKET_NAME_USWEST }} @@ -36,10 +36,10 @@ jobs: RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN }} RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS }} RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST }} - USAGE_REPORTING_BASE_URL: ${{ secrets.USAGE_REPORTING_BASE_URL_TESTING }} - USAGE_REPORTING_AUTH_URL: ${{ secrets.USAGE_REPORTING_AUTH_URL_TESTING }} - USAGE_REPORTING_AUTH_CLIENT_ID: ${{ secrets.USAGE_REPORTING_AUTH_CLIENT_ID_TESTING }} - USAGE_REPORTING_AUTH_CLIENT_SECRET: ${{ secrets.USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING }} + USAGE_REPORTING_BASE_URL_TESTING: ${{ secrets.USAGE_REPORTING_BASE_URL_TESTING }} + USAGE_REPORTING_AUTH_URL_TESTING: ${{ secrets.USAGE_REPORTING_AUTH_URL_TESTING }} + USAGE_REPORTING_AUTH_CLIENT_ID_TESTING: ${{ secrets.USAGE_REPORTING_AUTH_CLIENT_ID_TESTING }} + USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING: ${{ secrets.USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING }} steps: - uses: actions/checkout@v4 From 5736e1bf0524e9a620c1409319f8a2dbb0255e11 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Wed, 18 Oct 2023 13:45:20 +0200 Subject: [PATCH 38/71] use better name for lint and integration-tests github actions --- .github/workflows/{on_push.yaml => lint_and_test.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{on_push.yaml => lint_and_test.yaml} (100%) diff --git a/.github/workflows/on_push.yaml b/.github/workflows/lint_and_test.yaml similarity index 100% rename from .github/workflows/on_push.yaml rename to .github/workflows/lint_and_test.yaml From a2f6806aca651f7345c618b72acb1260dc040c4b Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Wed, 18 Oct 2023 14:59:56 +0200 Subject: [PATCH 39/71] add github action for deploying to lambda --- .github/workflows/deploy_testing.yaml | 95 +++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 .github/workflows/deploy_testing.yaml diff --git a/.github/workflows/deploy_testing.yaml b/.github/workflows/deploy_testing.yaml new file mode 100644 index 00000000..10aae0e7 --- /dev/null +++ b/.github/workflows/deploy_testing.yaml @@ -0,0 +1,95 @@ +name: Deploy testing +on: + workflow_dispatch + +jobs: + deploy-testing: + if: success() && startsWith(github.ref, 'refs/tags/v') + + runs-on: ubuntu-latest + + env: + GITHUB_REF_TAG: ${{ github.ref }} + DEPLOYMENT_NAME: "testing-docker" + TESTINGDATA_AWS_ACCESS_KEY_ID: ${{ secrets.TESTINGDATA_AWS_ACCESS_KEY_ID }} + TESTINGDATA_AWS_SECRET_ACCESS_KEY: ${{ secrets.TESTINGDATA_AWS_SECRET_ACCESS_KEY }} + TESTING_SH_CLIENT_ID: ${{ secrets.TESTING_SH_CLIENT_ID }} + TESTING_SH_CLIENT_SECRET: ${{ secrets.TESTING_SH_CLIENT_SECRET }} + RESULTS_S3_BUCKET_NAME_MAIN: ${{ secrets.RESULTS_S3_BUCKET_NAME_MAIN }} + RESULTS_S3_BUCKET_NAME_CREODIAS: ${{ secrets.RESULTS_S3_BUCKET_NAME_CREODIAS }} + RESULTS_S3_BUCKET_NAME_USWEST: ${{ secrets.RESULTS_S3_BUCKET_NAME_USWEST }} + RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN: ${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN }} + RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS: ${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS }} + RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST: ${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST }} + RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN }} + RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS }} + RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST }} + USAGE_REPORTING_BASE_URL_TESTING: ${{ secrets.USAGE_REPORTING_BASE_URL_TESTING }} + USAGE_REPORTING_AUTH_URL_TESTING: ${{ secrets.USAGE_REPORTING_AUTH_URL_TESTING }} + USAGE_REPORTING_AUTH_CLIENT_ID_TESTING: ${{ secrets.USAGE_REPORTING_AUTH_CLIENT_ID_TESTING }} + USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING: ${{ secrets.USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING }} + + steps: + - uses: actions/checkout@v1 + + - name: configure aws access credentials + run: | + mkdir -p ~/.aws + echo -e "[default]\nregion=eu-central-1" > ~/.aws/config + echo -e "[default]\naws_access_key_id=$TESTINGZAPPA_AWS_ACCESS_KEY_ID\naws_secret_access_key=$TESTINGZAPPA_AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials + + - name: install pipenv and aws + run: | + pip install pipenv==2021.5.29 + pip install awscli --no-build-isolation + + - name: download process definitions + run: | + chmod +x download-process-definitions.sh + ./download-process-definitions.sh + + - name: move to rest folder + run: cd rest/ + + - name: install dependencies needed for deployment + run: pipenv install --dev + + - name: create zappa_settings.json on-the-fly + run: | + cp zappa_settings.json.template zappa_settings.json + sed -i "s/@@AWS_ACCESS_KEY_ID@@/$TESTINGDATA_AWS_ACCESS_KEY_ID/g" zappa_settings.json + sed -i "s#@@AWS_SECRET_ACCESS_KEY@@#$TESTINGDATA_AWS_SECRET_ACCESS_KEY#g" zappa_settings.json + sed -i "s/@@TESTING_SH_CLIENT_ID@@/$TESTING_SH_CLIENT_ID/g" zappa_settings.json + sed -i "s/@@TESTING_SH_CLIENT_SECRET@@/$TESTING_SH_CLIENT_SECRET/g" zappa_settings.json + sed -i "s/@@BACKEND_VERSION@@/$CI_COMMIT_TAG/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_NAME_MAIN@@/$RESULTS_S3_BUCKET_NAME_MAIN/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_NAME_CREODIAS@@/$RESULTS_S3_BUCKET_NAME_CREODIAS/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_NAME_USWEST@@/$RESULTS_S3_BUCKET_NAME_USWEST/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST/g" zappa_settings.json + sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN@@#$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN#g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS@@/$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS/g" zappa_settings.json + sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST@@#$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST#g" zappa_settings.json + sed -i "s#@@USAGE_REPORTING_BASE_URL_TESTING@@#$USAGE_REPORTING_BASE_URL_TESTING#g" zappa_settings.json + sed -i "s#@@USAGE_REPORTING_AUTH_URL_TESTING@@#$USAGE_REPORTING_AUTH_URL_TESTING#g" zappa_settings.json + sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_ID_TESTING@@#$USAGE_REPORTING_AUTH_CLIENT_ID_TESTING#g" zappa_settings.json + sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING@@#$USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING#g" zappa_settings.json + sed -i "s#@@LOGGING_LEVEL@@#$LOGGING_LEVEL_TESTING#g" zappa_settings.json + + - name: generate zappa_settings.py for docker image from zappa_settings.json + run: pipenv run zappa save-python-settings-file "$DEPLOYMENT_NAME" + + - name: build docker image with correct tags + run: docker build -t "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:$GITHUB_REF_TAG" -t "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest" --build-arg VERSION=$GITHUB_REF_TAG --build-arg VCS_REF=$GITHUB_SHA --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') . + + - name: login for AWS ECR docker + run: aws ecr get-login-password --region eu-central-1 | docker login --username AWS --password-stdin "$TESTING_ECR_REGISTRY" + + - name: push docker images (versioned with CI tag and "latest") to AWS ECR with docker + run: | + docker push "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:$GITHUB_REF_TAG" + docker push "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest" + + - name: deploy lambda with new docker image + run: pipenv run zappa deploy "$DEPLOYMENT_NAME" -d "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest" || pipenv run zappa update "$DEPLOYMENT_NAME" -d "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest" \ No newline at end of file From b416e7a3d8e4d30e3f862e1faca448cbf42744d6 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Wed, 18 Oct 2023 16:19:06 +0200 Subject: [PATCH 40/71] different expression for checking if tag exists for deploy-testing github action --- .github/workflows/deploy_testing.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy_testing.yaml b/.github/workflows/deploy_testing.yaml index 10aae0e7..2b10993d 100644 --- a/.github/workflows/deploy_testing.yaml +++ b/.github/workflows/deploy_testing.yaml @@ -4,7 +4,7 @@ on: jobs: deploy-testing: - if: success() && startsWith(github.ref, 'refs/tags/v') + if: startsWith(github.ref, 'refs/tags/v') runs-on: ubuntu-latest From 6c356eae9f5775f1ef5226dda869e2df6ebba69f Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Wed, 18 Oct 2023 16:31:22 +0200 Subject: [PATCH 41/71] run deploy-testing github action on release --- .github/workflows/deploy_testing.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy_testing.yaml b/.github/workflows/deploy_testing.yaml index 2b10993d..ba37ae5e 100644 --- a/.github/workflows/deploy_testing.yaml +++ b/.github/workflows/deploy_testing.yaml @@ -1,6 +1,7 @@ name: Deploy testing on: - workflow_dispatch + release: + types: [published] jobs: deploy-testing: From 2b40a8cacf29e23cdf99dd0ac89fa6863bfd7414 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Wed, 18 Oct 2023 17:02:30 +0200 Subject: [PATCH 42/71] run deploy-testing github action on tag --- .github/workflows/deploy_testing.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy_testing.yaml b/.github/workflows/deploy_testing.yaml index ba37ae5e..6e5e7b49 100644 --- a/.github/workflows/deploy_testing.yaml +++ b/.github/workflows/deploy_testing.yaml @@ -1,7 +1,9 @@ name: Deploy testing on: - release: - types: [published] + push: + tags: + - '[0-9]+.[0-9]+.[0-9]+' + - '[0-9]+.[0-9]+.[0-9]+rc[0-9]+' jobs: deploy-testing: From 3d0de1a12cc17ac859544e522bc88f3f26594989 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Wed, 18 Oct 2023 17:07:23 +0200 Subject: [PATCH 43/71] rename file with deploy testing github action --- .github/workflows/{deploy_testing.yaml => deploytesting.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{deploy_testing.yaml => deploytesting.yaml} (100%) diff --git a/.github/workflows/deploy_testing.yaml b/.github/workflows/deploytesting.yaml similarity index 100% rename from .github/workflows/deploy_testing.yaml rename to .github/workflows/deploytesting.yaml From 51b26f38aafc4aee998741d9168abf9a72c17f0a Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Thu, 19 Oct 2023 11:11:21 +0200 Subject: [PATCH 44/71] revise tags regex for deploy-testing github action --- .github/workflows/deploytesting.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploytesting.yaml b/.github/workflows/deploytesting.yaml index 6e5e7b49..224830c6 100644 --- a/.github/workflows/deploytesting.yaml +++ b/.github/workflows/deploytesting.yaml @@ -2,8 +2,8 @@ name: Deploy testing on: push: tags: - - '[0-9]+.[0-9]+.[0-9]+' - - '[0-9]+.[0-9]+.[0-9]+rc[0-9]+' + - 'v[0-9]+[.][0-9]+[.][0-9]+' + - 'v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+' jobs: deploy-testing: From de55b87be000bdee51a237305a03c083b9f98342 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Thu, 19 Oct 2023 11:15:41 +0200 Subject: [PATCH 45/71] revise tags regex for deploy-testing github action again --- .github/workflows/deploytesting.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploytesting.yaml b/.github/workflows/deploytesting.yaml index 224830c6..9eaab681 100644 --- a/.github/workflows/deploytesting.yaml +++ b/.github/workflows/deploytesting.yaml @@ -2,8 +2,8 @@ name: Deploy testing on: push: tags: - - 'v[0-9]+[.][0-9]+[.][0-9]+' - - 'v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+' + - 'v[0-9]+.[0-9]+.[0-9]+' + - 'v[0-9]+.[0-9]+.[0-9]+[-]rc.[0-9]+' jobs: deploy-testing: From 1c745059c0d62d49f4c5cf929087d893e3b85f18 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Thu, 19 Oct 2023 11:19:34 +0200 Subject: [PATCH 46/71] revise tags regex for deploy-testing github action again 2 --- .github/workflows/deploytesting.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploytesting.yaml b/.github/workflows/deploytesting.yaml index 9eaab681..4aef1ba9 100644 --- a/.github/workflows/deploytesting.yaml +++ b/.github/workflows/deploytesting.yaml @@ -3,7 +3,7 @@ on: push: tags: - 'v[0-9]+.[0-9]+.[0-9]+' - - 'v[0-9]+.[0-9]+.[0-9]+[-]rc.[0-9]+' + - 'v[0-9]+[\.][0-9]+[\.][0-9]+[\-]rc[\.][0-9]+' jobs: deploy-testing: From 87b393741e8c01e9aff5342d5cf65caeac9f6bd0 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Thu, 19 Oct 2023 11:22:10 +0200 Subject: [PATCH 47/71] revise tags regex for deploy-testing github action again 3 --- .github/workflows/deploytesting.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploytesting.yaml b/.github/workflows/deploytesting.yaml index 4aef1ba9..4eb3398a 100644 --- a/.github/workflows/deploytesting.yaml +++ b/.github/workflows/deploytesting.yaml @@ -3,7 +3,7 @@ on: push: tags: - 'v[0-9]+.[0-9]+.[0-9]+' - - 'v[0-9]+[\.][0-9]+[\.][0-9]+[\-]rc[\.][0-9]+' + - 'v[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+' jobs: deploy-testing: From 1c5056648986f6676a0989c84a05fd6ed9fe2b71 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Thu, 19 Oct 2023 11:27:43 +0200 Subject: [PATCH 48/71] better name for file that contains deploy-testing github action --- .github/workflows/{deploytesting.yaml => deploy_testing.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{deploytesting.yaml => deploy_testing.yaml} (100%) diff --git a/.github/workflows/deploytesting.yaml b/.github/workflows/deploy_testing.yaml similarity index 100% rename from .github/workflows/deploytesting.yaml rename to .github/workflows/deploy_testing.yaml From fe26d693cdc27764c1c7db3a9ed9dd34d10544d1 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Thu, 19 Oct 2023 11:33:24 +0200 Subject: [PATCH 49/71] check if in the correct directory for deploy-testing github job --- .github/workflows/deploy_testing.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy_testing.yaml b/.github/workflows/deploy_testing.yaml index 4eb3398a..4d905997 100644 --- a/.github/workflows/deploy_testing.yaml +++ b/.github/workflows/deploy_testing.yaml @@ -56,7 +56,10 @@ jobs: - name: install dependencies needed for deployment run: pipenv install --dev - + + - name: list dir + run: ls + - name: create zappa_settings.json on-the-fly run: | cp zappa_settings.json.template zappa_settings.json From 4ef607cd2f9c7e5e660ccc680c7e920efac40df5 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Thu, 19 Oct 2023 13:11:00 +0200 Subject: [PATCH 50/71] execute commands in github action in correct directory --- .github/workflows/deploy_testing.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy_testing.yaml b/.github/workflows/deploy_testing.yaml index 4d905997..19e7e529 100644 --- a/.github/workflows/deploy_testing.yaml +++ b/.github/workflows/deploy_testing.yaml @@ -51,16 +51,12 @@ jobs: chmod +x download-process-definitions.sh ./download-process-definitions.sh - - name: move to rest folder - run: cd rest/ - - name: install dependencies needed for deployment + working-directory: ./rest run: pipenv install --dev - - name: list dir - run: ls - - name: create zappa_settings.json on-the-fly + working-directory: ./rest run: | cp zappa_settings.json.template zappa_settings.json sed -i "s/@@AWS_ACCESS_KEY_ID@@/$TESTINGDATA_AWS_ACCESS_KEY_ID/g" zappa_settings.json @@ -84,18 +80,23 @@ jobs: sed -i "s#@@LOGGING_LEVEL@@#$LOGGING_LEVEL_TESTING#g" zappa_settings.json - name: generate zappa_settings.py for docker image from zappa_settings.json + working-directory: ./rest run: pipenv run zappa save-python-settings-file "$DEPLOYMENT_NAME" - name: build docker image with correct tags + working-directory: ./rest run: docker build -t "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:$GITHUB_REF_TAG" -t "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest" --build-arg VERSION=$GITHUB_REF_TAG --build-arg VCS_REF=$GITHUB_SHA --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') . - name: login for AWS ECR docker + working-directory: ./rest run: aws ecr get-login-password --region eu-central-1 | docker login --username AWS --password-stdin "$TESTING_ECR_REGISTRY" - name: push docker images (versioned with CI tag and "latest") to AWS ECR with docker + working-directory: ./rest run: | docker push "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:$GITHUB_REF_TAG" docker push "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest" - name: deploy lambda with new docker image + working-directory: ./rest run: pipenv run zappa deploy "$DEPLOYMENT_NAME" -d "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest" || pipenv run zappa update "$DEPLOYMENT_NAME" -d "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest" \ No newline at end of file From 8efa7bceddf7f4b48c972f136ba8b71ce5bda753 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Thu, 19 Oct 2023 13:17:22 +0200 Subject: [PATCH 51/71] set correct python version in deploy-testing github action --- .github/workflows/deploy_testing.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy_testing.yaml b/.github/workflows/deploy_testing.yaml index 19e7e529..d4c454b2 100644 --- a/.github/workflows/deploy_testing.yaml +++ b/.github/workflows/deploy_testing.yaml @@ -33,7 +33,12 @@ jobs: USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING: ${{ secrets.USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING }} steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' - name: configure aws access credentials run: | From 1a263d6ce645dd3223e6253c330520509f369ebe Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Thu, 19 Oct 2023 13:24:47 +0200 Subject: [PATCH 52/71] use exact git tag name instead of the whole ref string in deploy-testing github action --- .github/workflows/deploy_testing.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy_testing.yaml b/.github/workflows/deploy_testing.yaml index d4c454b2..a3c8a259 100644 --- a/.github/workflows/deploy_testing.yaml +++ b/.github/workflows/deploy_testing.yaml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest env: - GITHUB_REF_TAG: ${{ github.ref }} + GITHUB_REF_TAG: ${{ github.ref_name }} DEPLOYMENT_NAME: "testing-docker" TESTINGDATA_AWS_ACCESS_KEY_ID: ${{ secrets.TESTINGDATA_AWS_ACCESS_KEY_ID }} TESTINGDATA_AWS_SECRET_ACCESS_KEY: ${{ secrets.TESTINGDATA_AWS_SECRET_ACCESS_KEY }} From 19b78bc8b4201a986adb30e245a672ebfdfaf601 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Thu, 19 Oct 2023 13:56:45 +0200 Subject: [PATCH 53/71] print github tag in deploy-testing github action --- .github/workflows/deploy_testing.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/deploy_testing.yaml b/.github/workflows/deploy_testing.yaml index a3c8a259..fc365450 100644 --- a/.github/workflows/deploy_testing.yaml +++ b/.github/workflows/deploy_testing.yaml @@ -35,6 +35,12 @@ jobs: steps: - uses: actions/checkout@v4 + - name: GitHub Tag Name example + run: | + echo "Tag name from GITHUB_REF_NAME: $GITHUB_REF_NAME" + echo "Tag name from github.ref_name: ${{ github.ref_name }}" + echo "Tag name from github.ref_name: ${{ env.GITHUB_REF_NAME }}" + - name: Set up Python uses: actions/setup-python@v4 with: From 9d7193fe6de232e452c8ee0ebe4a871be1ce5e8b Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Thu, 19 Oct 2023 14:04:51 +0200 Subject: [PATCH 54/71] use GITHUB_REF_NAME for tags in deploy-testing github action --- .github/workflows/deploy_testing.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy_testing.yaml b/.github/workflows/deploy_testing.yaml index fc365450..61049359 100644 --- a/.github/workflows/deploy_testing.yaml +++ b/.github/workflows/deploy_testing.yaml @@ -39,7 +39,7 @@ jobs: run: | echo "Tag name from GITHUB_REF_NAME: $GITHUB_REF_NAME" echo "Tag name from github.ref_name: ${{ github.ref_name }}" - echo "Tag name from github.ref_name: ${{ env.GITHUB_REF_NAME }}" + echo "Tag name from GITHUB_REF_TAG: ${{ GITHUB_REF_TAG }}" - name: Set up Python uses: actions/setup-python@v4 @@ -96,7 +96,7 @@ jobs: - name: build docker image with correct tags working-directory: ./rest - run: docker build -t "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:$GITHUB_REF_TAG" -t "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest" --build-arg VERSION=$GITHUB_REF_TAG --build-arg VCS_REF=$GITHUB_SHA --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') . + run: docker build -t "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:$GITHUB_REF_NAME" -t "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest" --build-arg VERSION=$GITHUB_REF_NAME --build-arg VCS_REF=$GITHUB_SHA --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') . - name: login for AWS ECR docker working-directory: ./rest @@ -105,7 +105,7 @@ jobs: - name: push docker images (versioned with CI tag and "latest") to AWS ECR with docker working-directory: ./rest run: | - docker push "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:$GITHUB_REF_TAG" + docker push "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:$GITHUB_REF_NAME" docker push "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest" - name: deploy lambda with new docker image From 068536d7108c37b49fa2db042bb509a8c6293e0a Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Thu, 19 Oct 2023 14:07:55 +0200 Subject: [PATCH 55/71] fix wrong variable reference in deploy-testing github action --- .github/workflows/deploy_testing.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy_testing.yaml b/.github/workflows/deploy_testing.yaml index 61049359..b8a73978 100644 --- a/.github/workflows/deploy_testing.yaml +++ b/.github/workflows/deploy_testing.yaml @@ -39,7 +39,7 @@ jobs: run: | echo "Tag name from GITHUB_REF_NAME: $GITHUB_REF_NAME" echo "Tag name from github.ref_name: ${{ github.ref_name }}" - echo "Tag name from GITHUB_REF_TAG: ${{ GITHUB_REF_TAG }}" + echo "Tag name from GITHUB_REF_TAG: $GITHUB_REF_TAG" - name: Set up Python uses: actions/setup-python@v4 From d8e949137e2fd1c1209e99074c85c4a7f3eea637 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Thu, 19 Oct 2023 14:29:07 +0200 Subject: [PATCH 56/71] improve whole deploy-testing github action --- .github/workflows/deploy_testing.yaml | 64 +++++++++------------------ 1 file changed, 21 insertions(+), 43 deletions(-) diff --git a/.github/workflows/deploy_testing.yaml b/.github/workflows/deploy_testing.yaml index b8a73978..1c29387b 100644 --- a/.github/workflows/deploy_testing.yaml +++ b/.github/workflows/deploy_testing.yaml @@ -12,35 +12,13 @@ jobs: runs-on: ubuntu-latest env: - GITHUB_REF_TAG: ${{ github.ref_name }} DEPLOYMENT_NAME: "testing-docker" - TESTINGDATA_AWS_ACCESS_KEY_ID: ${{ secrets.TESTINGDATA_AWS_ACCESS_KEY_ID }} - TESTINGDATA_AWS_SECRET_ACCESS_KEY: ${{ secrets.TESTINGDATA_AWS_SECRET_ACCESS_KEY }} - TESTING_SH_CLIENT_ID: ${{ secrets.TESTING_SH_CLIENT_ID }} - TESTING_SH_CLIENT_SECRET: ${{ secrets.TESTING_SH_CLIENT_SECRET }} - RESULTS_S3_BUCKET_NAME_MAIN: ${{ secrets.RESULTS_S3_BUCKET_NAME_MAIN }} - RESULTS_S3_BUCKET_NAME_CREODIAS: ${{ secrets.RESULTS_S3_BUCKET_NAME_CREODIAS }} - RESULTS_S3_BUCKET_NAME_USWEST: ${{ secrets.RESULTS_S3_BUCKET_NAME_USWEST }} - RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN: ${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN }} - RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS: ${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS }} - RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST: ${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST }} - RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN }} - RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS }} - RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST }} - USAGE_REPORTING_BASE_URL_TESTING: ${{ secrets.USAGE_REPORTING_BASE_URL_TESTING }} - USAGE_REPORTING_AUTH_URL_TESTING: ${{ secrets.USAGE_REPORTING_AUTH_URL_TESTING }} - USAGE_REPORTING_AUTH_CLIENT_ID_TESTING: ${{ secrets.USAGE_REPORTING_AUTH_CLIENT_ID_TESTING }} - USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING: ${{ secrets.USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING }} + TESTING_ECR_REGISTRY: ${{ secrets.TESTING_ECR_REGISTRY }} + TESTING_ECR_REGISTRY_IMAGE: ${{ secrets.TESTING_ECR_REGISTRY_IMAGE }} steps: - uses: actions/checkout@v4 - - name: GitHub Tag Name example - run: | - echo "Tag name from GITHUB_REF_NAME: $GITHUB_REF_NAME" - echo "Tag name from github.ref_name: ${{ github.ref_name }}" - echo "Tag name from GITHUB_REF_TAG: $GITHUB_REF_TAG" - - name: Set up Python uses: actions/setup-python@v4 with: @@ -50,7 +28,7 @@ jobs: run: | mkdir -p ~/.aws echo -e "[default]\nregion=eu-central-1" > ~/.aws/config - echo -e "[default]\naws_access_key_id=$TESTINGZAPPA_AWS_ACCESS_KEY_ID\naws_secret_access_key=$TESTINGZAPPA_AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials + echo -e "[default]\naws_access_key_id=${{ secrets.TESTINGZAPPA_AWS_ACCESS_KEY_ID }}\naws_secret_access_key=${{ secrets.TESTINGZAPPA_AWS_SECRET_ACCESS_KEY }}" > ~/.aws/credentials - name: install pipenv and aws run: | @@ -70,24 +48,24 @@ jobs: working-directory: ./rest run: | cp zappa_settings.json.template zappa_settings.json - sed -i "s/@@AWS_ACCESS_KEY_ID@@/$TESTINGDATA_AWS_ACCESS_KEY_ID/g" zappa_settings.json - sed -i "s#@@AWS_SECRET_ACCESS_KEY@@#$TESTINGDATA_AWS_SECRET_ACCESS_KEY#g" zappa_settings.json - sed -i "s/@@TESTING_SH_CLIENT_ID@@/$TESTING_SH_CLIENT_ID/g" zappa_settings.json - sed -i "s/@@TESTING_SH_CLIENT_SECRET@@/$TESTING_SH_CLIENT_SECRET/g" zappa_settings.json - sed -i "s/@@BACKEND_VERSION@@/$CI_COMMIT_TAG/g" zappa_settings.json - sed -i "s/@@RESULTS_S3_BUCKET_NAME_MAIN@@/$RESULTS_S3_BUCKET_NAME_MAIN/g" zappa_settings.json - sed -i "s/@@RESULTS_S3_BUCKET_NAME_CREODIAS@@/$RESULTS_S3_BUCKET_NAME_CREODIAS/g" zappa_settings.json - sed -i "s/@@RESULTS_S3_BUCKET_NAME_USWEST@@/$RESULTS_S3_BUCKET_NAME_USWEST/g" zappa_settings.json - sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN/g" zappa_settings.json - sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS/g" zappa_settings.json - sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST/g" zappa_settings.json - sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN@@#$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN#g" zappa_settings.json - sed -i "s/@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS@@/$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS/g" zappa_settings.json - sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST@@#$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST#g" zappa_settings.json - sed -i "s#@@USAGE_REPORTING_BASE_URL_TESTING@@#$USAGE_REPORTING_BASE_URL_TESTING#g" zappa_settings.json - sed -i "s#@@USAGE_REPORTING_AUTH_URL_TESTING@@#$USAGE_REPORTING_AUTH_URL_TESTING#g" zappa_settings.json - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_ID_TESTING@@#$USAGE_REPORTING_AUTH_CLIENT_ID_TESTING#g" zappa_settings.json - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING@@#$USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING#g" zappa_settings.json + sed -i "s/@@AWS_ACCESS_KEY_ID@@/${{ secrets.TESTINGDATA_AWS_ACCESS_KEY_ID }}/g" zappa_settings.json + sed -i "s#@@AWS_SECRET_ACCESS_KEY@@#${{ secrets.TESTINGDATA_AWS_SECRET_ACCESS_KEY }}#g" zappa_settings.json + sed -i "s/@@TESTING_SH_CLIENT_ID@@/${{ secrets.TESTING_SH_CLIENT_ID }}/g" zappa_settings.json + sed -i "s/@@TESTING_SH_CLIENT_SECRET@@/${{ secrets.TESTING_SH_CLIENT_SECRET }}/g" zappa_settings.json + sed -i "s/@@BACKEND_VERSION@@/$GITHUB_REF_NAME/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_NAME_MAIN@@/${{ secrets.RESULTS_S3_BUCKET_NAME_MAIN }}/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_NAME_CREODIAS@@/${{ secrets.RESULTS_S3_BUCKET_NAME_CREODIAS }}/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_NAME_USWEST@@/${{ secrets.RESULTS_S3_BUCKET_NAME_USWEST }}/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN@@/${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN }}/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS@@/${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS }}/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST@@/${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST }}/g" zappa_settings.json + sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN@@#${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN }}#g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS@@/${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS }}/g" zappa_settings.json + sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST@@#${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST }}#g" zappa_settings.json + sed -i "s#@@USAGE_REPORTING_BASE_URL_TESTING@@#${{ secrets.USAGE_REPORTING_BASE_URL_TESTING }}#g" zappa_settings.json + sed -i "s#@@USAGE_REPORTING_AUTH_URL_TESTING@@#${{ secrets.USAGE_REPORTING_AUTH_URL_TESTING }}#g" zappa_settings.json + sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_ID_TESTING@@#${{ secrets.USAGE_REPORTING_AUTH_CLIENT_ID_TESTING }}#g" zappa_settings.json + sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING@@#${{ secrets.USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING }}#g" zappa_settings.json sed -i "s#@@LOGGING_LEVEL@@#$LOGGING_LEVEL_TESTING#g" zappa_settings.json - name: generate zappa_settings.py for docker image from zappa_settings.json From eedbb9343ee215d116e6b9eaf940d771edfb5c0c Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Thu, 19 Oct 2023 16:20:10 +0200 Subject: [PATCH 57/71] print zappa settings in deploy-testing github action --- .github/workflows/deploy_testing.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/deploy_testing.yaml b/.github/workflows/deploy_testing.yaml index 1c29387b..a83bcb53 100644 --- a/.github/workflows/deploy_testing.yaml +++ b/.github/workflows/deploy_testing.yaml @@ -68,6 +68,10 @@ jobs: sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING@@#${{ secrets.USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING }}#g" zappa_settings.json sed -i "s#@@LOGGING_LEVEL@@#$LOGGING_LEVEL_TESTING#g" zappa_settings.json + - name: print zappa settings + working-directory: ./rest + run: cat zappa_settings.json + - name: generate zappa_settings.py for docker image from zappa_settings.json working-directory: ./rest run: pipenv run zappa save-python-settings-file "$DEPLOYMENT_NAME" From 0846e5512d5c80ff9da827a2c9e78771c0c4c3ea Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Thu, 19 Oct 2023 16:24:00 +0200 Subject: [PATCH 58/71] list process definitions files in deploy-testing github action --- .github/workflows/deploy_testing.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/deploy_testing.yaml b/.github/workflows/deploy_testing.yaml index a83bcb53..fca944a4 100644 --- a/.github/workflows/deploy_testing.yaml +++ b/.github/workflows/deploy_testing.yaml @@ -72,6 +72,10 @@ jobs: working-directory: ./rest run: cat zappa_settings.json + - name: list process definitions + working-directory: ./rest/process_definitions + run: ls + - name: generate zappa_settings.py for docker image from zappa_settings.json working-directory: ./rest run: pipenv run zappa save-python-settings-file "$DEPLOYMENT_NAME" From cc557652b7f370ccb317f96aa8b09151b64baae4 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Thu, 19 Oct 2023 16:31:46 +0200 Subject: [PATCH 59/71] set correct logging level variable to zappa settings for deploying to lambda --- .github/workflows/deploy_testing.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy_testing.yaml b/.github/workflows/deploy_testing.yaml index fca944a4..07c69818 100644 --- a/.github/workflows/deploy_testing.yaml +++ b/.github/workflows/deploy_testing.yaml @@ -66,7 +66,7 @@ jobs: sed -i "s#@@USAGE_REPORTING_AUTH_URL_TESTING@@#${{ secrets.USAGE_REPORTING_AUTH_URL_TESTING }}#g" zappa_settings.json sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_ID_TESTING@@#${{ secrets.USAGE_REPORTING_AUTH_CLIENT_ID_TESTING }}#g" zappa_settings.json sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING@@#${{ secrets.USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING }}#g" zappa_settings.json - sed -i "s#@@LOGGING_LEVEL@@#$LOGGING_LEVEL_TESTING#g" zappa_settings.json + sed -i "s#@@LOGGING_LEVEL@@#${{ secrets.LOGGING_LEVEL_TESTING }}#g" zappa_settings.json - name: print zappa settings working-directory: ./rest From df08011bfb1613bcca4e18dfed58c130c3e0efea Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Thu, 19 Oct 2023 16:41:51 +0200 Subject: [PATCH 60/71] remove debugging stuff from deploy_testing.yaml --- .github/workflows/deploy_testing.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/deploy_testing.yaml b/.github/workflows/deploy_testing.yaml index 07c69818..0d1b63b6 100644 --- a/.github/workflows/deploy_testing.yaml +++ b/.github/workflows/deploy_testing.yaml @@ -68,14 +68,6 @@ jobs: sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING@@#${{ secrets.USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING }}#g" zappa_settings.json sed -i "s#@@LOGGING_LEVEL@@#${{ secrets.LOGGING_LEVEL_TESTING }}#g" zappa_settings.json - - name: print zappa settings - working-directory: ./rest - run: cat zappa_settings.json - - - name: list process definitions - working-directory: ./rest/process_definitions - run: ls - - name: generate zappa_settings.py for docker image from zappa_settings.json working-directory: ./rest run: pipenv run zappa save-python-settings-file "$DEPLOYMENT_NAME" From 3d3c99295164eac4183e0317947946a01395fac5 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Thu, 19 Oct 2023 17:47:25 +0200 Subject: [PATCH 61/71] wait for lint and test using Wait on check --- .github/workflows/deploy_testing.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/deploy_testing.yaml b/.github/workflows/deploy_testing.yaml index 0d1b63b6..61a898a4 100644 --- a/.github/workflows/deploy_testing.yaml +++ b/.github/workflows/deploy_testing.yaml @@ -17,6 +17,14 @@ jobs: TESTING_ECR_REGISTRY_IMAGE: ${{ secrets.TESTING_ECR_REGISTRY_IMAGE }} steps: + - name: Wait for other checks to succeed + uses: lewagon/wait-on-check-action@v1.3.1 + with: + ref: ${{ github.ref }} + running-workflow-name: 'Lint and test' + repo-token: ${{ secrets.GITHUB_TOKEN }} + wait-interval: 10 + - uses: actions/checkout@v4 - name: Set up Python From 69ed3dae908ae6a148b3f20f52f9f613ec029b65 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Thu, 19 Oct 2023 18:07:20 +0200 Subject: [PATCH 62/71] duplicate lint and integration-tests in deploy_testing.yaml so these 2 jobs are done before deploying to testing --- .github/workflows/deploy_testing.yaml | 67 +++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy_testing.yaml b/.github/workflows/deploy_testing.yaml index 61a898a4..1fc4324c 100644 --- a/.github/workflows/deploy_testing.yaml +++ b/.github/workflows/deploy_testing.yaml @@ -6,7 +6,66 @@ on: - 'v[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+' jobs: + + # hackish solution to run lint and integration-tests before deploying + # duplicate from lint_and_test.yaml + + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Install black + run: python -m pip install --upgrade black==22.3.0 + + - name: make sure that the *same* version is used in Pipfiles to avoid incompatibilities + run: grep 'black = "==22.3.0"' rest/Pipfile + + - name: check the files for correct formatting + run: black -l 120 --check . || exit 1 + + integration-tests: + runs-on: ubuntu-latest + + env: + TESTS_SH_CLIENT_ID: ${{ secrets.TESTS_SH_CLIENT_ID }} + TESTS_SH_CLIENT_SECRET: ${{ secrets.TESTS_SH_CLIENT_SECRET }} + RESULTS_S3_BUCKET_NAME_MAIN: ${{ secrets.RESULTS_S3_BUCKET_NAME_MAIN }} + RESULTS_S3_BUCKET_NAME_CREODIAS: ${{ secrets.RESULTS_S3_BUCKET_NAME_CREODIAS }} + RESULTS_S3_BUCKET_NAME_USWEST: ${{ secrets.RESULTS_S3_BUCKET_NAME_USWEST }} + RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN: ${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN }} + RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS: ${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS }} + RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST: ${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST }} + RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN }} + RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS }} + RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST }} + USAGE_REPORTING_BASE_URL_TESTING: ${{ secrets.USAGE_REPORTING_BASE_URL_TESTING }} + USAGE_REPORTING_AUTH_URL_TESTING: ${{ secrets.USAGE_REPORTING_AUTH_URL_TESTING }} + USAGE_REPORTING_AUTH_CLIENT_ID_TESTING: ${{ secrets.USAGE_REPORTING_AUTH_CLIENT_ID_TESTING }} + USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING: ${{ secrets.USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING }} + + steps: + - uses: actions/checkout@v4 + + - name: Docker-compose build + run: docker-compose -f docker-compose.yml -f docker-compose.pytest.yml build + + - name: Docker-compose up and run pytest + run: docker-compose -f docker-compose.yml -f docker-compose.pytest.yml up --exit-code-from pytest + + - name: Docker-compose cleanup + if: always() + run: docker-compose -f docker-compose.yml -f docker-compose.pytest.yml down -v + + deploy-testing: + needs: [lint, integration-tests] + if: startsWith(github.ref, 'refs/tags/v') runs-on: ubuntu-latest @@ -17,14 +76,6 @@ jobs: TESTING_ECR_REGISTRY_IMAGE: ${{ secrets.TESTING_ECR_REGISTRY_IMAGE }} steps: - - name: Wait for other checks to succeed - uses: lewagon/wait-on-check-action@v1.3.1 - with: - ref: ${{ github.ref }} - running-workflow-name: 'Lint and test' - repo-token: ${{ secrets.GITHUB_TOKEN }} - wait-interval: 10 - - uses: actions/checkout@v4 - name: Set up Python From 191f9bd5326b3d4548612ad5f67c936e1464699d Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Fri, 20 Oct 2023 10:35:21 +0200 Subject: [PATCH 63/71] use reusable github actions instead of duplicating code --- .github/workflows/deploy_testing.yaml | 59 +------------------ .github/workflows/lint_and_test.yaml | 53 +---------------- .github/workflows/lint_and_test_reusable.yaml | 56 ++++++++++++++++++ 3 files changed, 61 insertions(+), 107 deletions(-) create mode 100644 .github/workflows/lint_and_test_reusable.yaml diff --git a/.github/workflows/deploy_testing.yaml b/.github/workflows/deploy_testing.yaml index 1fc4324c..0dadbe44 100644 --- a/.github/workflows/deploy_testing.yaml +++ b/.github/workflows/deploy_testing.yaml @@ -6,65 +6,12 @@ on: - 'v[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+' jobs: - - # hackish solution to run lint and integration-tests before deploying - # duplicate from lint_and_test.yaml - - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install black - run: python -m pip install --upgrade black==22.3.0 - - - name: make sure that the *same* version is used in Pipfiles to avoid incompatibilities - run: grep 'black = "==22.3.0"' rest/Pipfile - - name: check the files for correct formatting - run: black -l 120 --check . || exit 1 + run-reusable-lint-and-test: + uses: ./.github/workflows/lint_and_test_reusable.yaml - integration-tests: - runs-on: ubuntu-latest - - env: - TESTS_SH_CLIENT_ID: ${{ secrets.TESTS_SH_CLIENT_ID }} - TESTS_SH_CLIENT_SECRET: ${{ secrets.TESTS_SH_CLIENT_SECRET }} - RESULTS_S3_BUCKET_NAME_MAIN: ${{ secrets.RESULTS_S3_BUCKET_NAME_MAIN }} - RESULTS_S3_BUCKET_NAME_CREODIAS: ${{ secrets.RESULTS_S3_BUCKET_NAME_CREODIAS }} - RESULTS_S3_BUCKET_NAME_USWEST: ${{ secrets.RESULTS_S3_BUCKET_NAME_USWEST }} - RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN: ${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN }} - RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS: ${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS }} - RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST: ${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST }} - RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN }} - RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS }} - RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST }} - USAGE_REPORTING_BASE_URL_TESTING: ${{ secrets.USAGE_REPORTING_BASE_URL_TESTING }} - USAGE_REPORTING_AUTH_URL_TESTING: ${{ secrets.USAGE_REPORTING_AUTH_URL_TESTING }} - USAGE_REPORTING_AUTH_CLIENT_ID_TESTING: ${{ secrets.USAGE_REPORTING_AUTH_CLIENT_ID_TESTING }} - USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING: ${{ secrets.USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING }} - - steps: - - uses: actions/checkout@v4 - - - name: Docker-compose build - run: docker-compose -f docker-compose.yml -f docker-compose.pytest.yml build - - - name: Docker-compose up and run pytest - run: docker-compose -f docker-compose.yml -f docker-compose.pytest.yml up --exit-code-from pytest - - - name: Docker-compose cleanup - if: always() - run: docker-compose -f docker-compose.yml -f docker-compose.pytest.yml down -v - - deploy-testing: - needs: [lint, integration-tests] + needs: run-reusable-lint-and-test if: startsWith(github.ref, 'refs/tags/v') diff --git a/.github/workflows/lint_and_test.yaml b/.github/workflows/lint_and_test.yaml index 5f13f5ff..cdb419ef 100644 --- a/.github/workflows/lint_and_test.yaml +++ b/.github/workflows/lint_and_test.yaml @@ -2,54 +2,5 @@ name: Lint and test on: [push, pull_request] jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install black - run: python -m pip install --upgrade black==22.3.0 - - - name: make sure that the *same* version is used in Pipfiles to avoid incompatibilities - run: grep 'black = "==22.3.0"' rest/Pipfile - - - name: check the files for correct formatting - run: black -l 120 --check . || exit 1 - - integration-tests: - runs-on: ubuntu-latest - - env: - TESTS_SH_CLIENT_ID: ${{ secrets.TESTS_SH_CLIENT_ID }} - TESTS_SH_CLIENT_SECRET: ${{ secrets.TESTS_SH_CLIENT_SECRET }} - RESULTS_S3_BUCKET_NAME_MAIN: ${{ secrets.RESULTS_S3_BUCKET_NAME_MAIN }} - RESULTS_S3_BUCKET_NAME_CREODIAS: ${{ secrets.RESULTS_S3_BUCKET_NAME_CREODIAS }} - RESULTS_S3_BUCKET_NAME_USWEST: ${{ secrets.RESULTS_S3_BUCKET_NAME_USWEST }} - RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN: ${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN }} - RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS: ${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS }} - RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST: ${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST }} - RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN }} - RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS }} - RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST }} - USAGE_REPORTING_BASE_URL_TESTING: ${{ secrets.USAGE_REPORTING_BASE_URL_TESTING }} - USAGE_REPORTING_AUTH_URL_TESTING: ${{ secrets.USAGE_REPORTING_AUTH_URL_TESTING }} - USAGE_REPORTING_AUTH_CLIENT_ID_TESTING: ${{ secrets.USAGE_REPORTING_AUTH_CLIENT_ID_TESTING }} - USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING: ${{ secrets.USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING }} - - steps: - - uses: actions/checkout@v4 - - - name: Docker-compose build - run: docker-compose -f docker-compose.yml -f docker-compose.pytest.yml build - - - name: Docker-compose up and run pytest - run: docker-compose -f docker-compose.yml -f docker-compose.pytest.yml up --exit-code-from pytest - - - name: Docker-compose cleanup - if: always() - run: docker-compose -f docker-compose.yml -f docker-compose.pytest.yml down -v \ No newline at end of file + run-reusable-lint-and-test: + uses: ./.github/workflows/lint_and_test_reusable.yaml \ No newline at end of file diff --git a/.github/workflows/lint_and_test_reusable.yaml b/.github/workflows/lint_and_test_reusable.yaml new file mode 100644 index 00000000..c7202ce1 --- /dev/null +++ b/.github/workflows/lint_and_test_reusable.yaml @@ -0,0 +1,56 @@ +name: Lint and test +on: + workflow_call: + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Install black + run: python -m pip install --upgrade black==22.3.0 + + - name: make sure that the *same* version is used in Pipfiles to avoid incompatibilities + run: grep 'black = "==22.3.0"' rest/Pipfile + + - name: check the files for correct formatting + run: black -l 120 --check . || exit 1 + + integration-tests: + runs-on: ubuntu-latest + + env: + TESTS_SH_CLIENT_ID: ${{ secrets.TESTS_SH_CLIENT_ID }} + TESTS_SH_CLIENT_SECRET: ${{ secrets.TESTS_SH_CLIENT_SECRET }} + RESULTS_S3_BUCKET_NAME_MAIN: ${{ secrets.RESULTS_S3_BUCKET_NAME_MAIN }} + RESULTS_S3_BUCKET_NAME_CREODIAS: ${{ secrets.RESULTS_S3_BUCKET_NAME_CREODIAS }} + RESULTS_S3_BUCKET_NAME_USWEST: ${{ secrets.RESULTS_S3_BUCKET_NAME_USWEST }} + RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN: ${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN }} + RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS: ${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS }} + RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST: ${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST }} + RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN }} + RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS }} + RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST }} + USAGE_REPORTING_BASE_URL_TESTING: ${{ secrets.USAGE_REPORTING_BASE_URL_TESTING }} + USAGE_REPORTING_AUTH_URL_TESTING: ${{ secrets.USAGE_REPORTING_AUTH_URL_TESTING }} + USAGE_REPORTING_AUTH_CLIENT_ID_TESTING: ${{ secrets.USAGE_REPORTING_AUTH_CLIENT_ID_TESTING }} + USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING: ${{ secrets.USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING }} + + steps: + - uses: actions/checkout@v4 + + - name: Docker-compose build + run: docker-compose -f docker-compose.yml -f docker-compose.pytest.yml build + + - name: Docker-compose up and run pytest + run: docker-compose -f docker-compose.yml -f docker-compose.pytest.yml up --exit-code-from pytest + + - name: Docker-compose cleanup + if: always() + run: docker-compose -f docker-compose.yml -f docker-compose.pytest.yml down -v \ No newline at end of file From ebdf5a0c1da6123ae3a7725291f1aeaf6eeede85 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Fri, 20 Oct 2023 11:04:59 +0200 Subject: [PATCH 64/71] raise errors if env variables for usage reporting are not set --- rest/usage_reporting/report_usage.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/rest/usage_reporting/report_usage.py b/rest/usage_reporting/report_usage.py index b86bb0e7..f576ec84 100644 --- a/rest/usage_reporting/report_usage.py +++ b/rest/usage_reporting/report_usage.py @@ -15,6 +15,22 @@ def __init__(self): self.auth_client_secret = os.environ.get("USAGE_REPORTING_AUTH_CLIENT_SECRET") self.base_url = os.environ.get("USAGE_REPORTING_BASE_URL") + if self.auth_url is None: + log(ERROR, "USAGE_REPORTING_AUTH_URL environment variable is not set") + raise Internal("USAGE_REPORTING_AUTH_URL environment variable is not set") + + if self.auth_client_id is None: + log(ERROR, "USAGE_REPORTING_AUTH_CLIENT_ID environment variable is not set") + raise Internal("USAGE_REPORTING_AUTH_CLIENT_ID environment variable is not set") + + if self.auth_client_secret is None: + log(ERROR, "USAGE_REPORTING_AUTH_CLIENT_SECRET environment variable is not set") + raise Internal("USAGE_REPORTING_AUTH_CLIENT_SECRET environment variable is not set") + + if self.base_url is None: + log(ERROR, "USAGE_REPORTING_BASE_URL environment variable is not set") + raise Internal("USAGE_REPORTING_BASE_URL environment variable is not set") + self.authenticate() def authenticate(self, max_tries=5): From 02f6bad5751b2feabdd0dd4b8b4730ba76d8a76f Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Fri, 20 Oct 2023 11:10:22 +0200 Subject: [PATCH 65/71] print to check if secret is set in reusable lint and test github actions --- .github/workflows/lint_and_test_reusable.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/lint_and_test_reusable.yaml b/.github/workflows/lint_and_test_reusable.yaml index c7202ce1..900025be 100644 --- a/.github/workflows/lint_and_test_reusable.yaml +++ b/.github/workflows/lint_and_test_reusable.yaml @@ -45,6 +45,9 @@ jobs: steps: - uses: actions/checkout@v4 + - name: print usage auth url + run: echo "$USAGE_REPORTING_AUTH_URL_TESTING" + - name: Docker-compose build run: docker-compose -f docker-compose.yml -f docker-compose.pytest.yml build From 799eca8c76ba4edc1583b0b44ab153cf7b5ab0c4 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Fri, 20 Oct 2023 11:19:04 +0200 Subject: [PATCH 66/71] pass all secrets to reusable github workflow --- .github/workflows/deploy_testing.yaml | 1 + .github/workflows/lint_and_test.yaml | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy_testing.yaml b/.github/workflows/deploy_testing.yaml index 0dadbe44..ecca7a42 100644 --- a/.github/workflows/deploy_testing.yaml +++ b/.github/workflows/deploy_testing.yaml @@ -9,6 +9,7 @@ jobs: run-reusable-lint-and-test: uses: ./.github/workflows/lint_and_test_reusable.yaml + secrets: inherit deploy-testing: needs: run-reusable-lint-and-test diff --git a/.github/workflows/lint_and_test.yaml b/.github/workflows/lint_and_test.yaml index cdb419ef..4e54c804 100644 --- a/.github/workflows/lint_and_test.yaml +++ b/.github/workflows/lint_and_test.yaml @@ -3,4 +3,5 @@ on: [push, pull_request] jobs: run-reusable-lint-and-test: - uses: ./.github/workflows/lint_and_test_reusable.yaml \ No newline at end of file + uses: ./.github/workflows/lint_and_test_reusable.yaml + secrets: inherit \ No newline at end of file From 2a47a71f10a2830035b94f339e098ca77c64d165 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Fri, 20 Oct 2023 11:27:27 +0200 Subject: [PATCH 67/71] reusable lint and test github workflow --- .github/workflows/deploy_testing.yaml | 2 +- .github/workflows/lint_and_test.yaml | 2 +- ...nt_and_test_reusable.yaml => reusable_lint_and_test.yaml} | 5 +---- 3 files changed, 3 insertions(+), 6 deletions(-) rename .github/workflows/{lint_and_test_reusable.yaml => reusable_lint_and_test.yaml} (95%) diff --git a/.github/workflows/deploy_testing.yaml b/.github/workflows/deploy_testing.yaml index ecca7a42..f8cc0bfb 100644 --- a/.github/workflows/deploy_testing.yaml +++ b/.github/workflows/deploy_testing.yaml @@ -8,7 +8,7 @@ on: jobs: run-reusable-lint-and-test: - uses: ./.github/workflows/lint_and_test_reusable.yaml + uses: ./.github/workflows/reusable_lint_and_test.yaml secrets: inherit deploy-testing: diff --git a/.github/workflows/lint_and_test.yaml b/.github/workflows/lint_and_test.yaml index 4e54c804..fd4c4c6f 100644 --- a/.github/workflows/lint_and_test.yaml +++ b/.github/workflows/lint_and_test.yaml @@ -3,5 +3,5 @@ on: [push, pull_request] jobs: run-reusable-lint-and-test: - uses: ./.github/workflows/lint_and_test_reusable.yaml + uses: ./.github/workflows/reusable_lint_and_test.yaml secrets: inherit \ No newline at end of file diff --git a/.github/workflows/lint_and_test_reusable.yaml b/.github/workflows/reusable_lint_and_test.yaml similarity index 95% rename from .github/workflows/lint_and_test_reusable.yaml rename to .github/workflows/reusable_lint_and_test.yaml index 900025be..77c3f213 100644 --- a/.github/workflows/lint_and_test_reusable.yaml +++ b/.github/workflows/reusable_lint_and_test.yaml @@ -1,4 +1,4 @@ -name: Lint and test +name: Reusable lint and test on: workflow_call: @@ -45,9 +45,6 @@ jobs: steps: - uses: actions/checkout@v4 - - name: print usage auth url - run: echo "$USAGE_REPORTING_AUTH_URL_TESTING" - - name: Docker-compose build run: docker-compose -f docker-compose.yml -f docker-compose.pytest.yml build From e40d6a55d0e60e022c2aa52fc63774b4570f946c Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Fri, 20 Oct 2023 12:53:34 +0200 Subject: [PATCH 68/71] add deploy-production workflow --- .github/workflows/deploy_production.yaml | 97 ++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 .github/workflows/deploy_production.yaml diff --git a/.github/workflows/deploy_production.yaml b/.github/workflows/deploy_production.yaml new file mode 100644 index 00000000..262119a9 --- /dev/null +++ b/.github/workflows/deploy_production.yaml @@ -0,0 +1,97 @@ +name: Deploy production +on: + push: + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + +jobs: + + run-reusable-lint-and-test: + uses: ./.github/workflows/reusable_lint_and_test.yaml + secrets: inherit + + deploy-testing: + needs: run-reusable-lint-and-test + + if: startsWith(github.ref, 'refs/tags/v') + + runs-on: ubuntu-latest + + env: + DEPLOYMENT_NAME: "production-docker" + PROD_ECR_REGISTRY: ${{ secrets.PROD_ECR_REGISTRY }} + PROD_ECR_REGISTRY_IMAGE: ${{ secrets.PROD_ECR_REGISTRY_IMAGE }} + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: configure aws access credentials + run: | + mkdir -p ~/.aws + echo -e "[default]\nregion=eu-central-1" > ~/.aws/config + echo -e "[default]\naws_access_key_id=${{ secrets.PRODZAPPA_AWS_ACCESS_KEY_ID }}\naws_secret_access_key=${{ secrets.PRODZAPPA_AWS_SECRET_ACCESS_KEY }}" > ~/.aws/credentials + + - name: install pipenv and aws + run: | + pip install pipenv==2021.5.29 + pip install awscli --no-build-isolation + + - name: download process definitions + run: | + chmod +x download-process-definitions.sh + ./download-process-definitions.sh + + - name: install dependencies needed for deployment + working-directory: ./rest + run: pipenv install --dev + + - name: create zappa_settings.json on-the-fly + working-directory: ./rest + run: | + cp zappa_settings.json.template zappa_settings.json + sed -i "s/@@AWS_ACCESS_KEY_ID@@/${{ secrets.PRODDATA_AWS_ACCESS_KEY_ID }}/g" zappa_settings.json + sed -i "s#@@AWS_SECRET_ACCESS_KEY@@#${{ secrets.PRODDATA_AWS_SECRET_ACCESS_KEY }}#g" zappa_settings.json + sed -i "s/@@TESTING_SH_CLIENT_ID@@/${{ secrets.PRODUCTION_SH_CLIENT_ID }}/g" zappa_settings.json + sed -i "s/@@TESTING_SH_CLIENT_SECRET@@/${{ secrets.PRODUCTION_SH_CLIENT_SECRET }}/g" zappa_settings.json + sed -i "s/@@BACKEND_VERSION@@/$GITHUB_REF_NAME/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_NAME_MAIN@@/${{ secrets.RESULTS_S3_BUCKET_NAME_MAIN_PRODUCTION }}/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_NAME_CREODIAS@@/${{ secrets.RESULTS_S3_BUCKET_NAME_CREODIAS_PRODUCTION }}/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_NAME_USWEST@@/${{ secrets.RESULTS_S3_BUCKET_NAME_USWEST_PRODUCTION }}/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN@@/${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN_PRODUCTION }}/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS@@/${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS_PRODUCTION }}/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST@@/${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST_PRODUCTION }}/g" zappa_settings.json + sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN@@#${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN_PRODUCTION }}#g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS@@/${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS_PRODUCTION }}/g" zappa_settings.json + sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST@@#${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST_PRODUCTION }}#g" zappa_settings.json + sed -i "s#@@USAGE_REPORTING_BASE_URL_TESTING@@#${{ secrets.USAGE_REPORTING_BASE_URL_PRODUCTION }}#g" zappa_settings.json + sed -i "s#@@USAGE_REPORTING_AUTH_URL_TESTING@@#${{ secrets.USAGE_REPORTING_AUTH_URL_PRODUCTION }}#g" zappa_settings.json + sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_ID_TESTING@@#${{ secrets.USAGE_REPORTING_AUTH_CLIENT_ID_PRODUCTION }}#g" zappa_settings.json + sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING@@#${{ secrets.USAGE_REPORTING_AUTH_CLIENT_SECRET_PRODUCTION }}#g" zappa_settings.json + sed -i "s#@@LOGGING_LEVEL@@#${{ secrets.LOGGING_LEVEL_PRODUCTION }}#g" zappa_settings.json + + - name: generate zappa_settings.py for docker image from zappa_settings.json + working-directory: ./rest + run: pipenv run zappa save-python-settings-file "$DEPLOYMENT_NAME" + + - name: build docker image with correct tags + working-directory: ./rest + run: docker build -t "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:$GITHUB_REF_NAME" -t "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:latest" --build-arg VERSION=$GITHUB_REF_NAME --build-arg VCS_REF=$GITHUB_SHA --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') . + + - name: login for AWS ECR docker + working-directory: ./rest + run: aws ecr get-login-password --region eu-central-1 | docker login --username AWS --password-stdin "$PROD_ECR_REGISTRY" + + - name: push docker images (versioned with CI tag and "latest") to AWS ECR with docker + working-directory: ./rest + run: | + docker push "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:$GITHUB_REF_NAME" + docker push "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:latest" + + - name: deploy lambda with new docker image + working-directory: ./rest + run: pipenv run zappa deploy "$DEPLOYMENT_NAME" -d "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:latest" || pipenv run zappa update "$DEPLOYMENT_NAME" -d "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:latest" \ No newline at end of file From 8f167a14282e8d99b7b5282a83a724737801d874 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Fri, 20 Oct 2023 12:59:09 +0200 Subject: [PATCH 69/71] fix variable names in deploy_production.yaml --- .github/workflows/deploy_production.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/deploy_production.yaml b/.github/workflows/deploy_production.yaml index 262119a9..f9426d93 100644 --- a/.github/workflows/deploy_production.yaml +++ b/.github/workflows/deploy_production.yaml @@ -10,7 +10,7 @@ jobs: uses: ./.github/workflows/reusable_lint_and_test.yaml secrets: inherit - deploy-testing: + deploy-production: needs: run-reusable-lint-and-test if: startsWith(github.ref, 'refs/tags/v') @@ -56,8 +56,8 @@ jobs: cp zappa_settings.json.template zappa_settings.json sed -i "s/@@AWS_ACCESS_KEY_ID@@/${{ secrets.PRODDATA_AWS_ACCESS_KEY_ID }}/g" zappa_settings.json sed -i "s#@@AWS_SECRET_ACCESS_KEY@@#${{ secrets.PRODDATA_AWS_SECRET_ACCESS_KEY }}#g" zappa_settings.json - sed -i "s/@@TESTING_SH_CLIENT_ID@@/${{ secrets.PRODUCTION_SH_CLIENT_ID }}/g" zappa_settings.json - sed -i "s/@@TESTING_SH_CLIENT_SECRET@@/${{ secrets.PRODUCTION_SH_CLIENT_SECRET }}/g" zappa_settings.json + sed -i "s/@@PRODUCTION_SH_CLIENT_ID@@/${{ secrets.PRODUCTION_SH_CLIENT_ID }}/g" zappa_settings.json + sed -i "s/@@PRODUCTION_SH_CLIENT_SECRET@@/${{ secrets.PRODUCTION_SH_CLIENT_SECRET }}/g" zappa_settings.json sed -i "s/@@BACKEND_VERSION@@/$GITHUB_REF_NAME/g" zappa_settings.json sed -i "s/@@RESULTS_S3_BUCKET_NAME_MAIN@@/${{ secrets.RESULTS_S3_BUCKET_NAME_MAIN_PRODUCTION }}/g" zappa_settings.json sed -i "s/@@RESULTS_S3_BUCKET_NAME_CREODIAS@@/${{ secrets.RESULTS_S3_BUCKET_NAME_CREODIAS_PRODUCTION }}/g" zappa_settings.json @@ -68,10 +68,10 @@ jobs: sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN@@#${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN_PRODUCTION }}#g" zappa_settings.json sed -i "s/@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS@@/${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS_PRODUCTION }}/g" zappa_settings.json sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST@@#${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST_PRODUCTION }}#g" zappa_settings.json - sed -i "s#@@USAGE_REPORTING_BASE_URL_TESTING@@#${{ secrets.USAGE_REPORTING_BASE_URL_PRODUCTION }}#g" zappa_settings.json - sed -i "s#@@USAGE_REPORTING_AUTH_URL_TESTING@@#${{ secrets.USAGE_REPORTING_AUTH_URL_PRODUCTION }}#g" zappa_settings.json - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_ID_TESTING@@#${{ secrets.USAGE_REPORTING_AUTH_CLIENT_ID_PRODUCTION }}#g" zappa_settings.json - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING@@#${{ secrets.USAGE_REPORTING_AUTH_CLIENT_SECRET_PRODUCTION }}#g" zappa_settings.json + sed -i "s#@@USAGE_REPORTING_BASE_URL_PRODUCTION@@#${{ secrets.USAGE_REPORTING_BASE_URL_PRODUCTION }}#g" zappa_settings.json + sed -i "s#@@USAGE_REPORTING_AUTH_URL_PRODUCTION@@#${{ secrets.USAGE_REPORTING_AUTH_URL_PRODUCTION }}#g" zappa_settings.json + sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_ID_PRODUCTION@@#${{ secrets.USAGE_REPORTING_AUTH_CLIENT_ID_PRODUCTION }}#g" zappa_settings.json + sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_PRODUCTION@@#${{ secrets.USAGE_REPORTING_AUTH_CLIENT_SECRET_PRODUCTION }}#g" zappa_settings.json sed -i "s#@@LOGGING_LEVEL@@#${{ secrets.LOGGING_LEVEL_PRODUCTION }}#g" zappa_settings.json - name: generate zappa_settings.py for docker image from zappa_settings.json From 8bb78d1dbf53c5d14d81e0017e755f1fe2900b80 Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Fri, 20 Oct 2023 13:21:54 +0200 Subject: [PATCH 70/71] use correct variables in .gitlab-ci.yml --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f849d2c7..b85726be 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -286,7 +286,7 @@ deploy-production-generate-zappa-settings: - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+$/ image: python:3.9.13 variables: - DEPLOYMENT_NAME: "testing-docker" + DEPLOYMENT_NAME: "production-docker" before_script: # configure aws access credentials: - mkdir -p ~/.aws @@ -341,7 +341,7 @@ deploy-production-create-docker-image: # configure aws access credentials: - mkdir -p ~/.aws - echo -e "[default]\nregion=eu-central-1" > ~/.aws/config - - echo -e "[default]\naws_access_key_id=$TESTINGZAPPA_AWS_ACCESS_KEY_ID\naws_secret_access_key=$TESTINGZAPPA_AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials + - echo -e "[default]\naws_access_key_id=$PRODZAPPA_AWS_ACCESS_KEY_ID\naws_secret_access_key=$PRODZAPPA_AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials script: - cd rest/ # build docker image with correct tags @@ -374,7 +374,7 @@ deploy-production-rest-lambda-docker: # configure aws access credentials: - mkdir -p ~/.aws - echo -e "[default]\nregion=eu-central-1" > ~/.aws/config - - echo -e "[default]\naws_access_key_id=$TESTINGZAPPA_AWS_ACCESS_KEY_ID\naws_secret_access_key=$TESTINGZAPPA_AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials + - echo -e "[default]\naws_access_key_id=$PRODZAPPA_AWS_ACCESS_KEY_ID\naws_secret_access_key=$PRODZAPPA_AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials - pip install pipenv==2021.5.29 script: - chmod +x download-process-definitions.sh From 3253ba966770d4048b320fee19e67e0b7de26b6c Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Tue, 24 Oct 2023 15:53:48 +0200 Subject: [PATCH 71/71] remove allow_failure: true --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b85726be..c6eaf1ae 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,7 +30,6 @@ integration tests: - docker-compose -f docker-compose.yml -f docker-compose.pytest.yml up --exit-code-from pytest after_script: - docker-compose -f docker-compose.yml -f docker-compose.pytest.yml down -v - allow_failure: true ## GITHUB: publish_github: