From 4f6403cdf5db8c35ef32387750389775b8d66778 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Wed, 10 Feb 2021 15:05:38 -0500 Subject: [PATCH 001/120] c4-519 make some progress, but will need to rethink wsgi --- deploy/docker/wsgi/Dockerfile | 80 +++++++++++++++++++++++++++++++++++ deploy/docker/wsgi/Makefile | 14 ++++++ pyproject.toml | 2 +- 3 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 deploy/docker/wsgi/Dockerfile create mode 100644 deploy/docker/wsgi/Makefile diff --git a/deploy/docker/wsgi/Dockerfile b/deploy/docker/wsgi/Dockerfile new file mode 100644 index 0000000000..5d394a3edc --- /dev/null +++ b/deploy/docker/wsgi/Dockerfile @@ -0,0 +1,80 @@ +FROM ubuntu:18.04 + +# XXX: We should consider a multi-base image setup where python, poetry, app etc are done +# separately and can be extended. +MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update + +# Add some tools +RUN apt-get -y install emacs vim +RUN apt-get -y install software-properties-common + +# Add Python +RUN add-apt-repository ppa:deadsnakes/ppa +RUN apt-get update +RUN apt-get -y install python3.6-dev +RUN python3.6 -V + +# Install mod_wsgi +RUN apt-get -y install git git-c++ wget +RUN apt-get -y install apache2-dev apache2-utils ssl-cert libapache2-mod-wsgi + +# Install mod_wsgi +RUN apt-get install libapache2-mod-wsgi +RUN systemctl restart apache2 + +# Environment Configuration +# Adapated from https://github.com/python-poetry/poetry/discussions/1879 +# Don't buffer output from Python +ENV PYTHONUNBUFFERED=1 +# Don't write .pyc files (to disk) +ENV PYTHONDONTWRITEBYTECODE=1 + +# Optimize pip +ENV PIP_NO_CACHE_DIR=off +ENV PIP_DISABLE_PIP_VERSION_CHECK=on +ENV PIP_DEFAULT_TIMEOUT=100 + +# Configure poetry +# Reference: https://python-poetry.org/docs/configuration/#using-environment-variables +ENV POETRY_VERSION=1.1.4 +# make poetry install to this location +ENV POETRY_HOME="/home/cgap-admin/poetry" +# make poetry create the virtual environment in the project's root +# it gets named `.venv` +ENV POETRY_VIRTUALENVS_IN_PROJECT=true +ENV POETRY_NO_INTERACTION=1 +# This is where our requirements + virtual environment will live +ENV PYSETUP_PATH="/home/cgap-admin/cgap-portal" +ENV VENV_PATH="/home/cgap-admin/.venv" + + +# Configure Path +ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH" + + +# Configure CGAP User +RUN useradd -ms /bin/bash cgap-admin +RUN mkdir -p /home/cgap-admin/cgap-portal +RUN chown cgap-admin $PYSETUP_PATH +USER cgap-admin + +# Clone to /home/cgap-admin/cgap-portal +WORKDIR /home/cgap-admin +RUN git clone https://github.com/dbmi-bgm/cgap-portal.git + +# Build, configure the back-end +WORKDIR /home/cgap-admin/cgap-portal +RUN curl -sSL https://raw.githubusercontent.com/sdispater/poetry/master/get-poetry.py | python +RUN poetry install && poetry run python setup_eb.py develop +RUN make aws-ip-ranges +RUN cat /dev/urandom | head -c 256 | base64 > session-secret.b64 + +# Execute commands with: +# docker run -it cgap-ingestion-docker: bash -c +# poetry run simulate-environment && +# poetry run reformat-vcf && +# poetry run ingest-vcf diff --git a/deploy/docker/wsgi/Makefile b/deploy/docker/wsgi/Makefile new file mode 100644 index 0000000000..dd7d5e09ee --- /dev/null +++ b/deploy/docker/wsgi/Makefile @@ -0,0 +1,14 @@ +configure: + brew install docker + +build: + docker build -t cgap-docker:0.0.0b0 . + +enter: + docker run -it cgap-docker:0.0.0b0 bash + +info: + @: $(info Here are some 'make' options:) + $(info - Use 'make configure' to install Docker on OSX via Brew.) + $(info - Use 'make build' to build a local version of the Docker image.) + $(info - Use 'make enter' to start a bash session in the build container.) diff --git a/pyproject.toml b/pyproject.toml index e85ae4cfab..14a5e9d060 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] # Note: Various modules refer to this system as "encoded", not "cgap-portal". name = "encoded" -version = "5.2.7" +version = "6.0.0" description = "Clinical Genomics Analysis Platform" authors = ["4DN-DCIC Team "] license = "MIT" From af849008916f3302d118785d7c7a5326dedc40c5 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Wed, 17 Feb 2021 16:50:11 -0500 Subject: [PATCH 002/120] provision some more stuff --- deploy/docker/wsgi/Dockerfile | 138 ++++++++++++++++++----------- deploy/docker/wsgi/cgap.conf | 38 ++++++++ deploy/docker/wsgi/development.ini | 92 +++++++++++++++++++ deploy/docker/wsgi/nginx.conf | 52 +++++++++++ 4 files changed, 269 insertions(+), 51 deletions(-) create mode 100644 deploy/docker/wsgi/cgap.conf create mode 100644 deploy/docker/wsgi/development.ini create mode 100644 deploy/docker/wsgi/nginx.conf diff --git a/deploy/docker/wsgi/Dockerfile b/deploy/docker/wsgi/Dockerfile index 5d394a3edc..05bcbb83af 100644 --- a/deploy/docker/wsgi/Dockerfile +++ b/deploy/docker/wsgi/Dockerfile @@ -1,59 +1,95 @@ -FROM ubuntu:18.04 +FROM python:3.6-buster -# XXX: We should consider a multi-base image setup where python, poetry, app etc are done -# separately and can be extended. MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update - -# Add some tools -RUN apt-get -y install emacs vim -RUN apt-get -y install software-properties-common - -# Add Python -RUN add-apt-repository ppa:deadsnakes/ppa -RUN apt-get update -RUN apt-get -y install python3.6-dev -RUN python3.6 -V - -# Install mod_wsgi -RUN apt-get -y install git git-c++ wget -RUN apt-get -y install apache2-dev apache2-utils ssl-cert libapache2-mod-wsgi - -# Install mod_wsgi -RUN apt-get install libapache2-mod-wsgi -RUN systemctl restart apache2 - -# Environment Configuration -# Adapated from https://github.com/python-poetry/poetry/discussions/1879 -# Don't buffer output from Python -ENV PYTHONUNBUFFERED=1 -# Don't write .pyc files (to disk) -ENV PYTHONDONTWRITEBYTECODE=1 - -# Optimize pip -ENV PIP_NO_CACHE_DIR=off -ENV PIP_DISABLE_PIP_VERSION_CHECK=on -ENV PIP_DEFAULT_TIMEOUT=100 - -# Configure poetry -# Reference: https://python-poetry.org/docs/configuration/#using-environment-variables -ENV POETRY_VERSION=1.1.4 -# make poetry install to this location -ENV POETRY_HOME="/home/cgap-admin/poetry" -# make poetry create the virtual environment in the project's root -# it gets named `.venv` -ENV POETRY_VIRTUALENVS_IN_PROJECT=true -ENV POETRY_NO_INTERACTION=1 -# This is where our requirements + virtual environment will live -ENV PYSETUP_PATH="/home/cgap-admin/cgap-portal" -ENV VENV_PATH="/home/cgap-admin/.venv" - - -# Configure Path -ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH" +# Copied from: https://github.com/nginxinc/docker-nginx/blob/594ce7a8bc26c85af88495ac94d5cd0096b306f7/mainline/buster/Dockerfile + +# Standard set up Nginx +ENV NGINX_VERSION=1.17.10 +ENV NJS_VERSION=0.3.9 +ENV PKG_RELEASE=1~buster + +RUN set -x \ + && apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y gnupg1 ca-certificates \ + && \ + NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \ + found=''; \ + for server in \ + ha.pool.sks-keyservers.net \ + hkp://keyserver.ubuntu.com:80 \ + hkp://p80.pool.sks-keyservers.net:80 \ + pgp.mit.edu \ + ; do \ + echo "Fetching GPG key $NGINX_GPGKEY from $server"; \ + apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \ + done; \ + test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \ + apt-get remove --purge --auto-remove -y gnupg1 && rm -rf /var/lib/apt/lists/* \ + && dpkgArch="$(dpkg --print-architecture)" \ + && nginxPackages=" \ + nginx=${NGINX_VERSION}-${PKG_RELEASE} \ + nginx-module-xslt=${NGINX_VERSION}-${PKG_RELEASE} \ + nginx-module-geoip=${NGINX_VERSION}-${PKG_RELEASE} \ + nginx-module-image-filter=${NGINX_VERSION}-${PKG_RELEASE} \ + nginx-module-njs=${NGINX_VERSION}.${NJS_VERSION}-${PKG_RELEASE} \ + " \ + && case "$dpkgArch" in \ + amd64|i386) \ + echo "deb https://nginx.org/packages/mainline/debian/ buster nginx" >> /etc/apt/sources.list.d/nginx.list \ + && apt-get update \ + ;; \ + *) \ + echo "deb-src https://nginx.org/packages/mainline/debian/ buster nginx" >> /etc/apt/sources.list.d/nginx.list \ + \ + && tempDir="$(mktemp -d)" \ + && chmod 777 "$tempDir" \ + \ + && savedAptMark="$(apt-mark showmanual)" \ + \ + && apt-get update \ + && apt-get build-dep -y $nginxPackages \ + && ( \ + cd "$tempDir" \ + && DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" \ + apt-get source --compile $nginxPackages \ + ) \ + \ + && apt-mark showmanual | xargs apt-mark auto > /dev/null \ + && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; } \ + \ + && ls -lAFh "$tempDir" \ + && ( cd "$tempDir" && dpkg-scanpackages . > Packages ) \ + && grep '^Package: ' "$tempDir/Packages" \ + && echo "deb [ trusted=yes ] file://$tempDir ./" > /etc/apt/sources.list.d/temp.list \ + && apt-get -o Acquire::GzipIndexes=false update \ + ;; \ + esac \ + \ + && apt-get install --no-install-recommends --no-install-suggests -y \ + $nginxPackages \ + gettext-base \ + && apt-get remove --purge --auto-remove -y ca-certificates && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx.list \ + \ + && if [ -n "$tempDir" ]; then \ + apt-get purge -y --auto-remove \ + && rm -rf "$tempDir" /etc/apt/sources.list.d/temp.list; \ + fi + +# forward request and error logs to docker log collector +RUN ln -sf /dev/stdout /var/log/nginx/access.log \ + && ln -sf /dev/stderr /var/log/nginx/error.log + +# end of nginx setup + +EXPOSE 80 +EXPOSE 443 + +# Remove default configuration from Nginx +RUN rm /etc/nginx/conf.d/default.conf +COPY nginx.conf /etc/nginx/conf.d/nginx.conf # Configure CGAP User diff --git a/deploy/docker/wsgi/cgap.conf b/deploy/docker/wsgi/cgap.conf new file mode 100644 index 0000000000..4086f33bf1 --- /dev/null +++ b/deploy/docker/wsgi/cgap.conf @@ -0,0 +1,38 @@ +# cgap.conf + +upstream cgap-portal { + server 127.0.0.1:6543; # one server for now +} + +server { + listen 80; + + # optional ssl configuration + + # listen 443 ssl; + # ssl_certificate /path/to/ssl/pem_file; + # ssl_certificate_key /path/to/ssl/certificate_key; + + # end of optional ssl configuration + + server_name cgap-portal.com; + + access_log /home/cgap/env/access.log; + + location / { + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $host:$server_port; + proxy_set_header X-Forwarded-Port $server_port; + + client_max_body_size 10m; + client_body_buffer_size 128k; + proxy_connect_timeout 60s; + proxy_send_timeout 90s; + proxy_read_timeout 90s; + proxy_buffering off; + proxy_temp_file_write_size 64k; + proxy_pass http://myapp-site; + proxy_redirect off; + } +} \ No newline at end of file diff --git a/deploy/docker/wsgi/development.ini b/deploy/docker/wsgi/development.ini new file mode 100644 index 0000000000..db10817702 --- /dev/null +++ b/deploy/docker/wsgi/development.ini @@ -0,0 +1,92 @@ +### +# app configuration +# http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/environment.html +### + +[app:app] +use = config:base.ini#app +sqlalchemy.url = postgresql://postgres@localhost:5441/postgres?host=/tmp/snovault/pgdata +blob_bucket = encoded-4dn-blobs +metadata_bundles_bucket = elasticbeanstalk-fourfront-cgaplocal-dev-metadata-bundles +load_test_only = true +create_tables = true +testing = true +postgresql.statement_timeout = 20 +mpindexer = true +indexer = true +elasticsearch.aws_auth = false +pyramid.reload_templates = true +pyramid.debug_authorization = false +pyramid.debug_notfound = true +pyramid.debug_routematch = false +pyramid.default_locale_name = en +# this line determines which load function is used in load_data +# most deployments use: "load_test_data = encoded.loadxl:load_test_data" +load_test_data = encoded.loadxl:load_local_data +encoded_version = 100.200.300 +snovault_version = 200.300.400 +utils_version = 300.400.500 +eb_app_version = app-v-development-simulation + +[pipeline:debug] +pipeline = + egg:PasteDeploy#prefix + egg:repoze.debug#pdbpm + app +set pyramid.includes = + pyramid_translogger + +[composite:main] +use = egg:rutter#urlmap +/ = debug +/_indexer = indexer + +[composite:indexer] +use = config:base.ini#indexer + +### +# wsgi server configuration +### + +[server:main] +use = egg:waitress#main +host = 0.0.0.0 +port = 6543 +threads = 1 + +### +# logging configuration +# http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/logging.html +### + +[loggers] +keys = root, wsgi, encoded + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = INFO +handlers = console + +[logger_wsgi] +level = DEBUG +handlers = +qualname = wsgi + +[logger_encoded] +level = DEBUG +handlers = +qualname = encoded + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s diff --git a/deploy/docker/wsgi/nginx.conf b/deploy/docker/wsgi/nginx.conf new file mode 100644 index 0000000000..b1ee845007 --- /dev/null +++ b/deploy/docker/wsgi/nginx.conf @@ -0,0 +1,52 @@ +# nginx.conf +# default from Pyramid Docs - likely needs modification -Will + +user www-data; +worker_processes 4; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; + # multi_accept on; +} + +http { + + ## + # Basic Settings + ## + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + # server_tokens off; + + # server_names_hash_bucket_size 64; + # server_name_in_redirect off; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + ## + # Logging Settings + ## + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + ## + # Gzip Settings + ## + + gzip on; + gzip_disable "msie6"; + + ## + # Virtual Host Configs + ## + + include /etc/nginx/conf.d/*.conf; + include /etc/nginx/sites-enabled/*; +} \ No newline at end of file From 939d7dcf28c1faab00d92deaf85b71588633969b Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Fri, 19 Feb 2021 16:03:37 -0500 Subject: [PATCH 003/120] building, needs development.ini configuration --- deploy/docker/wsgi/Dockerfile | 51 +++++++++++++++++++++++--------- deploy/docker/wsgi/Makefile | 2 +- deploy/docker/wsgi/entrypoint.sh | 4 +++ 3 files changed, 42 insertions(+), 15 deletions(-) create mode 100644 deploy/docker/wsgi/entrypoint.sh diff --git a/deploy/docker/wsgi/Dockerfile b/deploy/docker/wsgi/Dockerfile index 05bcbb83af..0f14e61835 100644 --- a/deploy/docker/wsgi/Dockerfile +++ b/deploy/docker/wsgi/Dockerfile @@ -11,6 +11,7 @@ ENV NGINX_VERSION=1.17.10 ENV NJS_VERSION=0.3.9 ENV PKG_RELEASE=1~buster +# Securely provisions deps, installs Nginx, then removes all artifacts RUN set -x \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y gnupg1 ca-certificates \ @@ -82,7 +83,14 @@ RUN set -x \ RUN ln -sf /dev/stdout /var/log/nginx/access.log \ && ln -sf /dev/stderr /var/log/nginx/error.log -# end of nginx setup +# end of Nginx setup + +# Intall things needed for our system +RUN apt-get update +RUN apt-get install -y curl +RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - +RUN apt-get install -y ca-certificates nodejs npm +ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1 EXPOSE 80 EXPOSE 443 @@ -91,26 +99,41 @@ EXPOSE 443 RUN rm /etc/nginx/conf.d/default.conf COPY nginx.conf /etc/nginx/conf.d/nginx.conf - -# Configure CGAP User +# Configure venv RUN useradd -ms /bin/bash cgap-admin +WORKDIR /home/cgap-admin +RUN python -m venv /opt/venv +RUN chown cgap-admin /opt/venv RUN mkdir -p /home/cgap-admin/cgap-portal -RUN chown cgap-admin $PYSETUP_PATH -USER cgap-admin +RUN mkdir -p /home/cgap-admin/.poetry +RUN chown cgap-admin /home/cgap-admin/cgap-portal +RUN chown cgap-admin /home/cgap-admin/.poetry -# Clone to /home/cgap-admin/cgap-portal -WORKDIR /home/cgap-admin +# Configure Poetry +ENV PYTHONFAULTHANDLER=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONHASHSEED=random \ + PIP_NO_CACHE_DIR=off \ + PIP_DISABLE_PIP_VERSION_CHECK=on \ + PIP_DEFAULT_TIMEOUT=100 \ + POETRY_VERSION=1.1.4 + +# Copy to /home/cgap-admin/cgap-portal +WORKDIR /home/cgap-admin/ RUN git clone https://github.com/dbmi-bgm/cgap-portal.git # Build, configure the back-end WORKDIR /home/cgap-admin/cgap-portal -RUN curl -sSL https://raw.githubusercontent.com/sdispater/poetry/master/get-poetry.py | python -RUN poetry install && poetry run python setup_eb.py develop +RUN . /opt/venv/bin/activate && pip install --upgrade pip +RUN . /opt/venv/bin/activate && make build RUN make aws-ip-ranges RUN cat /dev/urandom | head -c 256 | base64 > session-secret.b64 +USER cgap-admin + +#COPY entrypoint.sh /home/cgap-admin/entrypoint.sh +#ENTRYPOINT ["/home/cgap-admin/entrypoint.sh"] +#TODO link up with nginx +#TODO expose back-end ports +#ie: make deploy1 on local machine, make deploy2/3 runs in the docker + -# Execute commands with: -# docker run -it cgap-ingestion-docker: bash -c -# poetry run simulate-environment && -# poetry run reformat-vcf && -# poetry run ingest-vcf diff --git a/deploy/docker/wsgi/Makefile b/deploy/docker/wsgi/Makefile index dd7d5e09ee..22bd985179 100644 --- a/deploy/docker/wsgi/Makefile +++ b/deploy/docker/wsgi/Makefile @@ -2,7 +2,7 @@ configure: brew install docker build: - docker build -t cgap-docker:0.0.0b0 . + docker build --progress=plain -t cgap-docker:0.0.0b0 . enter: docker run -it cgap-docker:0.0.0b0 bash diff --git a/deploy/docker/wsgi/entrypoint.sh b/deploy/docker/wsgi/entrypoint.sh new file mode 100644 index 0000000000..5eba02e837 --- /dev/null +++ b/deploy/docker/wsgi/entrypoint.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +make deploy2 +exec "$@" From 72fd02658b57e78ce7ef46f313e45327f979c1f2 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Mon, 22 Feb 2021 11:33:31 -0500 Subject: [PATCH 004/120] cleanup, cannot get to postgres --- deploy/docker/wsgi/Dockerfile | 133 ++++++---------------------- deploy/docker/wsgi/Makefile | 2 +- deploy/docker/wsgi/development.ini | 2 +- deploy/docker/wsgi/install_nginx.sh | 77 ++++++++++++++++ deploy/docker/wsgi/nginx.conf | 76 ++++++---------- 5 files changed, 136 insertions(+), 154 deletions(-) create mode 100644 deploy/docker/wsgi/install_nginx.sh diff --git a/deploy/docker/wsgi/Dockerfile b/deploy/docker/wsgi/Dockerfile index 0f14e61835..2b18238ed5 100644 --- a/deploy/docker/wsgi/Dockerfile +++ b/deploy/docker/wsgi/Dockerfile @@ -2,132 +2,57 @@ FROM python:3.6-buster MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" +# Configure (global) Env +ENV CGAP_REPO=https://github.com/dbmi-bgm/cgap-portal.git +ENV CGAP_BRANCH=c4_519 ENV DEBIAN_FRONTEND=noninteractive +ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1 +ENV PYTHONFAULTHANDLER=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONHASHSEED=random \ + PIP_NO_CACHE_DIR=off \ + PIP_DISABLE_PIP_VERSION_CHECK=on \ + PIP_DEFAULT_TIMEOUT=100 \ + POETRY_VERSION=1.1.4 -# Copied from: https://github.com/nginxinc/docker-nginx/blob/594ce7a8bc26c85af88495ac94d5cd0096b306f7/mainline/buster/Dockerfile - -# Standard set up Nginx -ENV NGINX_VERSION=1.17.10 -ENV NJS_VERSION=0.3.9 -ENV PKG_RELEASE=1~buster - -# Securely provisions deps, installs Nginx, then removes all artifacts -RUN set -x \ - && apt-get update \ - && apt-get install --no-install-recommends --no-install-suggests -y gnupg1 ca-certificates \ - && \ - NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \ - found=''; \ - for server in \ - ha.pool.sks-keyservers.net \ - hkp://keyserver.ubuntu.com:80 \ - hkp://p80.pool.sks-keyservers.net:80 \ - pgp.mit.edu \ - ; do \ - echo "Fetching GPG key $NGINX_GPGKEY from $server"; \ - apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \ - done; \ - test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \ - apt-get remove --purge --auto-remove -y gnupg1 && rm -rf /var/lib/apt/lists/* \ - && dpkgArch="$(dpkg --print-architecture)" \ - && nginxPackages=" \ - nginx=${NGINX_VERSION}-${PKG_RELEASE} \ - nginx-module-xslt=${NGINX_VERSION}-${PKG_RELEASE} \ - nginx-module-geoip=${NGINX_VERSION}-${PKG_RELEASE} \ - nginx-module-image-filter=${NGINX_VERSION}-${PKG_RELEASE} \ - nginx-module-njs=${NGINX_VERSION}.${NJS_VERSION}-${PKG_RELEASE} \ - " \ - && case "$dpkgArch" in \ - amd64|i386) \ - echo "deb https://nginx.org/packages/mainline/debian/ buster nginx" >> /etc/apt/sources.list.d/nginx.list \ - && apt-get update \ - ;; \ - *) \ - echo "deb-src https://nginx.org/packages/mainline/debian/ buster nginx" >> /etc/apt/sources.list.d/nginx.list \ - \ - && tempDir="$(mktemp -d)" \ - && chmod 777 "$tempDir" \ - \ - && savedAptMark="$(apt-mark showmanual)" \ - \ - && apt-get update \ - && apt-get build-dep -y $nginxPackages \ - && ( \ - cd "$tempDir" \ - && DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" \ - apt-get source --compile $nginxPackages \ - ) \ - \ - && apt-mark showmanual | xargs apt-mark auto > /dev/null \ - && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; } \ - \ - && ls -lAFh "$tempDir" \ - && ( cd "$tempDir" && dpkg-scanpackages . > Packages ) \ - && grep '^Package: ' "$tempDir/Packages" \ - && echo "deb [ trusted=yes ] file://$tempDir ./" > /etc/apt/sources.list.d/temp.list \ - && apt-get -o Acquire::GzipIndexes=false update \ - ;; \ - esac \ - \ - && apt-get install --no-install-recommends --no-install-suggests -y \ - $nginxPackages \ - gettext-base \ - && apt-get remove --purge --auto-remove -y ca-certificates && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx.list \ - \ - && if [ -n "$tempDir" ]; then \ - apt-get purge -y --auto-remove \ - && rm -rf "$tempDir" /etc/apt/sources.list.d/temp.list; \ - fi - -# forward request and error logs to docker log collector -RUN ln -sf /dev/stdout /var/log/nginx/access.log \ - && ln -sf /dev/stderr /var/log/nginx/error.log - -# end of Nginx setup +# Install nginx (unused for local deployment as this runs as part of `make deploy1`) +COPY install_nginx.sh / +RUN bash /install_nginx.sh +# Remove default configuration from Nginx +RUN rm /etc/nginx/conf.d/default.conf +COPY nginx.conf /etc/nginx/conf.d/nginx.conf # Intall things needed for our system RUN apt-get update -RUN apt-get install -y curl +RUN apt-get install -y curl postgresql-client RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - RUN apt-get install -y ca-certificates nodejs npm -ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1 EXPOSE 80 EXPOSE 443 -# Remove default configuration from Nginx -RUN rm /etc/nginx/conf.d/default.conf -COPY nginx.conf /etc/nginx/conf.d/nginx.conf - -# Configure venv +# Configure CGAP User RUN useradd -ms /bin/bash cgap-admin WORKDIR /home/cgap-admin -RUN python -m venv /opt/venv -RUN chown cgap-admin /opt/venv RUN mkdir -p /home/cgap-admin/cgap-portal -RUN mkdir -p /home/cgap-admin/.poetry RUN chown cgap-admin /home/cgap-admin/cgap-portal -RUN chown cgap-admin /home/cgap-admin/.poetry -# Configure Poetry -ENV PYTHONFAULTHANDLER=1 \ - PYTHONUNBUFFERED=1 \ - PYTHONHASHSEED=random \ - PIP_NO_CACHE_DIR=off \ - PIP_DISABLE_PIP_VERSION_CHECK=on \ - PIP_DEFAULT_TIMEOUT=100 \ - POETRY_VERSION=1.1.4 +# Configure venv +ENV VIRTUAL_ENV=/opt/venv +RUN python -m venv /opt/venv +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +RUN chown cgap-admin /opt/venv # Copy to /home/cgap-admin/cgap-portal -WORKDIR /home/cgap-admin/ -RUN git clone https://github.com/dbmi-bgm/cgap-portal.git +RUN git clone $CGAP_REPO --branch $CGAP_BRANCH -# Build, configure the back-end +# Build the application WORKDIR /home/cgap-admin/cgap-portal -RUN . /opt/venv/bin/activate && pip install --upgrade pip -RUN . /opt/venv/bin/activate && make build +RUN pip install --upgrade pip +RUN make build RUN make aws-ip-ranges RUN cat /dev/urandom | head -c 256 | base64 > session-secret.b64 +COPY development.ini . USER cgap-admin #COPY entrypoint.sh /home/cgap-admin/entrypoint.sh diff --git a/deploy/docker/wsgi/Makefile b/deploy/docker/wsgi/Makefile index 22bd985179..f99d24b060 100644 --- a/deploy/docker/wsgi/Makefile +++ b/deploy/docker/wsgi/Makefile @@ -5,7 +5,7 @@ build: docker build --progress=plain -t cgap-docker:0.0.0b0 . enter: - docker run -it cgap-docker:0.0.0b0 bash + docker run -v /tmp/snovault/pgdata:/tmp/snovault/pgdata -p 5441:5441 -p 6543:6543 -it cgap-docker:0.0.0b0 bash info: @: $(info Here are some 'make' options:) diff --git a/deploy/docker/wsgi/development.ini b/deploy/docker/wsgi/development.ini index db10817702..112d4ac141 100644 --- a/deploy/docker/wsgi/development.ini +++ b/deploy/docker/wsgi/development.ini @@ -5,7 +5,7 @@ [app:app] use = config:base.ini#app -sqlalchemy.url = postgresql://postgres@localhost:5441/postgres?host=/tmp/snovault/pgdata +sqlalchemy.url = postgresql://postgres@host.docker.internal:5441/postgres?host=/tmp/snovault/pgdata blob_bucket = encoded-4dn-blobs metadata_bundles_bucket = elasticbeanstalk-fourfront-cgaplocal-dev-metadata-bundles load_test_only = true diff --git a/deploy/docker/wsgi/install_nginx.sh b/deploy/docker/wsgi/install_nginx.sh new file mode 100644 index 0000000000..98e82d6187 --- /dev/null +++ b/deploy/docker/wsgi/install_nginx.sh @@ -0,0 +1,77 @@ +# Copied from: https://github.com/nginxinc/docker-nginx/blob/594ce7a8bc26c85af88495ac94d5cd0096b306f7/mainline/buster/Dockerfile + +# Standard set up Nginx +export NGINX_VERSION=1.17.10 +export NJS_VERSION=0.3.9 +export PKG_RELEASE=1~buster + +# Securely provisions deps, installs Nginx, then removes all artifacts +set -x \ + && apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y gnupg1 ca-certificates \ + && \ + NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \ + found=''; \ + for server in \ + ha.pool.sks-keyservers.net \ + hkp://keyserver.ubuntu.com:80 \ + hkp://p80.pool.sks-keyservers.net:80 \ + pgp.mit.edu \ + ; do \ + echo "Fetching GPG key $NGINX_GPGKEY from $server"; \ + apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \ + done; \ + test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \ + apt-get remove --purge --auto-remove -y gnupg1 && rm -rf /var/lib/apt/lists/* \ + && dpkgArch="$(dpkg --print-architecture)" \ + && nginxPackages=" \ + nginx=${NGINX_VERSION}-${PKG_RELEASE} \ + nginx-module-xslt=${NGINX_VERSION}-${PKG_RELEASE} \ + nginx-module-geoip=${NGINX_VERSION}-${PKG_RELEASE} \ + nginx-module-image-filter=${NGINX_VERSION}-${PKG_RELEASE} \ + nginx-module-njs=${NGINX_VERSION}.${NJS_VERSION}-${PKG_RELEASE} \ + " \ + && case "$dpkgArch" in \ + amd64|i386) \ + echo "deb https://nginx.org/packages/mainline/debian/ buster nginx" >> /etc/apt/sources.list.d/nginx.list \ + && apt-get update \ + ;; \ + *) \ + echo "deb-src https://nginx.org/packages/mainline/debian/ buster nginx" >> /etc/apt/sources.list.d/nginx.list \ + \ + && tempDir="$(mktemp -d)" \ + && chmod 777 "$tempDir" \ + \ + && savedAptMark="$(apt-mark showmanual)" \ + \ + && apt-get update \ + && apt-get build-dep -y $nginxPackages \ + && ( \ + cd "$tempDir" \ + && DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" \ + apt-get source --compile $nginxPackages \ + ) \ + \ + && apt-mark showmanual | xargs apt-mark auto > /dev/null \ + && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; } \ + \ + && ls -lAFh "$tempDir" \ + && ( cd "$tempDir" && dpkg-scanpackages . > Packages ) \ + && grep '^Package: ' "$tempDir/Packages" \ + && echo "deb [ trusted=yes ] file://$tempDir ./" > /etc/apt/sources.list.d/temp.list \ + && apt-get -o Acquire::GzipIndexes=false update \ + ;; \ + esac \ + \ + && apt-get install --no-install-recommends --no-install-suggests -y \ + $nginxPackages \ + gettext-base \ + && apt-get remove --purge --auto-remove -y ca-certificates && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx.list \ + \ + && if [ -n "$tempDir" ]; then \ + apt-get purge -y --auto-remove \ + && rm -rf "$tempDir" /etc/apt/sources.list.d/temp.list; \ + fi + +# forward request and error logs to docker log collector +ln -sf /dev/stdout /var/log/nginx/access.log && ln -sf /dev/stderr /var/log/nginx/error.log diff --git a/deploy/docker/wsgi/nginx.conf b/deploy/docker/wsgi/nginx.conf index b1ee845007..b89b2d6117 100644 --- a/deploy/docker/wsgi/nginx.conf +++ b/deploy/docker/wsgi/nginx.conf @@ -1,52 +1,32 @@ -# nginx.conf -# default from Pyramid Docs - likely needs modification -Will - -user www-data; -worker_processes 4; -pid /var/run/nginx.pid; +# Minimal nginx proxy for development +# Used for local deployment events { - worker_connections 1024; - # multi_accept on; + worker_connections 2048; } - http { - - ## - # Basic Settings - ## - - sendfile on; - tcp_nopush on; - tcp_nodelay on; - keepalive_timeout 65; - types_hash_max_size 2048; - # server_tokens off; - - # server_names_hash_bucket_size 64; - # server_name_in_redirect off; - - include /etc/nginx/mime.types; - default_type application/octet-stream; - - ## - # Logging Settings - ## - - access_log /var/log/nginx/access.log; - error_log /var/log/nginx/error.log; - - ## - # Gzip Settings - ## - - gzip on; - gzip_disable "msie6"; - - ## - # Virtual Host Configs - ## - - include /etc/nginx/conf.d/*.conf; - include /etc/nginx/sites-enabled/*; -} \ No newline at end of file + resolver 8.8.8.8; + upstream app { + server 127.0.0.1:6543; + keepalive 10; + } + server { + listen 8000; + location / { + # Normalize duplicate slashes + if ($request ~ ^(GET|HEAD)\s([^?]*)//(.*)\sHTTP/[0-9.]+$) { + return 301 $2/$3; + } + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_pass http://app; + proxy_set_header Connection ""; + } + location ~ ^/_proxy/(.*)$ { + internal; + proxy_buffering off; + proxy_pass $1$is_args$args; + } + } +} From 098d8753d7ff300d282de50218d9071468239cd3 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Mon, 22 Feb 2021 15:02:48 -0500 Subject: [PATCH 005/120] C4-519 containerize postgres, es, main application - client fails to build? --- Makefile | 6 + deploy/docker/docker-compose.yml | 48 ++ deploy/docker/elasticsearch/Dockerfile | 9 + deploy/docker/{wsgi => local}/Dockerfile | 22 +- deploy/docker/{wsgi => local}/Makefile | 2 +- deploy/docker/{wsgi => local}/development.ini | 3 +- deploy/docker/local/entrypoint.sh | 7 + .../docker/{wsgi => local}/install_nginx.sh | 0 deploy/docker/{wsgi => local}/nginx.conf | 2 +- deploy/docker/postgres/Dockerfile | 14 + deploy/docker/postgres/postgresql.conf | 642 ++++++++++++++++++ deploy/docker/wsgi/cgap.conf | 38 -- deploy/docker/wsgi/entrypoint.sh | 4 - 13 files changed, 740 insertions(+), 57 deletions(-) create mode 100644 deploy/docker/docker-compose.yml create mode 100644 deploy/docker/elasticsearch/Dockerfile rename deploy/docker/{wsgi => local}/Dockerfile (86%) rename deploy/docker/{wsgi => local}/Makefile (66%) rename deploy/docker/{wsgi => local}/development.ini (94%) create mode 100644 deploy/docker/local/entrypoint.sh rename deploy/docker/{wsgi => local}/install_nginx.sh (100%) rename deploy/docker/{wsgi => local}/nginx.conf (96%) create mode 100644 deploy/docker/postgres/Dockerfile create mode 100644 deploy/docker/postgres/postgresql.conf delete mode 100644 deploy/docker/wsgi/cgap.conf delete mode 100644 deploy/docker/wsgi/entrypoint.sh diff --git a/Makefile b/Makefile index 49ee905275..a1ca449539 100644 --- a/Makefile +++ b/Makefile @@ -149,6 +149,12 @@ travis-test: update: # updates dependencies poetry update +build-docker: + pushd deploy/docker/ && docker-compose build && popd + +deploy-docker: + pushd deploy/docker/ && docker-compose up + help: @make info diff --git a/deploy/docker/docker-compose.yml b/deploy/docker/docker-compose.yml new file mode 100644 index 0000000000..1cf155d1bb --- /dev/null +++ b/deploy/docker/docker-compose.yml @@ -0,0 +1,48 @@ +version: "3.8" + +# use this in development.ini +# sqlalchemy.url = postgresql://postgres:postgres@db:5432/postgres +services: + + # Postgres Component + # Connect to it in development.ini with + # sqlalchemy.url = postgresql://postgres:postgres@db:5441/postgres + db: + build: ./postgres + container_name: pg1 + environment: + POSTGRES_DB: postgres + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_PORT: 5432 + + # ElasticSearch Component + es: + build: ./elasticsearch + container_name: es1 + environment: + - node.name=es01 + - cluster.name=es-docker-cluster + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + ports: + - "9200:9200" + + # Main Application Component + app: + build: ./local + container_name: cgap + command: "/home/cgap-admin/cgap-portal/entrypoint.sh" + environment: + AWS_ACCESS_KEY_ID: "$AWS_ACCESS_KEY_ID" + AWS_SECRET_ACCESS_KEY: "$AWS_SECRET_ACCESS_KEY" + ports: + - "6543:6543" + - "8000:8000" + depends_on: + - db + - es diff --git a/deploy/docker/elasticsearch/Dockerfile b/deploy/docker/elasticsearch/Dockerfile new file mode 100644 index 0000000000..951db9d476 --- /dev/null +++ b/deploy/docker/elasticsearch/Dockerfile @@ -0,0 +1,9 @@ +FROM docker.elastic.co/elasticsearch/elasticsearch:6.8.14 + +MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" + +ENV ELASTICSEARCH_VERSION="6.8.14" + +# Set port to 9200 +ENV ELASTICSEARCH_SERVICE_PORT=9200 +EXPOSE $ELASTICSEARCH_SERVICE_PORT diff --git a/deploy/docker/wsgi/Dockerfile b/deploy/docker/local/Dockerfile similarity index 86% rename from deploy/docker/wsgi/Dockerfile rename to deploy/docker/local/Dockerfile index 2b18238ed5..db724bdc2b 100644 --- a/deploy/docker/wsgi/Dockerfile +++ b/deploy/docker/local/Dockerfile @@ -18,9 +18,6 @@ ENV PYTHONFAULTHANDLER=1 \ # Install nginx (unused for local deployment as this runs as part of `make deploy1`) COPY install_nginx.sh / RUN bash /install_nginx.sh -# Remove default configuration from Nginx -RUN rm /etc/nginx/conf.d/default.conf -COPY nginx.conf /etc/nginx/conf.d/nginx.conf # Intall things needed for our system RUN apt-get update @@ -28,9 +25,6 @@ RUN apt-get install -y curl postgresql-client RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - RUN apt-get install -y ca-certificates nodejs npm -EXPOSE 80 -EXPOSE 443 - # Configure CGAP User RUN useradd -ms /bin/bash cgap-admin WORKDIR /home/cgap-admin @@ -52,13 +46,17 @@ RUN pip install --upgrade pip RUN make build RUN make aws-ip-ranges RUN cat /dev/urandom | head -c 256 | base64 > session-secret.b64 + +# Copy config files in (down here for quick debugging) +# Remove default configuration from Nginx +RUN rm /etc/nginx/conf.d/default.conf +COPY nginx.conf /etc/nginx/conf.d/nginx.conf COPY development.ini . +COPY entrypoint.sh . +RUN chmod 777 entrypoint.sh +EXPOSE 8000 USER cgap-admin -#COPY entrypoint.sh /home/cgap-admin/entrypoint.sh -#ENTRYPOINT ["/home/cgap-admin/entrypoint.sh"] -#TODO link up with nginx -#TODO expose back-end ports -#ie: make deploy1 on local machine, make deploy2/3 runs in the docker - +CMD service nginx start +ENTRYPOINT ["/home/cgap-admin/cgap-portal/entrypoint.sh"] diff --git a/deploy/docker/wsgi/Makefile b/deploy/docker/local/Makefile similarity index 66% rename from deploy/docker/wsgi/Makefile rename to deploy/docker/local/Makefile index f99d24b060..1697fbf88a 100644 --- a/deploy/docker/wsgi/Makefile +++ b/deploy/docker/local/Makefile @@ -5,7 +5,7 @@ build: docker build --progress=plain -t cgap-docker:0.0.0b0 . enter: - docker run -v /tmp/snovault/pgdata:/tmp/snovault/pgdata -p 5441:5441 -p 6543:6543 -it cgap-docker:0.0.0b0 bash + docker run -v /tmp/snovault/pgdata:/tmp/snovault/pgdata -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -p 6543:6543 -it cgap-docker:0.0.0b0 bash info: @: $(info Here are some 'make' options:) diff --git a/deploy/docker/wsgi/development.ini b/deploy/docker/local/development.ini similarity index 94% rename from deploy/docker/wsgi/development.ini rename to deploy/docker/local/development.ini index 112d4ac141..9cbd0ed8da 100644 --- a/deploy/docker/wsgi/development.ini +++ b/deploy/docker/local/development.ini @@ -5,7 +5,8 @@ [app:app] use = config:base.ini#app -sqlalchemy.url = postgresql://postgres@host.docker.internal:5441/postgres?host=/tmp/snovault/pgdata +sqlalchemy.url = postgres://postgres:postgres@db:5432/postgres +elasticsearch.server = host.docker.internal:9200 blob_bucket = encoded-4dn-blobs metadata_bundles_bucket = elasticbeanstalk-fourfront-cgaplocal-dev-metadata-bundles load_test_only = true diff --git a/deploy/docker/local/entrypoint.sh b/deploy/docker/local/entrypoint.sh new file mode 100644 index 0000000000..ec6d85c10f --- /dev/null +++ b/deploy/docker/local/entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +# load local data +poetry run load-data development.ini --app-name app + +# Start application +make deploy2 diff --git a/deploy/docker/wsgi/install_nginx.sh b/deploy/docker/local/install_nginx.sh similarity index 100% rename from deploy/docker/wsgi/install_nginx.sh rename to deploy/docker/local/install_nginx.sh diff --git a/deploy/docker/wsgi/nginx.conf b/deploy/docker/local/nginx.conf similarity index 96% rename from deploy/docker/wsgi/nginx.conf rename to deploy/docker/local/nginx.conf index b89b2d6117..7d964b0c14 100644 --- a/deploy/docker/wsgi/nginx.conf +++ b/deploy/docker/local/nginx.conf @@ -2,7 +2,7 @@ # Used for local deployment events { - worker_connections 2048; + worker_connections 2048; } http { resolver 8.8.8.8; diff --git a/deploy/docker/postgres/Dockerfile b/deploy/docker/postgres/Dockerfile new file mode 100644 index 0000000000..9fd7b73bf0 --- /dev/null +++ b/deploy/docker/postgres/Dockerfile @@ -0,0 +1,14 @@ +FROM postgres:11.11 + +MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" + +# Install some system level dependencies +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates htop vim emacs curl \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Copy over our custom conf, enable inbound connections +COPY postgresql.conf /etc/postgresql/postgresql.conf +RUN echo "host all all 0.0.0.0/0 trust" >> /etc/postgresql/pg_hba.conf + +ENV PGDATA=/var/lib/postgresql/data/pgdata diff --git a/deploy/docker/postgres/postgresql.conf b/deploy/docker/postgres/postgresql.conf new file mode 100644 index 0000000000..b01d14fc41 --- /dev/null +++ b/deploy/docker/postgres/postgresql.conf @@ -0,0 +1,642 @@ +# ----------------------------- +# PostgreSQL configuration file +# ----------------------------- +# +# This file consists of lines of the form: +# +# name = value +# +# (The "=" is optional.) Whitespace may be used. Comments are introduced with +# "#" anywhere on a line. The complete list of parameter names and allowed +# values can be found in the PostgreSQL documentation. +# +# The commented-out settings shown in this file represent the default values. +# Re-commenting a setting is NOT sufficient to revert it to the default value; +# you need to reload the server. +# +# This file is read on server startup and when the server receives a SIGHUP +# signal. If you edit the file on a running system, you have to SIGHUP the +# server for the changes to take effect, or use "pg_ctl reload". Some +# parameters, which are marked below, require a server shutdown and restart to +# take effect. +# +# Any parameter can also be given as a command-line option to the server, e.g., +# "postgres -c log_connections=on". Some parameters can be changed at run time +# with the "SET" SQL command. +# +# Memory units: kB = kilobytes Time units: ms = milliseconds +# MB = megabytes s = seconds +# GB = gigabytes min = minutes +# TB = terabytes h = hours +# d = days + + +#------------------------------------------------------------------------------ +# FILE LOCATIONS +#------------------------------------------------------------------------------ + +# The default values of these variables are driven from the -D command-line +# option or PGDATA environment variable, represented here as ConfigDir. + +#data_directory = 'ConfigDir' # use data in another directory +# (change requires restart) +#hba_file = 'ConfigDir/pg_hba.conf' # host-based authentication file +# (change requires restart) +#ident_file = 'ConfigDir/pg_ident.conf' # ident configuration file +# (change requires restart) + +# If external_pid_file is not explicitly set, no extra PID file is written. +#external_pid_file = '' # write an extra PID file +# (change requires restart) + + +#------------------------------------------------------------------------------ +# CONNECTIONS AND AUTHENTICATION +#------------------------------------------------------------------------------ + +# - Connection Settings - + +# Only necessary change from defaults +listen_addresses = '*' +# comma-separated list of addresses; +# defaults to 'localhost'; use '*' for all +# (change requires restart) +#port = 5432 # (change requires restart) +#max_connections = 100 # (change requires restart) +#superuser_reserved_connections = 3 # (change requires restart) +#unix_socket_directories = '/tmp' # comma-separated list of directories +# (change requires restart) +#unix_socket_group = '' # (change requires restart) +#unix_socket_permissions = 0777 # begin with 0 to use octal notation +# (change requires restart) +#bonjour = off # advertise server via Bonjour +# (change requires restart) +#bonjour_name = '' # defaults to the computer name +# (change requires restart) + +# - Security and Authentication - + +#authentication_timeout = 1min # 1s-600s +#ssl = off # (change requires restart) +#ssl_ciphers = 'HIGH:MEDIUM:+3DES:!aNULL' # allowed SSL ciphers +# (change requires restart) +#ssl_prefer_server_ciphers = on # (change requires restart) +#ssl_ecdh_curve = 'prime256v1' # (change requires restart) +#ssl_cert_file = 'server.crt' # (change requires restart) +#ssl_key_file = 'server.key' # (change requires restart) +#ssl_ca_file = '' # (change requires restart) +#ssl_crl_file = '' # (change requires restart) +#password_encryption = on +#db_user_namespace = off +#row_security = on + +# GSSAPI using Kerberos +#krb_server_keyfile = '' +#krb_caseins_users = off + +# - TCP Keepalives - +# see "man 7 tcp" for details + +#tcp_keepalives_idle = 0 # TCP_KEEPIDLE, in seconds; +# 0 selects the system default +#tcp_keepalives_interval = 0 # TCP_KEEPINTVL, in seconds; +# 0 selects the system default +#tcp_keepalives_count = 0 # TCP_KEEPCNT; +# 0 selects the system default + + +#------------------------------------------------------------------------------ +# RESOURCE USAGE (except WAL) +#------------------------------------------------------------------------------ + +# - Memory - + +#shared_buffers = 32MB # min 128kB +# (change requires restart) +#huge_pages = try # on, off, or try +# (change requires restart) +#temp_buffers = 8MB # min 800kB +#max_prepared_transactions = 0 # zero disables the feature +# (change requires restart) +# Caution: it is not advisable to set max_prepared_transactions nonzero unless +# you actively intend to use prepared transactions. +#work_mem = 4MB # min 64kB +#maintenance_work_mem = 64MB # min 1MB +#replacement_sort_tuples = 150000 # limits use of replacement selection sort +#autovacuum_work_mem = -1 # min 1MB, or -1 to use maintenance_work_mem +#max_stack_depth = 2MB # min 100kB +#dynamic_shared_memory_type = posix # the default is the first option +# supported by the operating system: +# posix +# sysv +# windows +# mmap +# use none to disable dynamic shared memory + +# - Disk - + +#temp_file_limit = -1 # limits per-process temp file space +# in kB, or -1 for no limit + +# - Kernel Resource Usage - + +#max_files_per_process = 1000 # min 25 +# (change requires restart) +#shared_preload_libraries = '' # (change requires restart) + +# - Cost-Based Vacuum Delay - + +#vacuum_cost_delay = 0 # 0-100 milliseconds +#vacuum_cost_page_hit = 1 # 0-10000 credits +#vacuum_cost_page_miss = 10 # 0-10000 credits +#vacuum_cost_page_dirty = 20 # 0-10000 credits +#vacuum_cost_limit = 200 # 1-10000 credits + +# - Background Writer - + +#bgwriter_delay = 200ms # 10-10000ms between rounds +#bgwriter_lru_maxpages = 100 # 0-1000 max buffers written/round +#bgwriter_lru_multiplier = 2.0 # 0-10.0 multiplier on buffers scanned/round +#bgwriter_flush_after = 0 # measured in pages, 0 disables + +# - Asynchronous Behavior - + +#effective_io_concurrency = 1 # 1-1000; 0 disables prefetching +#max_worker_processes = 8 # (change requires restart) +#max_parallel_workers_per_gather = 0 # taken from max_worker_processes +#old_snapshot_threshold = -1 # 1min-60d; -1 disables; 0 is immediate +# (change requires restart) +#backend_flush_after = 0 # measured in pages, 0 disables + + +#------------------------------------------------------------------------------ +# WRITE AHEAD LOG +#------------------------------------------------------------------------------ + +# - Settings - + +#wal_level = minimal # minimal, replica, or logical +# (change requires restart) +#fsync = on # flush data to disk for crash safety +# (turning this off can cause +# unrecoverable data corruption) +#synchronous_commit = on # synchronization level; +# off, local, remote_write, remote_apply, or on +#wal_sync_method = fsync # the default is the first option +# supported by the operating system: +# open_datasync +# fdatasync (default on Linux) +# fsync +# fsync_writethrough +# open_sync +#full_page_writes = on # recover from partial page writes +#wal_compression = off # enable compression of full-page writes +#wal_log_hints = off # also do full page writes of non-critical updates +# (change requires restart) +#wal_buffers = -1 # min 32kB, -1 sets based on shared_buffers +# (change requires restart) +#wal_writer_delay = 200ms # 1-10000 milliseconds +#wal_writer_flush_after = 1MB # measured in pages, 0 disables + +#commit_delay = 0 # range 0-100000, in microseconds +#commit_siblings = 5 # range 1-1000 + +# - Checkpoints - + +#checkpoint_timeout = 5min # range 30s-1d +#max_wal_size = 1GB +#min_wal_size = 80MB +#checkpoint_completion_target = 0.5 # checkpoint target duration, 0.0 - 1.0 +#checkpoint_flush_after = 0 # measured in pages, 0 disables +#checkpoint_warning = 30s # 0 disables + +# - Archiving - + +#archive_mode = off # enables archiving; off, on, or always +# (change requires restart) +#archive_command = '' # command to use to archive a logfile segment +# placeholders: %p = path of file to archive +# %f = file name only +# e.g. 'test ! -f /mnt/server/archivedir/%f && cp %p /mnt/server/archivedir/%f' +#archive_timeout = 0 # force a logfile segment switch after this +# number of seconds; 0 disables + + +#------------------------------------------------------------------------------ +# REPLICATION +#------------------------------------------------------------------------------ + +# - Sending Server(s) - + +# Set these on the master and on any standby that will send replication data. + +#max_wal_senders = 0 # max number of walsender processes +# (change requires restart) +#wal_keep_segments = 0 # in logfile segments, 16MB each; 0 disables +#wal_sender_timeout = 60s # in milliseconds; 0 disables + +#max_replication_slots = 0 # max number of replication slots +# (change requires restart) +#track_commit_timestamp = off # collect timestamp of transaction commit +# (change requires restart) + +# - Master Server - + +# These settings are ignored on a standby server. + +#synchronous_standby_names = '' # standby servers that provide sync rep +# number of sync standbys and comma-separated list of application_name +# from standby(s); '*' = all +#vacuum_defer_cleanup_age = 0 # number of xacts by which cleanup is delayed + +# - Standby Servers - + +# These settings are ignored on a master server. + +#hot_standby = off # "on" allows queries during recovery +# (change requires restart) +#max_standby_archive_delay = 30s # max delay before canceling queries +# when reading WAL from archive; +# -1 allows indefinite delay +#max_standby_streaming_delay = 30s # max delay before canceling queries +# when reading streaming WAL; +# -1 allows indefinite delay +#wal_receiver_status_interval = 10s # send replies at least this often +# 0 disables +#hot_standby_feedback = off # send info from standby to prevent +# query conflicts +#wal_receiver_timeout = 60s # time that receiver waits for +# communication from master +# in milliseconds; 0 disables +#wal_retrieve_retry_interval = 5s # time to wait before retrying to +# retrieve WAL after a failed attempt + + +#------------------------------------------------------------------------------ +# QUERY TUNING +#------------------------------------------------------------------------------ + +# - Planner Method Configuration - + +#enable_bitmapscan = on +#enable_hashagg = on +#enable_hashjoin = on +#enable_indexscan = on +#enable_indexonlyscan = on +#enable_material = on +#enable_mergejoin = on +#enable_nestloop = on +#enable_seqscan = on +#enable_sort = on +#enable_tidscan = on + +# - Planner Cost Constants - + +#seq_page_cost = 1.0 # measured on an arbitrary scale +#random_page_cost = 4.0 # same scale as above +#cpu_tuple_cost = 0.01 # same scale as above +#cpu_index_tuple_cost = 0.005 # same scale as above +#cpu_operator_cost = 0.0025 # same scale as above +#parallel_tuple_cost = 0.1 # same scale as above +#parallel_setup_cost = 1000.0 # same scale as above +#min_parallel_relation_size = 8MB +#effective_cache_size = 4GB + +# - Genetic Query Optimizer - + +#geqo = on +#geqo_threshold = 12 +#geqo_effort = 5 # range 1-10 +#geqo_pool_size = 0 # selects default based on effort +#geqo_generations = 0 # selects default based on effort +#geqo_selection_bias = 2.0 # range 1.5-2.0 +#geqo_seed = 0.0 # range 0.0-1.0 + +# - Other Planner Options - + +#default_statistics_target = 100 # range 1-10000 +#constraint_exclusion = partition # on, off, or partition +#cursor_tuple_fraction = 0.1 # range 0.0-1.0 +#from_collapse_limit = 8 +#join_collapse_limit = 8 # 1 disables collapsing of explicit +# JOIN clauses +#force_parallel_mode = off + + +#------------------------------------------------------------------------------ +# ERROR REPORTING AND LOGGING +#------------------------------------------------------------------------------ + +# - Where to Log - + +#log_destination = 'stderr' # Valid values are combinations of +# stderr, csvlog, syslog, and eventlog, +# depending on platform. csvlog +# requires logging_collector to be on. + +# This is used when logging to stderr: +#logging_collector = off # Enable capturing of stderr and csvlog +# into log files. Required to be on for +# csvlogs. +# (change requires restart) + +# These are only used if logging_collector is on: +#log_directory = 'pg_log' # directory where log files are written, +# can be absolute or relative to PGDATA +#log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log' # log file name pattern, +# can include strftime() escapes +#log_file_mode = 0600 # creation mode for log files, +# begin with 0 to use octal notation +#log_truncate_on_rotation = off # If on, an existing log file with the +# same name as the new log file will be +# truncated rather than appended to. +# But such truncation only occurs on +# time-driven rotation, not on restarts +# or size-driven rotation. Default is +# off, meaning append to existing files +# in all cases. +#log_rotation_age = 1d # Automatic rotation of logfiles will +# happen after that time. 0 disables. +#log_rotation_size = 10MB # Automatic rotation of logfiles will +# happen after that much log output. +# 0 disables. + +# These are relevant when logging to syslog: +#syslog_facility = 'LOCAL0' +#syslog_ident = 'postgres' +#syslog_sequence_numbers = on +#syslog_split_messages = on + +# This is only relevant when logging to eventlog (win32): +#event_source = 'PostgreSQL' + +# - When to Log - + +#client_min_messages = notice # values in order of decreasing detail: +# debug5 +# debug4 +# debug3 +# debug2 +# debug1 +# log +# notice +# warning +# error + +#log_min_messages = warning # values in order of decreasing detail: +# debug5 +# debug4 +# debug3 +# debug2 +# debug1 +# info +# notice +# warning +# error +# log +# fatal +# panic + +#log_min_error_statement = error # values in order of decreasing detail: +# debug5 +# debug4 +# debug3 +# debug2 +# debug1 +# info +# notice +# warning +# error +# log +# fatal +# panic (effectively off) + +#log_min_duration_statement = -1 # -1 is disabled, 0 logs all statements +# and their durations, > 0 logs only +# statements running at least this number +# of milliseconds + + +# - What to Log - + +#debug_print_parse = off +#debug_print_rewritten = off +#debug_print_plan = off +#debug_pretty_print = on +#log_checkpoints = off +#log_connections = off +#log_disconnections = off +#log_duration = off +#log_error_verbosity = default # terse, default, or verbose messages +#log_hostname = off +#log_line_prefix = '' # special values: +# %a = application name +# %u = user name +# %d = database name +# %r = remote host and port +# %h = remote host +# %p = process ID +# %t = timestamp without milliseconds +# %m = timestamp with milliseconds +# %n = timestamp with milliseconds (as a Unix epoch) +# %i = command tag +# %e = SQL state +# %c = session ID +# %l = session line number +# %s = session start timestamp +# %v = virtual transaction ID +# %x = transaction ID (0 if none) +# %q = stop here in non-session +# processes +# %% = '%' +# e.g. '<%u%%%d> ' +#log_lock_waits = off # log lock waits >= deadlock_timeout +#log_statement = 'none' # none, ddl, mod, all +#log_replication_commands = off +#log_temp_files = -1 # log temporary files equal or larger +# than the specified size in kilobytes; +# -1 disables, 0 logs all temp files +#log_timezone = 'GMT' + + +# - Process Title - + +#cluster_name = '' # added to process titles if nonempty +# (change requires restart) +#update_process_title = on + + +#------------------------------------------------------------------------------ +# RUNTIME STATISTICS +#------------------------------------------------------------------------------ + +# - Query/Index Statistics Collector - + +#track_activities = on +#track_counts = on +#track_io_timing = off +#track_functions = none # none, pl, all +#track_activity_query_size = 1024 # (change requires restart) +#stats_temp_directory = 'pg_stat_tmp' + + +# - Statistics Monitoring - + +#log_parser_stats = off +#log_planner_stats = off +#log_executor_stats = off +#log_statement_stats = off + + +#------------------------------------------------------------------------------ +# AUTOVACUUM PARAMETERS +#------------------------------------------------------------------------------ + +#autovacuum = on # Enable autovacuum subprocess? 'on' +# requires track_counts to also be on. +#log_autovacuum_min_duration = -1 # -1 disables, 0 logs all actions and +# their durations, > 0 logs only +# actions running at least this number +# of milliseconds. +#autovacuum_max_workers = 3 # max number of autovacuum subprocesses +# (change requires restart) +#autovacuum_naptime = 1min # time between autovacuum runs +#autovacuum_vacuum_threshold = 50 # min number of row updates before +# vacuum +#autovacuum_analyze_threshold = 50 # min number of row updates before +# analyze +#autovacuum_vacuum_scale_factor = 0.2 # fraction of table size before vacuum +#autovacuum_analyze_scale_factor = 0.1 # fraction of table size before analyze +#autovacuum_freeze_max_age = 200000000 # maximum XID age before forced vacuum +# (change requires restart) +#autovacuum_multixact_freeze_max_age = 400000000 # maximum multixact age +# before forced vacuum +# (change requires restart) +#autovacuum_vacuum_cost_delay = 20ms # default vacuum cost delay for +# autovacuum, in milliseconds; +# -1 means use vacuum_cost_delay +#autovacuum_vacuum_cost_limit = -1 # default vacuum cost limit for +# autovacuum, -1 means use +# vacuum_cost_limit + + +#------------------------------------------------------------------------------ +# CLIENT CONNECTION DEFAULTS +#------------------------------------------------------------------------------ + +# - Statement Behavior - + +#search_path = '"$user", public' # schema names +#default_tablespace = '' # a tablespace name, '' uses the default +#temp_tablespaces = '' # a list of tablespace names, '' uses +# only default tablespace +#check_function_bodies = on +#default_transaction_isolation = 'read committed' +#default_transaction_read_only = off +#default_transaction_deferrable = off +#session_replication_role = 'origin' +#statement_timeout = 0 # in milliseconds, 0 is disabled +#lock_timeout = 0 # in milliseconds, 0 is disabled +#idle_in_transaction_session_timeout = 0 # in milliseconds, 0 is disabled +#vacuum_freeze_min_age = 50000000 +#vacuum_freeze_table_age = 150000000 +#vacuum_multixact_freeze_min_age = 5000000 +#vacuum_multixact_freeze_table_age = 150000000 +#bytea_output = 'hex' # hex, escape +#xmlbinary = 'base64' +#xmloption = 'content' +#gin_fuzzy_search_limit = 0 +#gin_pending_list_limit = 4MB + +# - Locale and Formatting - + +#datestyle = 'iso, mdy' +#intervalstyle = 'postgres' +#timezone = 'GMT' +#timezone_abbreviations = 'Default' # Select the set of available time zone +# abbreviations. Currently, there are +# Default +# Australia (historical usage) +# India +# You can create your own file in +# share/timezonesets/. +#extra_float_digits = 0 # min -15, max 3 +#client_encoding = sql_ascii # actually, defaults to database +# encoding + +# These settings are initialized by initdb, but they can be changed. +#lc_messages = 'C' # locale for system error message +# strings +#lc_monetary = 'C' # locale for monetary formatting +#lc_numeric = 'C' # locale for number formatting +#lc_time = 'C' # locale for time formatting + +# default configuration for text search +#default_text_search_config = 'pg_catalog.simple' + +# - Other Defaults - + +#dynamic_library_path = '$libdir' +#local_preload_libraries = '' +#session_preload_libraries = '' + + +#------------------------------------------------------------------------------ +# LOCK MANAGEMENT +#------------------------------------------------------------------------------ + +#deadlock_timeout = 1s +#max_locks_per_transaction = 64 # min 10 +# (change requires restart) +#max_pred_locks_per_transaction = 64 # min 10 +# (change requires restart) + + +#------------------------------------------------------------------------------ +# VERSION/PLATFORM COMPATIBILITY +#------------------------------------------------------------------------------ + +# - Previous PostgreSQL Versions - + +#array_nulls = on +#backslash_quote = safe_encoding # on, off, or safe_encoding +#default_with_oids = off +#escape_string_warning = on +#lo_compat_privileges = off +#operator_precedence_warning = off +#quote_all_identifiers = off +#sql_inheritance = on +#standard_conforming_strings = on +#synchronize_seqscans = on + +# - Other Platforms and Clients - + +#transform_null_equals = off + + +#------------------------------------------------------------------------------ +# ERROR HANDLING +#------------------------------------------------------------------------------ + +#exit_on_error = off # terminate session on any error? +#restart_after_crash = on # reinitialize after backend crash? + + +#------------------------------------------------------------------------------ +# CONFIG FILE INCLUDES +#------------------------------------------------------------------------------ + +# These options allow settings to be loaded from files other than the +# default postgresql.conf. + +#include_dir = 'conf.d' # include files ending in '.conf' from +# directory 'conf.d' +#include_if_exists = 'exists.conf' # include file only if it exists +#include = 'special.conf' # include file + + +#------------------------------------------------------------------------------ +# CUSTOMIZED OPTIONS +#------------------------------------------------------------------------------ + +# Add settings for extensions here \ No newline at end of file diff --git a/deploy/docker/wsgi/cgap.conf b/deploy/docker/wsgi/cgap.conf deleted file mode 100644 index 4086f33bf1..0000000000 --- a/deploy/docker/wsgi/cgap.conf +++ /dev/null @@ -1,38 +0,0 @@ -# cgap.conf - -upstream cgap-portal { - server 127.0.0.1:6543; # one server for now -} - -server { - listen 80; - - # optional ssl configuration - - # listen 443 ssl; - # ssl_certificate /path/to/ssl/pem_file; - # ssl_certificate_key /path/to/ssl/certificate_key; - - # end of optional ssl configuration - - server_name cgap-portal.com; - - access_log /home/cgap/env/access.log; - - location / { - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Host $host:$server_port; - proxy_set_header X-Forwarded-Port $server_port; - - client_max_body_size 10m; - client_body_buffer_size 128k; - proxy_connect_timeout 60s; - proxy_send_timeout 90s; - proxy_read_timeout 90s; - proxy_buffering off; - proxy_temp_file_write_size 64k; - proxy_pass http://myapp-site; - proxy_redirect off; - } -} \ No newline at end of file diff --git a/deploy/docker/wsgi/entrypoint.sh b/deploy/docker/wsgi/entrypoint.sh deleted file mode 100644 index 5eba02e837..0000000000 --- a/deploy/docker/wsgi/entrypoint.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -make deploy2 -exec "$@" From 3da1159a2588158725070aa672e446094d96d48c Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 23 Feb 2021 09:40:43 -0500 Subject: [PATCH 006/120] small changes, disable parallel terser build as it breaks docker --- deploy/docker/local/Dockerfile | 7 ++++++- deploy/docker/local/entrypoint.sh | 3 +++ webpack.config.js | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/deploy/docker/local/Dockerfile b/deploy/docker/local/Dockerfile index db724bdc2b..ff9de61847 100644 --- a/deploy/docker/local/Dockerfile +++ b/deploy/docker/local/Dockerfile @@ -43,7 +43,12 @@ RUN git clone $CGAP_REPO --branch $CGAP_BRANCH # Build the application WORKDIR /home/cgap-admin/cgap-portal RUN pip install --upgrade pip -RUN make build +RUN pip install poetry==1.1.4 wheel==0.29.0 +RUN poetry install +RUN python setup_eb.py develop +RUN npm install --no-fund --no-progress --python=/opt/venv/bin/python +RUN npm run build +RUN npm run build-scss RUN make aws-ip-ranges RUN cat /dev/urandom | head -c 256 | base64 > session-secret.b64 diff --git a/deploy/docker/local/entrypoint.sh b/deploy/docker/local/entrypoint.sh index ec6d85c10f..d6d1896e70 100644 --- a/deploy/docker/local/entrypoint.sh +++ b/deploy/docker/local/entrypoint.sh @@ -1,5 +1,8 @@ #!/bin/sh +# Enter venv +export PATH="/opt/venv/bin:$PATH" + # load local data poetry run load-data development.ini --app-name app diff --git a/webpack.config.js b/webpack.config.js index 5aea079cb6..57f2e32fd1 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -79,7 +79,7 @@ const optimization = { // sourceMap: true //}) new TerserPlugin({ - parallel: true, + parallel: false, // XXX: this option causes docker build to fail. - Will sourceMap: true, terserOptions:{ compress: true, From 5751335ffd8f68e0a5421c8bbfdce85162d8c46e Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 23 Feb 2021 11:57:53 -0500 Subject: [PATCH 007/120] push working configuration, nginx still not working though --- deploy/docker/docker-compose.yml | 37 +- deploy/docker/local/Dockerfile | 7 +- deploy/docker/local/development.ini | 9 +- deploy/docker/local/entrypoint.sh | 10 +- deploy/docker/local/nginx.conf | 2 +- .../master-inserts/higlass_view_config.json | 860 ------------------ 6 files changed, 41 insertions(+), 884 deletions(-) delete mode 100644 src/encoded/tests/data/master-inserts/higlass_view_config.json diff --git a/deploy/docker/docker-compose.yml b/deploy/docker/docker-compose.yml index 1cf155d1bb..60e80a2a5f 100644 --- a/deploy/docker/docker-compose.yml +++ b/deploy/docker/docker-compose.yml @@ -17,20 +17,23 @@ services: POSTGRES_PORT: 5432 # ElasticSearch Component - es: - build: ./elasticsearch - container_name: es1 - environment: - - node.name=es01 - - cluster.name=es-docker-cluster - - bootstrap.memory_lock=true - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - ulimits: - memlock: - soft: -1 - hard: -1 - ports: - - "9200:9200" + # Disabled for now as too compute intensive + # To connect use: + # elasticsearch.server = es:9200 +# es: +# build: ./elasticsearch +# container_name: es1 +# environment: +# - node.name=es01 +# - cluster.name=es-docker-cluster +# - bootstrap.memory_lock=true +# - "ES_JAVA_OPTS=-Xms512m -Xmx512m" +# ulimits: +# memlock: +# soft: -1 +# hard: -1 +# ports: +# - "9200:9200" # Main Application Component app: @@ -41,8 +44,8 @@ services: AWS_ACCESS_KEY_ID: "$AWS_ACCESS_KEY_ID" AWS_SECRET_ACCESS_KEY: "$AWS_SECRET_ACCESS_KEY" ports: - - "6543:6543" - - "8000:8000" + - "6543:6543" # access app directly + - "8000:8000" # nginx proxy port (not functional currently) depends_on: - db - - es + #- es diff --git a/deploy/docker/local/Dockerfile b/deploy/docker/local/Dockerfile index ff9de61847..4d11909966 100644 --- a/deploy/docker/local/Dockerfile +++ b/deploy/docker/local/Dockerfile @@ -2,6 +2,10 @@ FROM python:3.6-buster MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" +# Build Arguments +ARG CGAP_ENV_NAME +ENV CGAP_ENV_NAME=${CGAP_ENV_NAME:-"cgap-docker-will-test"} + # Configure (global) Env ENV CGAP_REPO=https://github.com/dbmi-bgm/cgap-portal.git ENV CGAP_BRANCH=c4_519 @@ -46,6 +50,7 @@ RUN pip install --upgrade pip RUN pip install poetry==1.1.4 wheel==0.29.0 RUN poetry install RUN python setup_eb.py develop +RUN make fix-dist-info RUN npm install --no-fund --no-progress --python=/opt/venv/bin/python RUN npm run build RUN npm run build-scss @@ -55,7 +60,7 @@ RUN cat /dev/urandom | head -c 256 | base64 > session-secret.b64 # Copy config files in (down here for quick debugging) # Remove default configuration from Nginx RUN rm /etc/nginx/conf.d/default.conf -COPY nginx.conf /etc/nginx/conf.d/nginx.conf +COPY nginx.conf /etc/nginx/conf.d/default.conf COPY development.ini . COPY entrypoint.sh . RUN chmod 777 entrypoint.sh diff --git a/deploy/docker/local/development.ini b/deploy/docker/local/development.ini index 9cbd0ed8da..e524b17360 100644 --- a/deploy/docker/local/development.ini +++ b/deploy/docker/local/development.ini @@ -5,8 +5,11 @@ [app:app] use = config:base.ini#app +session.secret = %(here)s/session-secret.b64 +env.name = cgap-docker-will-test sqlalchemy.url = postgres://postgres:postgres@db:5432/postgres -elasticsearch.server = host.docker.internal:9200 +elasticsearch.server = search-cgap-testing-6-8-vo4mdkmkshvmyddc65ux7dtaou.us-east-1.es.amazonaws.com:443 +elasticsearch.aws_auth = true blob_bucket = encoded-4dn-blobs metadata_bundles_bucket = elasticbeanstalk-fourfront-cgaplocal-dev-metadata-bundles load_test_only = true @@ -15,7 +18,7 @@ testing = true postgresql.statement_timeout = 20 mpindexer = true indexer = true -elasticsearch.aws_auth = false +indexer.namespace = cgap-docker-will-test pyramid.reload_templates = true pyramid.debug_authorization = false pyramid.debug_notfound = true @@ -23,7 +26,7 @@ pyramid.debug_routematch = false pyramid.default_locale_name = en # this line determines which load function is used in load_data # most deployments use: "load_test_data = encoded.loadxl:load_test_data" -load_test_data = encoded.loadxl:load_local_data +load_test_data = encoded.loadxl:load_prod_data encoded_version = 100.200.300 snovault_version = 200.300.400 utils_version = 300.400.500 diff --git a/deploy/docker/local/entrypoint.sh b/deploy/docker/local/entrypoint.sh index d6d1896e70..3a4fd4ba31 100644 --- a/deploy/docker/local/entrypoint.sh +++ b/deploy/docker/local/entrypoint.sh @@ -3,8 +3,14 @@ # Enter venv export PATH="/opt/venv/bin:$PATH" -# load local data -poetry run load-data development.ini --app-name app +# Clear db/es since this is the local entry point +poetry run clear-db-es-contents development.ini --app-name app --env $CGAP_ENV_NAME + +# Create mapping +poetry run create-mapping-on-deploy development.ini --app-name app + +# Load Data (based on development.ini, for now just master-inserts) +poetry run load-data development.ini --app-name app --prod # Start application make deploy2 diff --git a/deploy/docker/local/nginx.conf b/deploy/docker/local/nginx.conf index 7d964b0c14..2a3203657c 100644 --- a/deploy/docker/local/nginx.conf +++ b/deploy/docker/local/nginx.conf @@ -7,7 +7,7 @@ events { http { resolver 8.8.8.8; upstream app { - server 127.0.0.1:6543; + server 0.0.0.0:6543; keepalive 10; } server { diff --git a/src/encoded/tests/data/master-inserts/higlass_view_config.json b/src/encoded/tests/data/master-inserts/higlass_view_config.json deleted file mode 100644 index d16dfd052a..0000000000 --- a/src/encoded/tests/data/master-inserts/higlass_view_config.json +++ /dev/null @@ -1,860 +0,0 @@ -[ - { - "viewconfig":{ - "editable":true, - "zoomFixed":false, - "exportViewUrl":"/api/v1/viewconfs", - "trackSourceServers": ["https://cgap-higlass.com/api/v1"], - "views":[ - { - "autocompleteSource":"/api/v1/suggest/?d=OHJakQICQD6gTD7skx4EWA&", - "genomePositionSearchBox":{ - "autocompleteServer":"https://cgap-higlass.com/api/v1", - "autocompleteId":"P0PLbQMwTYGy-5uPIQid7A", - "chromInfoServer":"https://cgap-higlass.com/api/v1", - "chromInfoId":"hg38", - "visible":true - }, - "chromInfoPath":"//s3.amazonaws.com/pkerp/data/hg38/chromSizes.tsv", - "tracks": { - "top": [ - { - "type": "combined", - "uid": "FkGY-Yv9T8avNXljXklukw", - "height": 55, - "width": 568, - "contents": [ - { - "filetype": "beddb", - "server": "https://cgap-higlass.com/api/v1", - "tilesetUid": "gene_annotation_hg38", - "uid": "FocNIVsfRVWMRzfmHfncsQ", - "type": "gene-annotations", - "options": { - "fontSize": 10, - "labelColor": "black", - "labelBackgroundColor": "#ffffff", - "labelPosition": "hidden", - "labelLeftMargin": 0, - "labelRightMargin": 0, - "labelTopMargin": 0, - "labelBottomMargin": 0, - "minHeight": 24, - "plusStrandColor": "#8a8ccf", - "minusStrandColor": "#e8727a", - "trackBorderWidth": 0, - "trackBorderColor": "black", - "showMousePosition": false, - "mousePositionColor": "#000000", - "geneAnnotationHeight": 12, - "geneLabelPosition": "outside", - "geneStrandSpacing": 2, - "name": "Gene Annotations (hg38)" - }, - "width": 568, - "height": 55 - }, - { - "uid": "d6KeVfkNSmq_YNj_rWJgBA", - "type": "viewport-projection-horizontal", - "fromViewUid": "ab", - "options": { - "projectionFillColor": "#777", - "projectionStrokeColor": "#777", - "projectionFillOpacity": 0.3, - "projectionStrokeOpacity": 0.7, - "strokeWidth": 1 - }, - "width": 568, - "height": 55 - } - ], - "options": {} - } - ], - "left": [], - "center": [], - "right": [], - "bottom": [], - "whole": [], - "gallery": [] - }, - "initialXDomain": [ - 594954043.6728096, - 2673883060.323411 - ], - "initialYDomain": [ - -2681117714.2684245, - -2681117686.505896 - ], - "layout": { - "w": 12, - "h": 2, - "x": 0, - "y": 0 - }, - "uid": "aa" - }, - { - "uid":"ab", - "initialXDomain":[ - 594954043.6728096, - 2673883060.323411 - ], - "genomePositionSearchBox":{ - "autocompleteServer":"https://cgap-higlass.com/api/v1", - "autocompleteId":"P0PLbQMwTYGy-5uPIQid7A", - "chromInfoServer":"https://cgap-higlass.com/api/v1", - "chromInfoId":"hg38", - "visible":true - }, - "chromInfoPath":"//s3.amazonaws.com/pkerp/data/hg38/chromSizes.tsv", - "tracks":{ - "top":[ - { - "filetype":"chromsizes-tsv", - "server":"https://cgap-higlass.com/api/v1", - "tilesetUid":"chromsizes_hg38", - "uid":"AdlJsUYFRzuJRZyYeKDX2A", - "type":"chromosome-labels", - "options":{ - "color":"#808080", - "stroke":"#ffffff", - "fontSize":12, - "fontIsLeftAligned":false, - "showMousePosition":false, - "mousePositionColor":"#000000" - }, - "width":811, - "height":30 - }, - { - "uid":"fastaex", - "type":"horizontal-sequence", - "server": "https://cgap-higlass.com/api/v1", - "data":{ - "type":"fasta", - "fastaUrl":"https://aveit.s3.amazonaws.com/higlass/data/sequence/hg38.fa", - "faiUrl":"https://aveit.s3.amazonaws.com/higlass/data/sequence/hg38.fa.fai", - "chromSizesUrl":"https://aveit.s3.amazonaws.com/higlass/data/sequence/hg38.chrom.sizes" - }, - "options":{ - "colorAggregationMode":"none", - "labelPosition":"topLeft", - "labelColor":"black", - "labelTextOpacity":0.4, - "valueScaling":"linear", - "trackBorderWidth":0, - "trackBorderColor":"white", - "name":"hg38", - "backgroundColor":"white", - "barBorder":true, - "barBorderColor":"white", - "sortLargestOnTop":true, - "extendedPreloading":false, - "colorScale": [ - "#08519c", - "#6baed6", - "#993404", - "#fe9929", - "#808080", - "#DCDCDC" - ] - }, - "width":768, - "height":25 - }, - { - "uid": "emptytrack_transcripts", - "type": "empty", - "options": {}, - "width": 568, - "height": 10 - }, - { - "uid": "texttrack_transcripts", - "type": "text", - "server": "https://cgap-higlass.com/api/v1", - "options": { - "backgroundColor": "#ededed", - "textColor": "#333333", - "fontSize": 11, - "fontFamily": "Arial", - "fontWeight": "normal", - "offsetY": 4, - "align": "left", - "text": "Transcripts" - }, - "width": 568, - "height": 20 - }, - { - "server": "https://cgap-higlass.com/api/v1", - "tilesetUid": "transcripts_hg38", - "uid": "transcript_annotation", - "type": "horizontal-transcripts", - "options": { - "fontSize": 9, - "labelFontColor": "#222222", - "labelBackgroundPlusStrandColor": "#e9e9e9", - "labelBackgroundMinusStrandColor": "#e9e9e9", - "minHeight": 24, - "plusStrandColor": "#bdbfff", - "minusStrandColor": "#fabec2", - "utrColor": "#C0EAAF", - "mousePositionColor": "#000000", - "transcriptHeight": 12, - "transcriptSpacing": 2, - "name": "Gene transcripts", - "fontFamily": "Helvetica", - "maxTexts": 100, - "showToggleTranscriptsButton": true, - "trackHeightAdjustment": "automatic", - "startCollapsed": false, - "sequenceData": { - "type": "fasta", - "fastaUrl": "https://aveit.s3.amazonaws.com/higlass/data/sequence/hg38.fa", - "faiUrl": "https://aveit.s3.amazonaws.com/higlass/data/sequence/hg38.fa.fai", - "chromSizesUrl": "https://aveit.s3.amazonaws.com/higlass/data/sequence/hg38.chrom.sizes" - } - }, - "width": 768, - "height": 134 - }, - { - "uid": "emptytrack_clinvar", - "type": "empty", - "options": {}, - "width": 568, - "height": 10 - }, - { - "uid": "texttrack_clinvar", - "type": "text", - "server": "https://cgap-higlass.com/api/v1", - "options": { - "backgroundColor": "#ededed", - "textColor": "#333333", - "fontSize": 11, - "fontFamily": "Arial", - "fontWeight": "normal", - "offsetY": 4, - "align": "left", - "text": "ClinVar variants" - }, - "width": 568, - "height": 20 - }, - { - "type": "horizontal-clinvar", - "height": 110, - "tilesetUid": "clinvar_20200824_hg38", - "server": "https://cgap-higlass.com/api/v1", - "uid": "clinvar_20200824_v5", - "options": { - "name": "Clinvar" - } - }, - { - "uid": "emptytrack_orthologs", - "type": "empty", - "options": {}, - "width": 568, - "height": 10 - }, - { - "uid": "texttrack_orthologs", - "type": "text", - "server": "https://cgap-higlass.com/api/v1", - "options": { - "backgroundColor": "#ededed", - "textColor": "#333333", - "fontSize": 11, - "fontFamily": "Arial", - "fontWeight": "normal", - "offsetY": 4, - "align": "left", - "text": "Orthologs" - }, - "width": 568, - "height": 20 - }, - { - "uid": "emptytrack_orthologs2", - "type": "empty", - "options": {}, - "width": 568, - "height": 7 - }, - { - "type": "horizontal-orthologs", - "height": 100, - "tilesetUid": "orthologs_transcripts_hg38", - "server": "https://cgap-higlass.com/api/v1", - "uid": "orthologs", - "options": { - "rowHeight": 11, - "name": "Orthologs", - "aminoAcidColor": "#333333", - "aminoAcidColorNoMatch": "#b0b0b0", - "fontSize": 10, - "fontFamily": "Arial", - "gapsColor": "#eb9c00", - "labelTextColor": "#888888", - "minusStrandColor1": "#ffe0e2", - "minusStrandColor2": "#fff0f1", - "minusStrandColorZoomedOut": "#fabec2", - "plusStrandColor1": "#ebebff", - "plusStrandColor2": "#dedeff", - "plusStrandColorZoomedOut": "#bdbfff", - "rowSpacing": 2, - "species": [ - "human", - "macaca_mulatta", - "mouse", - "dog", - "elephant", - "chicken", - "zebrafish" - ] - } - }, - { - "uid": "emptytrack_gnomad", - "type": "empty", - "options": {}, - "width": 568, - "height": 10 - }, - { - "uid": "texttrack_gnomad", - "type": "text", - "server": "https://cgap-higlass.com/api/v1", - "options": { - "backgroundColor": "#ededed", - "textColor": "#333333", - "fontSize": 11, - "fontFamily": "Arial", - "fontWeight": "normal", - "offsetY": 4, - "align": "left", - "text": "GnomAD (allele frequencies)" - }, - "width": 568, - "height": 20 - }, - { - "uid": "emptytrack_gnomad2", - "type": "empty", - "options": {}, - "width": 568, - "height": 7 - }, - { - "filetype": "bigwig", - "server": "https://cgap-higlass.com/api/v1", - "tilesetUid": "gnomad_coverage", - "uid": "dGGE208qQNmBcvlBnfpFuA", - "type": "bar", - "options": { - "align": "bottom", - "labelPosition": "topLeft", - "labelLeftMargin": 0, - "labelRightMargin": 0, - "labelTopMargin": 0, - "labelBottomMargin": 0, - "labelShowResolution": false, - "labelShowAssembly": false, - "axisLabelFormatting": "scientific", - "axisPositionHorizontal": "right", - "barFillColor": "grey", - "valueScaling": "linear", - "trackBorderWidth": 0, - "trackBorderColor": "black", - "labelTextOpacity": 0.4, - "barOpacity": 1, - "valueScaleMin": 0, - "valueScaleMax": 60, - "name": "GnomAd - median coverage" - }, - "width": 768, - "height": 45 - }, - { - "uid": "emptytrack_gnomad3", - "type": "empty", - "options": {}, - "width": 568, - "height": 7 - }, - { - "uid": "gnomad", - "type": "gnomad", - "options": { - "colorScale": [ - [0.3, 0.3, 0.3, 0.6], - [0.6, 0.6, 0.0, 0.7], - [1, 0.0, 0.0, 0.6] - ], - "showMousePosition": false, - "workerScriptLocation": "/static/build/", - "variantHeight": 12 - }, - "data": { - "type": "vcf", - "vcfUrl": "https://cgap-higlass.s3.amazonaws.com/gnomad/gnomad.higlass.v3.1.sites.vcf.gz", - "tbiUrl": "https://cgap-higlass.s3.amazonaws.com/gnomad/gnomad.higlass.v3.1.sites.vcf.gz.tbi", - "chromSizesUrl": "https://aveit.s3.amazonaws.com/higlass/data/sequence/hg38.chrom.sizes" - }, - "width": 768, - "height": 200 - }, - { - "uid": "emptytrack", - "type": "empty", - "options": {}, - "width": 568, - "height": 20 - } - ], - "left":[ - - ], - "center":[ - - ], - "right":[ - - ], - "bottom":[ - - ], - "whole":[ - { - "type": "vertical-rule", - "x": 1000, - "options": { - "color": "lightgrey" - }, - "uid": "vr1", - "width": 20, - "height": 20 - }, - { - "type": "vertical-rule", - "x": 1001, - "options": { - "color": "lightgrey" - }, - "uid": "vr2", - "width": 20, - "height": 20 - } - ], - "gallery":[ - - ] - }, - "layout":{ - "w":12, - "h":12, - "x":0, - "y":0 - }, - "initialYDomain":[ - 655540254.4718345, - 2044910818.0040488 - ] - } - ], - "zoomLocks":{ - "locksByViewUid":{ - - }, - "locksDict":{ - - } - }, - "locationLocks":{ - "locksByViewUid":{ - - }, - "locksDict":{ - - } - }, - "valueScaleLocks":{ - "locksByViewUid":{ - - }, - "locksDict":{ - - } - } - }, - "name": "higlass-default-viewconf", - "title": "Default CGAP hg38 viewconf", - "genome_assembly": "GRCh38", - "uuid": "00000000-1111-0000-1111-000000000000", - "schema_version": "1" - }, - { - "viewconfig":{ - "views": [ - { - "autocompleteSource":"/api/v1/suggest/?d=OHJakQICQD6gTD7skx4EWA&", - "genomePositionSearchBox":{ - "autocompleteServer":"https://cgap-higlass.com/api/v1", - "autocompleteId":"P0PLbQMwTYGy-5uPIQid7A", - "chromInfoServer":"https://cgap-higlass.com/api/v1", - "chromInfoId":"hg38", - "visible":true - }, - "chromInfoPath":"//s3.amazonaws.com/pkerp/data/hg38/chromSizes.tsv", - "tracks": { - "top": [ - { - "type": "combined", - "uid": "FkGY-Yv9T8avNXljXklukw", - "height": 55, - "width": 568, - "contents": [ - { - "filetype": "beddb", - "server": "https://cgap-higlass.com/api/v1", - "tilesetUid": "gene_annotation_hg38", - "uid": "FocNIVsfRVWMRzfmHfncsQ", - "type": "gene-annotations", - "options": { - "fontSize": 10, - "labelColor": "black", - "labelBackgroundColor": "#ffffff", - "labelPosition": "hidden", - "labelLeftMargin": 0, - "labelRightMargin": 0, - "labelTopMargin": 0, - "labelBottomMargin": 0, - "minHeight": 24, - "plusStrandColor": "#8a8ccf", - "minusStrandColor": "#e8727a", - "trackBorderWidth": 0, - "trackBorderColor": "black", - "showMousePosition": false, - "mousePositionColor": "#000000", - "geneAnnotationHeight": 12, - "geneLabelPosition": "outside", - "geneStrandSpacing": 2, - "name": "Gene Annotations (hg38)" - }, - "width": 568, - "height": 55 - }, - { - "uid": "d6KeVfkNSmq_YNj_rWJgBA", - "type": "viewport-projection-horizontal", - "fromViewUid": "ab", - "options": { - "projectionFillColor": "#777", - "projectionStrokeColor": "#777", - "projectionFillOpacity": 0.3, - "projectionStrokeOpacity": 0.7, - "strokeWidth": 1 - }, - "width": 568, - "height": 55 - } - ], - "options": {} - } - ], - "left": [], - "center": [], - "right": [], - "bottom": [], - "whole": [], - "gallery": [] - }, - "initialXDomain": [ - 594954043.6728096, - 2673883060.323411 - ], - "initialYDomain": [ - -2681117714.2684245, - -2681117686.505896 - ], - "layout": { - "w": 12, - "h": 2, - "x": 0, - "y": 0 - }, - "uid": "aa" - }, - { - "uid": "ab", - "layout": { - "h": 12, - "w": 12, - "x": 0, - "y": 0 - }, - "autocompleteSource":"/api/v1/suggest/?d=OHJakQICQD6gTD7skx4EWA&", - "genomePositionSearchBox":{ - "autocompleteServer":"https://cgap-higlass.com/api/v1", - "autocompleteId":"P0PLbQMwTYGy-5uPIQid7A", - "chromInfoServer":"https://cgap-higlass.com/api/v1", - "chromInfoId":"hg38", - "visible":true - }, - "chromInfoPath":"//s3.amazonaws.com/pkerp/data/hg38/chromSizes.tsv", - "tracks": { - "top": [ - { - "filetype":"chromsizes-tsv", - "server":"https://cgap-higlass.com/api/v1", - "tilesetUid":"chromsizes_hg38", - "uid":"AdlJsUYFRzuJRZyYeKDX2A", - "type":"chromosome-labels", - "options":{ - "color":"#808080", - "stroke":"#ffffff", - "fontSize":12, - "fontIsLeftAligned":false, - "showMousePosition":false, - "mousePositionColor":"#000000" - }, - "width":811, - "height":30 - }, - { - "uid": "fastaex", - "data": { - "type": "fasta", - "faiUrl": "https://aveit.s3.amazonaws.com/higlass/data/sequence/hg38.fa.fai", - "fastaUrl": "https://aveit.s3.amazonaws.com/higlass/data/sequence/hg38.fa", - "chromSizesUrl": "https://aveit.s3.amazonaws.com/higlass/data/sequence/hg38.chrom.sizes" - }, - "type": "horizontal-sequence", - "width": 768, - "height": 25, - "options": { - "name": "hg38", - "barBorder": true, - "colorScale": [ - "#08519c", - "#6baed6", - "#993404", - "#fe9929", - "#808080", - "#DCDCDC" - ], - "labelColor": "black", - "valueScaling": "linear", - "labelPosition": "topLeft", - "barBorderColor": "white", - "backgroundColor": "white", - "labelTextOpacity": 0.4, - "sortLargestOnTop": true, - "trackBorderColor": "white", - "trackBorderWidth": 0, - "extendedPreloading": false, - "colorAggregationMode": "none", - "notificationText": "Zoom in to see nucleotides...", - "fontSize": 16, - "fontFamily": "Arial", - "fontColor": "white", - "textOption": { - "fontSize": "32px", - "fontFamily": "Arial", - "fill": 16777215, - "fontWeight": "bold" - } - } - }, - { - "uid": "emptytrack_text_transcripts", - "type": "empty", - "options": {}, - "width": 568, - "height": 10 - }, - { - "uid": "texttrack_transcripts", - "type": "text", - "server": "https://cgap-higlass.com/api/v1", - "options": { - "backgroundColor": "#ededed", - "textColor": "#333333", - "fontSize": 11, - "fontFamily": "Arial", - "fontWeight": "normal", - "offsetY": 4, - "align": "left", - "text": "Canonical transcripts" - }, - "width": 568, - "height": 20 - }, - { - "uid": "emptytrack_transcripts", - "type": "empty", - "options": {}, - "width": 568, - "height": 10 - }, - { - "server": "https://cgap-higlass.com/api/v1", - "tilesetUid": "canonical_transcripts_hg38", - "uid": "transcript_annotation2", - "type": "horizontal-transcripts", - "options": { - "fontSize": 9, - "labelFontColor": "#222222", - "labelBackgroundPlusStrandColor": "#e9e9e9", - "labelBackgroundMinusStrandColor": "#e9e9e9", - "minHeight": 24, - "plusStrandColor": "#bdbfff", - "minusStrandColor": "#fabec2", - "utrColor": "#C0EAAF", - "mousePositionColor": "#000000", - "transcriptHeight": 12, - "transcriptSpacing": 2, - "name": "Gene transcripts", - "fontFamily": "Helvetica", - "maxTexts": 100, - "showToggleTranscriptsButton": false, - "startCollapsed": false, - "sequenceData": { - "type": "fasta", - "fastaUrl": "https://aveit.s3.amazonaws.com/higlass/data/sequence/hg38.fa", - "faiUrl": "https://aveit.s3.amazonaws.com/higlass/data/sequence/hg38.fa.fai", - "chromSizesUrl": "https://aveit.s3.amazonaws.com/higlass/data/sequence/hg38.chrom.sizes" - } - }, - "width": 768, - "height": 40 - }, - { - "uid": "emptytrack_a", - "type": "empty", - "options": {}, - "width": 470, - "height": 10 - }, - { - "uid": "texttrack", - "type": "text", - "server": "https://cgap-higlass.com/api/v1", - "options": { - "backgroundColor": "#ededed", - "textColor": "#333333", - "fontSize": 12, - "fontFamily": "Arial", - "fontWeight": "bold", - "offsetY": 4, - "align": "left", - "text": "Proband" - }, - "width": 568, - "height": 23 - }, - { - "uid": "emptytrack_b", - "type": "empty", - "options": {}, - "width": 470, - "height": 5 - }, - { - "type": "pileup", - "options": { - "axisPositionHorizontal": "right", - "axisLabelFormatting": "normal", - "showCoverage": true, - "outlineReadOnHover": true, - "groupBy": "strand", - "minusStrandColor": "#ffd1d4", - "plusStrandColor": "#cfd0ff", - "colorScale": [ - "#08519c", - "#6baed6", - "#993404", - "#fe9929", - "#808080", - "#DCDCDC" - ], - "workerScriptLocation": "/static/build/" - }, - "height": 200, - "uid": "pileup", - "data": { - "type": "bam", - "chromSizesUrl": "https://aveit.s3.amazonaws.com/higlass/data/sequence/hg38.chrom.sizes" - }, - "width": 470 - } - ], - "left": [], - "right": [], - "whole": [ - { - "x": 1558492997, - "uid": "vr1", - "type": "vertical-rule", - "width": 20, - "height": 20, - "options": { - "color": "lightgrey" - } - }, - { - "x": 1558492998, - "uid": "vr2", - "type": "vertical-rule", - "width": 20, - "height": 20, - "options": { - "color": "lightgrey" - } - } - ], - "bottom": [], - "center": [], - "gallery": [] - }, - "initialXDomain": [ - 1069795.099783097, - 1070508.6726894341 - ], - "initialYDomain": [ - 1350225534.8565273, - 1350225534.8565273 - ] - } - ], - "editable": true, - "zoomFixed": false, - "zoomLocks": { - "locksByViewUid": {}, - "locksDict": {} - }, - "exportViewUrl": "/api/v1/viewconfs", - "locationLocks": { - "locksByViewUid": {}, - "locksDict": {} - }, - "valueScaleLocks": { - "locksByViewUid": {}, - "locksDict": {} - }, - "trackSourceServers": [ - "https://cgap-higlass.com/api/v1" - ] - }, - "name": "higlass-bam-viewconf", - "title": "Default CGAP BAM viewconf", - "genome_assembly": "GRCh38", - "uuid": "9146eeba-ebb8-41aa-93a8-ada8efaff64b", - "schema_version": "1" - } -] From 4751df4653ca802763d2dad0eec7d0f8a087189d Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Wed, 24 Feb 2021 08:22:21 -0500 Subject: [PATCH 008/120] small changes --- deploy/docker/docker-compose.yml | 4 ++-- deploy/docker/local/Dockerfile | 3 +++ deploy/docker/local/development.ini | 4 ++-- deploy/docker/postgres/Dockerfile | 3 ++- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/deploy/docker/docker-compose.yml b/deploy/docker/docker-compose.yml index 60e80a2a5f..ccdc07adca 100644 --- a/deploy/docker/docker-compose.yml +++ b/deploy/docker/docker-compose.yml @@ -1,7 +1,5 @@ version: "3.8" -# use this in development.ini -# sqlalchemy.url = postgresql://postgres:postgres@db:5432/postgres services: # Postgres Component @@ -15,6 +13,8 @@ services: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_PORT: 5432 + # Enable to get verbose logs + # command: ["postgres", "-c", "log_statement=all"] # ElasticSearch Component # Disabled for now as too compute intensive diff --git a/deploy/docker/local/Dockerfile b/deploy/docker/local/Dockerfile index 4d11909966..429f74c8cf 100644 --- a/deploy/docker/local/Dockerfile +++ b/deploy/docker/local/Dockerfile @@ -1,3 +1,6 @@ +# CGAP-Portal Dockerfile + +# TODO: appropriately pin/verify this image FROM python:3.6-buster MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" diff --git a/deploy/docker/local/development.ini b/deploy/docker/local/development.ini index e524b17360..4421c083a3 100644 --- a/deploy/docker/local/development.ini +++ b/deploy/docker/local/development.ini @@ -21,12 +21,12 @@ indexer = true indexer.namespace = cgap-docker-will-test pyramid.reload_templates = true pyramid.debug_authorization = false -pyramid.debug_notfound = true +pyramid.debug_notfound = false pyramid.debug_routematch = false pyramid.default_locale_name = en # this line determines which load function is used in load_data # most deployments use: "load_test_data = encoded.loadxl:load_test_data" -load_test_data = encoded.loadxl:load_prod_data +load_test_data = encoded.loadxl:load_local_data encoded_version = 100.200.300 snovault_version = 200.300.400 utils_version = 300.400.500 diff --git a/deploy/docker/postgres/Dockerfile b/deploy/docker/postgres/Dockerfile index 9fd7b73bf0..344b48aba9 100644 --- a/deploy/docker/postgres/Dockerfile +++ b/deploy/docker/postgres/Dockerfile @@ -1,4 +1,5 @@ -FROM postgres:11.11 +# TODO: upgrade to latest version we can tolerate +FROM postgres:11.8 MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" From 9e075b65ac11c374b6ed3f1ad59911df5b237484 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Wed, 24 Feb 2021 10:33:05 -0500 Subject: [PATCH 009/120] get traceback on patch failure --- deploy/docker/docker-compose.yml | 2 +- deploy/docker/local/Dockerfile | 10 +++++++--- deploy/docker/postgres/Dockerfile | 2 +- src/encoded/loadxl.py | 2 ++ 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/deploy/docker/docker-compose.yml b/deploy/docker/docker-compose.yml index ccdc07adca..1e12e30461 100644 --- a/deploy/docker/docker-compose.yml +++ b/deploy/docker/docker-compose.yml @@ -14,7 +14,7 @@ services: POSTGRES_PASSWORD: postgres POSTGRES_PORT: 5432 # Enable to get verbose logs - # command: ["postgres", "-c", "log_statement=all"] + command: ["postgres", "-c", "log_statement=all"] # ElasticSearch Component # Disabled for now as too compute intensive diff --git a/deploy/docker/local/Dockerfile b/deploy/docker/local/Dockerfile index 429f74c8cf..0aa484fa24 100644 --- a/deploy/docker/local/Dockerfile +++ b/deploy/docker/local/Dockerfile @@ -12,6 +12,7 @@ ENV CGAP_ENV_NAME=${CGAP_ENV_NAME:-"cgap-docker-will-test"} # Configure (global) Env ENV CGAP_REPO=https://github.com/dbmi-bgm/cgap-portal.git ENV CGAP_BRANCH=c4_519 +ENV NGINX_USER=cgap-admin ENV DEBIAN_FRONTEND=noninteractive ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1 ENV PYTHONFAULTHANDLER=1 \ @@ -62,14 +63,17 @@ RUN cat /dev/urandom | head -c 256 | base64 > session-secret.b64 # Copy config files in (down here for quick debugging) # Remove default configuration from Nginx -RUN rm /etc/nginx/conf.d/default.conf -COPY nginx.conf /etc/nginx/conf.d/default.conf +RUN rm /etc/nginx/nginx.conf +COPY nginx.conf /etc/nginx/nginx.conf COPY development.ini . COPY entrypoint.sh . RUN chmod 777 entrypoint.sh EXPOSE 8000 +RUN nginx -v +RUN cat /etc/nginx/nginx.conf + +RUN service nginx start USER cgap-admin -CMD service nginx start ENTRYPOINT ["/home/cgap-admin/cgap-portal/entrypoint.sh"] diff --git a/deploy/docker/postgres/Dockerfile b/deploy/docker/postgres/Dockerfile index 344b48aba9..f096e6b7ac 100644 --- a/deploy/docker/postgres/Dockerfile +++ b/deploy/docker/postgres/Dockerfile @@ -1,5 +1,5 @@ # TODO: upgrade to latest version we can tolerate -FROM postgres:11.8 +FROM postgres:12.3 MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" diff --git a/src/encoded/loadxl.py b/src/encoded/loadxl.py index 224a3c9c22..4c3d63a20b 100644 --- a/src/encoded/loadxl.py +++ b/src/encoded/loadxl.py @@ -458,8 +458,10 @@ def load_all_gen(testapp, inserts, docsdir, overwrite=True, itype=None, from_jso # yield bytes to work with Response.app_iter yield str.encode('PATCH: %s\n' % an_item['uuid']) except Exception as e: + import traceback print('Patching {} failed. Patch body:\n{}\n\nError Message:\n{}'.format( a_type, str(an_item), str(e))) + print('Full error: %s' % traceback.format_exc()) e_str = str(e).replace('\n', '') # import pdb; pdb.set_trace() yield str.encode('ERROR: %s\n' % e_str) From 5c05b3d941784415db71f4fa1004fd50aa09332f Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Wed, 24 Feb 2021 15:41:16 -0500 Subject: [PATCH 010/120] C4-519 working local deployment w/ nginx proxy, load_data still an issue --- deploy/docker/docker-compose.yml | 6 ++-- deploy/docker/local/Dockerfile | 44 +++++++++++++++++------------ deploy/docker/local/development.ini | 2 +- deploy/docker/local/entrypoint.sh | 3 ++ deploy/docker/local/nginx.conf | 15 ++++++++-- 5 files changed, 45 insertions(+), 25 deletions(-) diff --git a/deploy/docker/docker-compose.yml b/deploy/docker/docker-compose.yml index 1e12e30461..8acb970ef1 100644 --- a/deploy/docker/docker-compose.yml +++ b/deploy/docker/docker-compose.yml @@ -13,8 +13,8 @@ services: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_PORT: 5432 - # Enable to get verbose logs - command: ["postgres", "-c", "log_statement=all"] + # Enable to get verbose logs from postgres + #command: ["postgres", "-c", "log_statement=all"] # ElasticSearch Component # Disabled for now as too compute intensive @@ -44,7 +44,7 @@ services: AWS_ACCESS_KEY_ID: "$AWS_ACCESS_KEY_ID" AWS_SECRET_ACCESS_KEY: "$AWS_SECRET_ACCESS_KEY" ports: - - "6543:6543" # access app directly + #- "6543:6543" # access app directly - "8000:8000" # nginx proxy port (not functional currently) depends_on: - db diff --git a/deploy/docker/local/Dockerfile b/deploy/docker/local/Dockerfile index 0aa484fa24..68a573082e 100644 --- a/deploy/docker/local/Dockerfile +++ b/deploy/docker/local/Dockerfile @@ -1,7 +1,7 @@ # CGAP-Portal Dockerfile # TODO: appropriately pin/verify this image -FROM python:3.6-buster +FROM python:3.6.12-buster MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" @@ -12,7 +12,7 @@ ENV CGAP_ENV_NAME=${CGAP_ENV_NAME:-"cgap-docker-will-test"} # Configure (global) Env ENV CGAP_REPO=https://github.com/dbmi-bgm/cgap-portal.git ENV CGAP_BRANCH=c4_519 -ENV NGINX_USER=cgap-admin +ENV NGINX_USER=nginx ENV DEBIAN_FRONTEND=noninteractive ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1 ENV PYTHONFAULTHANDLER=1 \ @@ -29,27 +29,26 @@ RUN bash /install_nginx.sh # Intall things needed for our system RUN apt-get update -RUN apt-get install -y curl postgresql-client +RUN apt-get install -y curl vim emacs postgresql-client net-tools RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - RUN apt-get install -y ca-certificates nodejs npm # Configure CGAP User -RUN useradd -ms /bin/bash cgap-admin -WORKDIR /home/cgap-admin -RUN mkdir -p /home/cgap-admin/cgap-portal -RUN chown cgap-admin /home/cgap-admin/cgap-portal +WORKDIR /home/nginx +RUN mkdir -p /home/nginx/cgap-portal +RUN chown nginx /home/nginx/cgap-portal # Configure venv ENV VIRTUAL_ENV=/opt/venv RUN python -m venv /opt/venv ENV PATH="$VIRTUAL_ENV/bin:$PATH" -RUN chown cgap-admin /opt/venv +RUN chown nginx /opt/venv -# Copy to /home/cgap-admin/cgap-portal +# Copy to /home/nginx/cgap-portal RUN git clone $CGAP_REPO --branch $CGAP_BRANCH # Build the application -WORKDIR /home/cgap-admin/cgap-portal +WORKDIR /home/nginx/cgap-portal RUN pip install --upgrade pip RUN pip install poetry==1.1.4 wheel==0.29.0 RUN poetry install @@ -64,16 +63,25 @@ RUN cat /dev/urandom | head -c 256 | base64 > session-secret.b64 # Copy config files in (down here for quick debugging) # Remove default configuration from Nginx RUN rm /etc/nginx/nginx.conf +RUN rm /etc/nginx/conf.d/default.conf COPY nginx.conf /etc/nginx/nginx.conf +# give nginx user permissions +RUN chown -R nginx:nginx /var/cache/nginx && \ + chown -R nginx:nginx /var/log/nginx && \ + chown -R nginx:nginx /etc/nginx/conf.d +RUN touch /var/run/nginx.pid && \ + chown -R nginx:nginx /var/run/nginx.pid +RUN rm -f /var/log/nginx/* +RUN touch /var/log/nginx/access.log && \ + chown -R nginx:nginx /var/log/nginx/access.log +RUN touch /var/log/nginx/error.log && \ + chown -R nginx:nginx /var/log/nginx/error.log + +# Copy over ini file, entrypoint COPY development.ini . COPY entrypoint.sh . -RUN chmod 777 entrypoint.sh +RUN chmod +x entrypoint.sh EXPOSE 8000 -RUN nginx -v -RUN cat /etc/nginx/nginx.conf - -RUN service nginx start -USER cgap-admin - +USER nginx -ENTRYPOINT ["/home/cgap-admin/cgap-portal/entrypoint.sh"] +ENTRYPOINT ["/home/nginx/cgap-portal/entrypoint.sh"] diff --git a/deploy/docker/local/development.ini b/deploy/docker/local/development.ini index 4421c083a3..79ee1d5c20 100644 --- a/deploy/docker/local/development.ini +++ b/deploy/docker/local/development.ini @@ -26,7 +26,7 @@ pyramid.debug_routematch = false pyramid.default_locale_name = en # this line determines which load function is used in load_data # most deployments use: "load_test_data = encoded.loadxl:load_test_data" -load_test_data = encoded.loadxl:load_local_data +load_test_data = encoded.loadxl:load_prod_data encoded_version = 100.200.300 snovault_version = 200.300.400 utils_version = 300.400.500 diff --git a/deploy/docker/local/entrypoint.sh b/deploy/docker/local/entrypoint.sh index 3a4fd4ba31..69d439cdc6 100644 --- a/deploy/docker/local/entrypoint.sh +++ b/deploy/docker/local/entrypoint.sh @@ -12,5 +12,8 @@ poetry run create-mapping-on-deploy development.ini --app-name app # Load Data (based on development.ini, for now just master-inserts) poetry run load-data development.ini --app-name app --prod +# Start nginx proxy +service nginx start + # Start application make deploy2 diff --git a/deploy/docker/local/nginx.conf b/deploy/docker/local/nginx.conf index 2a3203657c..3119c3b11a 100644 --- a/deploy/docker/local/nginx.conf +++ b/deploy/docker/local/nginx.conf @@ -1,8 +1,9 @@ # Minimal nginx proxy for development -# Used for local deployment + +error_log /var/log/nginx/error.log warn; events { - worker_connections 2048; + worker_connections 2048; } http { resolver 8.8.8.8; @@ -10,8 +11,14 @@ http { server 0.0.0.0:6543; keepalive 10; } + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + server { listen 8000; + access_log /var/log/nginx/access.log main; location / { # Normalize duplicate slashes if ($request ~ ^(GET|HEAD)\s([^?]*)//(.*)\sHTTP/[0-9.]+$) { @@ -25,8 +32,10 @@ http { } location ~ ^/_proxy/(.*)$ { internal; + proxy_set_header Authorization ""; + proxy_set_header Content-Type ""; proxy_buffering off; proxy_pass $1$is_args$args; } } -} +} \ No newline at end of file From 6ff7ce77df42c223898e1a25daa77f97148e5b92 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Wed, 24 Feb 2021 15:56:13 -0500 Subject: [PATCH 011/120] point at broken inserts --- deploy/docker/docker-compose.yml | 5 ++--- deploy/docker/local/development.ini | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/deploy/docker/docker-compose.yml b/deploy/docker/docker-compose.yml index 8acb970ef1..c4e6e593f8 100644 --- a/deploy/docker/docker-compose.yml +++ b/deploy/docker/docker-compose.yml @@ -4,7 +4,7 @@ services: # Postgres Component # Connect to it in development.ini with - # sqlalchemy.url = postgresql://postgres:postgres@db:5441/postgres + # sqlalchemy.url = postgresql://postgres:postgres@db:5432/postgres db: build: ./postgres container_name: pg1 @@ -44,8 +44,7 @@ services: AWS_ACCESS_KEY_ID: "$AWS_ACCESS_KEY_ID" AWS_SECRET_ACCESS_KEY: "$AWS_SECRET_ACCESS_KEY" ports: - #- "6543:6543" # access app directly - - "8000:8000" # nginx proxy port (not functional currently) + - "8000:8000" # nginx proxy port (note application traffic is not forwarded) depends_on: - db #- es diff --git a/deploy/docker/local/development.ini b/deploy/docker/local/development.ini index 79ee1d5c20..4421c083a3 100644 --- a/deploy/docker/local/development.ini +++ b/deploy/docker/local/development.ini @@ -26,7 +26,7 @@ pyramid.debug_routematch = false pyramid.default_locale_name = en # this line determines which load function is used in load_data # most deployments use: "load_test_data = encoded.loadxl:load_test_data" -load_test_data = encoded.loadxl:load_prod_data +load_test_data = encoded.loadxl:load_local_data encoded_version = 100.200.300 snovault_version = 200.300.400 utils_version = 300.400.500 From a19ed4a84fe27bca72b1e6a9991ab82f21ca0e95 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 25 Feb 2021 08:57:51 -0500 Subject: [PATCH 012/120] small changes in hope of resolving load problem --- deploy/docker/local/Dockerfile | 2 +- src/encoded/types/base.py | 1 + src/encoded/types/family.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/deploy/docker/local/Dockerfile b/deploy/docker/local/Dockerfile index 68a573082e..c0c67ab95e 100644 --- a/deploy/docker/local/Dockerfile +++ b/deploy/docker/local/Dockerfile @@ -45,7 +45,7 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH" RUN chown nginx /opt/venv # Copy to /home/nginx/cgap-portal -RUN git clone $CGAP_REPO --branch $CGAP_BRANCH +RUN git clone $CGAP_REPO --branch c4_519 # Build the application WORKDIR /home/nginx/cgap-portal diff --git a/src/encoded/types/base.py b/src/encoded/types/base.py index 6873786945..1940ca593f 100644 --- a/src/encoded/types/base.py +++ b/src/encoded/types/base.py @@ -337,6 +337,7 @@ def _update(self, properties, sheets=None): except KeyError: pass add_last_modified(properties) + print('Updating with props: %s, sheets: %s' % (properties, sheets)) super(Item, self)._update(properties, sheets) @snovault.calculated_property(schema={ diff --git a/src/encoded/types/family.py b/src/encoded/types/family.py index c4fa42a7e7..1bf4b7bae2 100644 --- a/src/encoded/types/family.py +++ b/src/encoded/types/family.py @@ -632,7 +632,7 @@ def process_pedigree(context, request): fam_props = context.upgrade_properties() post_extra = {'project': fam_props['project'], 'institution': fam_props['institution']} - xml_extra = {'ped_datetime': ped_datetime} + xml_extra = {'ped_datetime': ped_datetime.isoformat()} family_uuids = create_family_proband(testapp, xml_data, refs, 'managedObjectID', family_item, post_extra, xml_extra) From e01e8a4be70c442766323ecaa6917595ee4b0e59 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 25 Feb 2021 09:23:35 -0500 Subject: [PATCH 013/120] cast Expiration to string ???? --- src/encoded/types/base.py | 7 ------- src/encoded/types/file.py | 2 ++ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/encoded/types/base.py b/src/encoded/types/base.py index 1940ca593f..36d36ce7ab 100644 --- a/src/encoded/types/base.py +++ b/src/encoded/types/base.py @@ -331,13 +331,6 @@ def is_update_by_admin_user(self): return False def _update(self, properties, sheets=None): - props = {} - try: - props = self.properties - except KeyError: - pass - add_last_modified(properties) - print('Updating with props: %s, sheets: %s' % (properties, sheets)) super(Item, self)._update(properties, sheets) @snovault.calculated_property(schema={ diff --git a/src/encoded/types/file.py b/src/encoded/types/file.py index 699edf5fff..6ef280f827 100644 --- a/src/encoded/types/file.py +++ b/src/encoded/types/file.py @@ -116,6 +116,8 @@ def external_creds(bucket, key, name=None, profile_name=None): token = conn.get_federation_token(Name=name, Policy=json.dumps(policy)) # 'access_key' 'secret_key' 'expiration' 'session_token' credentials = token.get('Credentials') + # XXX: this is a datetime object in docker ???? -Will + credentials['Expiration'] = str(credentials['Expiration']) credentials.update({ 'upload_url': 's3://{bucket}/{key}'.format(bucket=bucket, key=key), 'federated_user_arn': token.get('FederatedUser').get('Arn'), From f62a841e38d82fcdad7939f2a5c2c811ca3f51b5 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 25 Feb 2021 13:36:33 -0500 Subject: [PATCH 014/120] C4-519 clean up in prep for review --- deploy/docker/local/development.ini | 3 ++- deploy/docker/local/nginx.conf | 2 +- src/encoded/loadxl.py | 2 +- src/encoded/types/base.py | 1 + src/encoded/types/file.py | 3 ++- webpack.config.js | 2 +- 6 files changed, 8 insertions(+), 5 deletions(-) diff --git a/deploy/docker/local/development.ini b/deploy/docker/local/development.ini index 4421c083a3..0155be332c 100644 --- a/deploy/docker/local/development.ini +++ b/deploy/docker/local/development.ini @@ -1,6 +1,7 @@ ### -# app configuration +# Docker App Configuration for local deployment # http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/environment.html +# NOTE: Still needs to be customized for multiple users ### [app:app] diff --git a/deploy/docker/local/nginx.conf b/deploy/docker/local/nginx.conf index 3119c3b11a..4921e4fdb9 100644 --- a/deploy/docker/local/nginx.conf +++ b/deploy/docker/local/nginx.conf @@ -38,4 +38,4 @@ http { proxy_pass $1$is_args$args; } } -} \ No newline at end of file +} diff --git a/src/encoded/loadxl.py b/src/encoded/loadxl.py index 4c3d63a20b..4078add0f8 100644 --- a/src/encoded/loadxl.py +++ b/src/encoded/loadxl.py @@ -7,6 +7,7 @@ import os import structlog import webtest +import traceback from base64 import b64encode from past.builtins import basestring @@ -458,7 +459,6 @@ def load_all_gen(testapp, inserts, docsdir, overwrite=True, itype=None, from_jso # yield bytes to work with Response.app_iter yield str.encode('PATCH: %s\n' % an_item['uuid']) except Exception as e: - import traceback print('Patching {} failed. Patch body:\n{}\n\nError Message:\n{}'.format( a_type, str(an_item), str(e))) print('Full error: %s' % traceback.format_exc()) diff --git a/src/encoded/types/base.py b/src/encoded/types/base.py index 36d36ce7ab..bbf267aea3 100644 --- a/src/encoded/types/base.py +++ b/src/encoded/types/base.py @@ -331,6 +331,7 @@ def is_update_by_admin_user(self): return False def _update(self, properties, sheets=None): + add_last_modified(properties) super(Item, self)._update(properties, sheets) @snovault.calculated_property(schema={ diff --git a/src/encoded/types/file.py b/src/encoded/types/file.py index 6ef280f827..40debe2bd7 100644 --- a/src/encoded/types/file.py +++ b/src/encoded/types/file.py @@ -116,7 +116,8 @@ def external_creds(bucket, key, name=None, profile_name=None): token = conn.get_federation_token(Name=name, Policy=json.dumps(policy)) # 'access_key' 'secret_key' 'expiration' 'session_token' credentials = token.get('Credentials') - # XXX: this is a datetime object in docker ???? -Will + # Convert Expiration datetime object to string via cast + # Uncaught serialization error picked up by Docker - Will 2/25/2020 credentials['Expiration'] = str(credentials['Expiration']) credentials.update({ 'upload_url': 's3://{bucket}/{key}'.format(bucket=bucket, key=key), diff --git a/webpack.config.js b/webpack.config.js index 57f2e32fd1..b19445fb09 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -79,7 +79,7 @@ const optimization = { // sourceMap: true //}) new TerserPlugin({ - parallel: false, // XXX: this option causes docker build to fail. - Will + parallel: false, // XXX: this option causes docker build to fail - Will 2/25/2020 sourceMap: true, terserOptions:{ compress: true, From 0f13d9334881fff5bdd86a65b42dc6a4ca00a1f3 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 25 Feb 2021 13:59:19 -0500 Subject: [PATCH 015/120] fix some dates, add some documentation --- deploy/docker/local/development.ini | 2 +- docs/source/docker-local.rst | 40 +++++++++++++++++++++++++++++ docs/source/introduction.rst | 2 +- src/encoded/types/file.py | 2 +- webpack.config.js | 2 +- 5 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 docs/source/docker-local.rst diff --git a/deploy/docker/local/development.ini b/deploy/docker/local/development.ini index 0155be332c..9d365c34ca 100644 --- a/deploy/docker/local/development.ini +++ b/deploy/docker/local/development.ini @@ -1,7 +1,7 @@ ### # Docker App Configuration for local deployment # http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/environment.html -# NOTE: Still needs to be customized for multiple users +# NOTE: Still needs to be customized for (automated use by) multiple users ### [app:app] diff --git a/docs/source/docker-local.rst b/docs/source/docker-local.rst new file mode 100644 index 0000000000..b26dadee6e --- /dev/null +++ b/docs/source/docker-local.rst @@ -0,0 +1,40 @@ +CGAP-Docker +=========== + +It is now possible to run a local deployment of CGAP without installing any system level +dependencies other than Docker. Start by installing Docker:: + + $ brew install docker + + +Prior to building the image, navigate to deploy/docker/local and open development.ini + +* Modify env.name and indexer.namespace - these values must be globally unique (feel free to just replace the name) + +There are two new Make targets that should be sufficient for normal use. To build the image locally, ensure your +AWS keys are sourced and run:: + + $ make build-docker # runs docker-compose build + $ make deploy-docker # runs docker-compose up + +The first command will take awhile the first time you run it but should speed up after. Since it is doing a fresh +rebuild every time it is a little slower than the old local deployment since it has to fully reinstall/rebuild both Python +and the client. + +To access the running container:: + + $ docker ps # will show running containers + $ docker exec -it /bin/bash + +Advanced Usage +-------------- + +There are several useful commands documented below that may be helpful when issues are encountered or changes need to be made. + +* ``docker-compose build --no-cache`` # will force a full rebuild of the entire image + +Configuration Notes +------------------- + +* TODO write me + diff --git a/docs/source/introduction.rst b/docs/source/introduction.rst index e774fa4cac..7cb9dd8ebf 100644 --- a/docs/source/introduction.rst +++ b/docs/source/introduction.rst @@ -1,5 +1,5 @@ Introduction --------- +------------ * The 4DN Data Portal will be the primary access point to the omics and imaging data, analysis tools, and integrative models generated and utilized by the 4DN Network. diff --git a/src/encoded/types/file.py b/src/encoded/types/file.py index 40debe2bd7..1e027a873f 100644 --- a/src/encoded/types/file.py +++ b/src/encoded/types/file.py @@ -117,7 +117,7 @@ def external_creds(bucket, key, name=None, profile_name=None): # 'access_key' 'secret_key' 'expiration' 'session_token' credentials = token.get('Credentials') # Convert Expiration datetime object to string via cast - # Uncaught serialization error picked up by Docker - Will 2/25/2020 + # Uncaught serialization error picked up by Docker - Will 2/25/2021 credentials['Expiration'] = str(credentials['Expiration']) credentials.update({ 'upload_url': 's3://{bucket}/{key}'.format(bucket=bucket, key=key), diff --git a/webpack.config.js b/webpack.config.js index b19445fb09..3feb2f7fab 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -79,7 +79,7 @@ const optimization = { // sourceMap: true //}) new TerserPlugin({ - parallel: false, // XXX: this option causes docker build to fail - Will 2/25/2020 + parallel: false, // XXX: this option causes docker build to fail - Will 2/25/2021 sourceMap: true, terserOptions:{ compress: true, From fb032875f5d8441042cc6e1eacaac8a7f00ea68c Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Fri, 26 Feb 2021 08:41:37 -0500 Subject: [PATCH 016/120] destroy the tests (temporarily) --- deploy/docker/local/Dockerfile | 3 +++ deploy/docker/local/entrypoint.sh | 29 +++++++++++++++++------------ src/encoded/tests/conftest.py | 9 ++++----- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/deploy/docker/local/Dockerfile b/deploy/docker/local/Dockerfile index c0c67ab95e..adcbded841 100644 --- a/deploy/docker/local/Dockerfile +++ b/deploy/docker/local/Dockerfile @@ -8,6 +8,8 @@ MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" # Build Arguments ARG CGAP_ENV_NAME ENV CGAP_ENV_NAME=${CGAP_ENV_NAME:-"cgap-docker-will-test"} +ARG RUN_TEST +ENV RUN_TEST=${RUN_TEST:-1} # Configure (global) Env ENV CGAP_REPO=https://github.com/dbmi-bgm/cgap-portal.git @@ -79,6 +81,7 @@ RUN touch /var/log/nginx/error.log && \ # Copy over ini file, entrypoint COPY development.ini . +COPY development.ini test.ini COPY entrypoint.sh . RUN chmod +x entrypoint.sh EXPOSE 8000 diff --git a/deploy/docker/local/entrypoint.sh b/deploy/docker/local/entrypoint.sh index 69d439cdc6..7ed9d5239b 100644 --- a/deploy/docker/local/entrypoint.sh +++ b/deploy/docker/local/entrypoint.sh @@ -1,19 +1,24 @@ #!/bin/sh -# Enter venv -export PATH="/opt/venv/bin:$PATH" +if [ $RUN_TEST -eq "0" ]; then -# Clear db/es since this is the local entry point -poetry run clear-db-es-contents development.ini --app-name app --env $CGAP_ENV_NAME + # Clear db/es since this is the local entry point + poetry run clear-db-es-contents development.ini --app-name app --env $CGAP_ENV_NAME -# Create mapping -poetry run create-mapping-on-deploy development.ini --app-name app + # Create mapping + poetry run create-mapping-on-deploy development.ini --app-name app -# Load Data (based on development.ini, for now just master-inserts) -poetry run load-data development.ini --app-name app --prod + # Load Data (based on development.ini, for now just master-inserts) + poetry run load-data development.ini --app-name app --prod -# Start nginx proxy -service nginx start + # Start nginx proxy + service nginx start -# Start application -make deploy2 + # Start application + make deploy2 + +else + + make test + +fi diff --git a/src/encoded/tests/conftest.py b/src/encoded/tests/conftest.py index 4644731747..6472370f23 100644 --- a/src/encoded/tests/conftest.py +++ b/src/encoded/tests/conftest.py @@ -46,20 +46,19 @@ def app_settings(request, wsgi_server_host_port, conn, DBSession): @pytest.fixture(scope='session') -def es_app_settings(wsgi_server_host_port, elasticsearch_server, postgresql_server, aws_auth): +def es_app_settings(wsgi_server_host_port, aws_auth): settings = make_app_settings_dictionary() settings['create_tables'] = True settings['persona.audiences'] = 'http://%s:%s' % wsgi_server_host_port # 2-tuple such as: ('localhost', '5000') - settings['elasticsearch.server'] = elasticsearch_server - settings['sqlalchemy.url'] = postgresql_server + settings['elasticsearch.server'] = 'search-cgap-testing-6-8-vo4mdkmkshvmyddc65ux7dtaou.us-east-1.es.amazonaws.com:443' + settings['sqlalchemy.url'] = 'postgres://postgres:postgres@db:5432/postgres' settings['collection_datastore'] = 'elasticsearch' settings['item_datastore'] = 'elasticsearch' settings['indexer'] = True settings['indexer.namespace'] = INDEXER_NAMESPACE_FOR_TESTING # use aws auth to access elasticsearch - if aws_auth: - settings['elasticsearch.aws_auth'] = aws_auth + settings['elasticsearch.aws_auth'] = True return settings From cde38d352d5a63fc3975877ce4036d84ca106e32 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Fri, 26 Feb 2021 13:18:51 -0500 Subject: [PATCH 017/120] add sqlalchemy url to app settings --- src/encoded/tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/encoded/tests/conftest.py b/src/encoded/tests/conftest.py index 6472370f23..de68533df7 100644 --- a/src/encoded/tests/conftest.py +++ b/src/encoded/tests/conftest.py @@ -37,6 +37,7 @@ def app_settings(request, wsgi_server_host_port, conn, DBSession): settings = make_app_settings_dictionary() settings['auth0.audiences'] = 'http://%s:%s' % wsgi_server_host_port + settings['sqlalchemy.url'] = 'postgres://postgres:postgres@db:5432/postgres' # add some here for file testing settings[DBSESSION] = DBSession return settings From d2e7d518d647c15721788cc9682d36cfd6d949be Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Fri, 26 Feb 2021 13:38:36 -0500 Subject: [PATCH 018/120] disable postgres setup for tests --- deploy/docker/local/Dockerfile | 2 +- src/encoded/tests/conftest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/docker/local/Dockerfile b/deploy/docker/local/Dockerfile index adcbded841..f4e8397f3a 100644 --- a/deploy/docker/local/Dockerfile +++ b/deploy/docker/local/Dockerfile @@ -47,7 +47,7 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH" RUN chown nginx /opt/venv # Copy to /home/nginx/cgap-portal -RUN git clone $CGAP_REPO --branch c4_519 +RUN git clone $CGAP_REPO --branch $CGAP_BRANCH # Build the application WORKDIR /home/nginx/cgap-portal diff --git a/src/encoded/tests/conftest.py b/src/encoded/tests/conftest.py index de68533df7..57c1118f7d 100644 --- a/src/encoded/tests/conftest.py +++ b/src/encoded/tests/conftest.py @@ -33,7 +33,7 @@ def autouse_external_tx(external_tx): @pytest.fixture(scope='session') -def app_settings(request, wsgi_server_host_port, conn, DBSession): +def app_settings(request, wsgi_server_host_port, DBSession): settings = make_app_settings_dictionary() settings['auth0.audiences'] = 'http://%s:%s' % wsgi_server_host_port From b89fffbc96ca2b32e520cacc781cb2d16889817c Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Fri, 26 Feb 2021 13:51:38 -0500 Subject: [PATCH 019/120] disable tests that use conn fixture --- src/encoded/tests/test_fixtures.py | 182 ++++++++++++++--------------- 1 file changed, 91 insertions(+), 91 deletions(-) diff --git a/src/encoded/tests/test_fixtures.py b/src/encoded/tests/test_fixtures.py index 65ae566292..8a2cf08651 100644 --- a/src/encoded/tests/test_fixtures.py +++ b/src/encoded/tests/test_fixtures.py @@ -7,97 +7,97 @@ pytestmark = [pytest.mark.setone, pytest.mark.working, pytest.mark.schema, pytest.mark.indexing] -@pytest.yield_fixture(scope='session') -def minitestdata(app, conn): - tx = conn.begin_nested() - - environ = { - 'HTTP_ACCEPT': 'application/json', - 'REMOTE_USER': 'TEST', - } - testapp = TestApp(app, environ) - - item = { - 'email': 'human@email.org', - 'first_name': 'Homo', - 'last_name': 'Sapien', - } - testapp.post_json('/user', item, status=201) - - yield - tx.rollback() - - - -@pytest.yield_fixture(scope='session') -def minitestdata2(app, conn): - tx = conn.begin_nested() - - environ = { - 'HTTP_ACCEPT': 'application/json', - 'REMOTE_USER': 'TEST', - } - testapp = TestApp(app, environ) - - item = { - 'email': 'human2@email.org', - 'first_name': 'Homo', - 'last_name': 'Erectus', - } - testapp.post_json('/user', item, status=201) - - yield - tx.rollback() - - -@pytest.mark.usefixtures('minitestdata') -def test_fixtures1(testapp): - """ This test is not really exhaustive. - - Still need to inspect the sql log to verify fixture correctness. - """ - res = testapp.get('/user').maybe_follow() - items = res.json['@graph'] - assert len(items) == 1 - - # Trigger an error - item = {'foo': 'bar'} - res = testapp.post_json('/user', item, status=422) - assert res.json['errors'] - - res = testapp.get('/user').maybe_follow() - items = res.json['@graph'] - assert len(items) == 1 - - item = { - 'email': 'human3@email.org', - 'first_name': 'Homo', - 'last_name': 'Habilis', - } - testapp.post_json('/user', item, status=201) - - res = testapp.get('/user').maybe_follow() - items = res.json['@graph'] - assert len(items) == 2 - - # Trigger an error - item = {'foo': 'bar'} - res = testapp.post_json('/user', item, status=422) - assert res.json['errors'] - - res = testapp.get('/user').maybe_follow() - items = res.json['@graph'] - assert len(items) == 2 - - -def test_fixtures2(minitestdata2, testapp): - # http://stackoverflow.com/questions/15775601/mutually-exclusive-fixtures - res = testapp.get('/user').maybe_follow() - items = res.json['@graph'] - assert len(items) == 1 - - -def test_order_complete(app, conn): +# @pytest.yield_fixture(scope='session') +# def minitestdata(app, conn): +# tx = conn.begin_nested() +# +# environ = { +# 'HTTP_ACCEPT': 'application/json', +# 'REMOTE_USER': 'TEST', +# } +# testapp = TestApp(app, environ) +# +# item = { +# 'email': 'human@email.org', +# 'first_name': 'Homo', +# 'last_name': 'Sapien', +# } +# testapp.post_json('/user', item, status=201) +# +# yield +# tx.rollback() +# +# +# +# @pytest.yield_fixture(scope='session') +# def minitestdata2(app, conn): +# tx = conn.begin_nested() +# +# environ = { +# 'HTTP_ACCEPT': 'application/json', +# 'REMOTE_USER': 'TEST', +# } +# testapp = TestApp(app, environ) +# +# item = { +# 'email': 'human2@email.org', +# 'first_name': 'Homo', +# 'last_name': 'Erectus', +# } +# testapp.post_json('/user', item, status=201) +# +# yield +# tx.rollback() +# +# +# @pytest.mark.usefixtures('minitestdata') +# def test_fixtures1(testapp): +# """ This test is not really exhaustive. +# +# Still need to inspect the sql log to verify fixture correctness. +# """ +# res = testapp.get('/user').maybe_follow() +# items = res.json['@graph'] +# assert len(items) == 1 +# +# # Trigger an error +# item = {'foo': 'bar'} +# res = testapp.post_json('/user', item, status=422) +# assert res.json['errors'] +# +# res = testapp.get('/user').maybe_follow() +# items = res.json['@graph'] +# assert len(items) == 1 +# +# item = { +# 'email': 'human3@email.org', +# 'first_name': 'Homo', +# 'last_name': 'Habilis', +# } +# testapp.post_json('/user', item, status=201) +# +# res = testapp.get('/user').maybe_follow() +# items = res.json['@graph'] +# assert len(items) == 2 +# +# # Trigger an error +# item = {'foo': 'bar'} +# res = testapp.post_json('/user', item, status=422) +# assert res.json['errors'] +# +# res = testapp.get('/user').maybe_follow() +# items = res.json['@graph'] +# assert len(items) == 2 +# +# +# def test_fixtures2(minitestdata2, testapp): +# # http://stackoverflow.com/questions/15775601/mutually-exclusive-fixtures +# res = testapp.get('/user').maybe_follow() +# items = res.json['@graph'] +# assert len(items) == 1 + + +def test_order_complete(app): order = ORDER + ['access_key'] environ = { 'HTTP_ACCEPT': 'application/json', From 2e96fd55670ac22bdedf990c69bffc6968535f8e Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Mon, 1 Mar 2021 10:01:08 -0500 Subject: [PATCH 020/120] move docker-compose up, attempt volume mount (not working), remove outdated files --- .travis.yml | 79 -------- .travis.yml.old | 171 ------------------ add_route | 0 deploy/docker/local/Dockerfile | 6 +- deploy/docker/local/development.ini | 2 +- deploy/docker/local/entrypoint.sh | 4 +- .../docker-compose.yml => docker-compose.yml | 34 ++-- magic_fix | 0 nginx.yml | 78 -------- setup.cfg | 16 -- test.cfg | 2 - testing.cfg | 8 - 12 files changed, 24 insertions(+), 376 deletions(-) delete mode 100644 .travis.yml delete mode 100644 .travis.yml.old delete mode 100644 add_route rename deploy/docker/docker-compose.yml => docker-compose.yml (66%) delete mode 100644 magic_fix delete mode 100644 nginx.yml delete mode 100644 setup.cfg delete mode 100644 test.cfg delete mode 100644 testing.cfg diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index b7c56f8adf..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,79 +0,0 @@ -dist: trusty -language: python -sudo: true -python: - - "3.6" -if: tag IS blank -cache: - directories: - - eggs - - .npm - - ~/.npm - - node_modules - - /home/travis/.cypress/Cypress -addons: - apt: - packages: - - bsdtar - - build-essential - - make - - graphviz - - nginx -env: - global: - - PGPORT=5433 - - BOTO_CONFIG=/bogus - - PATH="/usr/share/elasticsearch/bin:/usr/lib/postgresql/11/bin:$PATH" - - ELASTIC_BEANSTALK_LABEL=$TRAVIS_COMMIT - - USER="4dn-dcic" - - SNO_REPO="snovault" -before_install: -- ls -dal /usr/lib/postgresql/*/bin/postgres -- find /usr/lib/postgresql -name 'postgres' -print -- ps auxww | grep postgres -- sudo apt-get install -yq --no-install-suggests --no-install-recommends postgresql-common -- sudo service postgresql stop -- sudo apt install -yq --no-install-suggests --no-install-recommends postgresql-11 postgresql-client-11 -- sudo service postgresql status -- sudo service postgresql start 11 -- sudo service postgresql status -- python -c "import fcntl; fcntl.fcntl(1, fcntl.F_SETFL, 0)" -- echo $tibanna_deploy -- postgres --version -- initdb --version -- ls -dal /usr/lib/postgresql/*/bin/postgres -- find /usr/lib/postgresql -name 'postgres' -print -- ps auxww | grep postgres -- nvm install 10 || (echo "Retrying nvm install" && sleep 5 && nvm install 10) -- node --version -- npm config set python /usr/bin/python2.7 -install: -- make build-full -before_script: - - configure-kibana-index --es-endpoint search-cgap-testing-6-8-vo4mdkmkshvmyddc65ux7dtaou.us-east-1.es.amazonaws.com:443 -script: -- make travis-test -after_script: -- echo leader=$BUILD_LEADER status=$BUILD_AGGREGATE_STATUS -- wipe-test-indices $TRAVIS_JOB_ID search-cgap-testing-6-8-vo4mdkmkshvmyddc65ux7dtaou.us-east-1.es.amazonaws.com:443 -after_success: -- coveralls -- ! "if test -n \"$NPM\"; then \n cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js\n - \ cat ./coverage/lcov.info | ./node_modules/.bin/codacy-coverage\n rm -rf ./coverage\nfi\n" -- if test -n "$UNIT"; then python-codacy-coverage -r coverage.xml; fi -- python deploy/travis_after_all.py https://api.travis-ci.org -- export $(cat .to_export_back) >> /dev/null -- echo $TRAVIS_COMMIT_MESSAGE -- MSG_NO_WHITESPACE="$(echo -e "${TRAVIS_COMMIT_MSG}" | tr -d '[:space:]')" -- echo $MSG_NO_WHITESPACE -- echo $TRAVIS_PULL_REQUEST -- echo $TRAVIS_BRANCH -- echo $tibanna_deploy -- | - if [[ $TRAVIS_BRANCH == 'master' ]]; then - echo 'Triggering docs build'; - curl -X POST -d "branches=master" -d "token=$DOCS_TOKEN" https://readthedocs.org/api/v2/webhook/cgap-portal/100087/; - fi -after_failure: -- python deploy/travis_after_all.py https://api.travis-ci.org -- export $(cat .to_export_back) >> /dev/null diff --git a/.travis.yml.old b/.travis.yml.old deleted file mode 100644 index 5fd778de26..0000000000 --- a/.travis.yml.old +++ /dev/null @@ -1,171 +0,0 @@ -language: python -sudo: true -cache: - pip: true - directories: - - eggs - - ".npm" - - node_modules -addons: - apt: - packages: - - oracle-java8-set-default - - bsdtar - - build-essential - - make - - graphviz - postgresql: "9.4" -env: - global: - - JAVA_HOME=/usr/lib/jvm/java-8-oracle - - BOTO_CONFIG=/bogus - - ES_VERSION=5.4.1 - - ES_DOWNLOAD_URL=https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}.deb - - PATH="/usr/share/elasticsearch/bin:/usr/lib/postgresql/9.4/bin:$PATH" - - ELASTIC_BEANSTALK_LABEL=$TRAVIS_COMMIT - - USER="4dn-dcic" - - SNO_REPO="snovault" - - secure: vovAkyWqRwZdzZYc061JuENHIPxnjek1t7TWnzeo8AwQdFQwhbt76z0XV9g5DvjYclUWdYxfOb7fiUyxc8M0M2wPfwU05uStewWTuBKILobaNqMdsWaxRQ9OSJ3NxqTLabnmIsAuCQPwHFqM6etyHXiMaAgK8EU+r23CRzklMY77vQMN/2Bs+3Y1ngZ17OVvdbmAwzlUS0uE7+P9X2YXz6QVeTvyPWnBr5SMZ/gyA1/LU+44svKLC85BLtG+2YWHwEOPl0+Jt8diOP+UMBVI0GkCl2cN0dLQhVP26Jcp479BpnPVMOsRqH+Ve9qxpfY6aesTjAVc3D7AJcKzlOnA9o7GTaVGj0pWV+eCv9n3pFacIJkTF1/JNI3lmqKlWyeRd1qMCgCi94mNXEKEgXfxVBSczALhZcxHWTMYo1jE1aDAaAj09yukMJ9UEIBVSQsAGywihlFvTtUTwZDpd0cSfl2RYx6z845HU5EX9oNASEYcbC237I83bObu232ljRokBixA5fo4y+TLNLwK97DsFNGocIXsPBKVBq8uCClD3S8kG9sB88vjM9bFnkenw7GfZLjJFChzd4Hw+x4xJQH0L99MbWeMYAl9R/ojqk3tVrpGEpXwmf/MOgTmVGPUEExgeRTSWUn95YrpenOx7FsJJQ4ZpD1rnVRev2X0A21l0H0= - - secure: VaAvhzAtXo69D1TthYVabBSu/ESHj+/Wura9+XuFA6aG27tWQSZJ1FWPBN0nax2/vs4MkZfW19qbXgxFZBE671eYiu53iXvi4JbJRJU6efG08gQH4wpsOhxg0WF4+uWIpbIok5dCUPVLh0nTkHBpFAlcgTn6O/ST7hYFliEUxwcRATBe6fslJcLpR6fmsnNVs6kwkv1MvkMikje3fCbfj+V8/dHqQRobwCbMlgJMuxV+MWx4RUfKimQ/oUGCef6YCzcilTEgW4fAdPLsxqZiGn8wihC3lpjNYyGOfX/NCzPo8iVtX49X+Dp+sePRxKgWvovwLQDCWkomMBpxtdea/bkkPg7v92G0ivCcYX8hs14WCWjkqScCk6B0XACyMmE7dCE2Xte86sarxGbsp5lp39OvQxAbFbK+QBEKfPOdoxkcM3rrhf+kAnq+PsH1HrzzGe1F3L0BTpeS9F8w26skgrXmR/6X78mMzLk2TtHmXf3qPpPcFEP+fEOfiZyJSLEvfti9fVjZWSkCDUDEuvgF5haYpKy+cgzMHyHIgJZxfokyRvY7mEWwDZzqnSGvvVx/km0LjkQUY0RCvoT5xAtFSC2AguTfJ1C1RlZvuuYxBHa3a7tFn+ahPfRtdtFpcEGjwKBDRTNY5xzKrzozXLVQHpPVYhFQndPSF2n3+0JwE6M= - - secure: Zi1Fg04BvfjBi54dvSs5vA82habY/SHakKnXd0Ogd3IMSzzNI5OWYAerNKhbz2iHqHx2Uy5Qz8JplICVw9Tbm2ENXma8xZfTONr264q9S6P4Pzc/xmoRXPXX7J4sqgzqDu7KoJJvan1jfdZVczQq1TIYrH/XxPz3gkEYlt4Jym3mvngb2t+Hd7I+6hwB/Wm/BO391R6CH7tc1KoKS05IylQCfjpdvbRMNFkAmZu59KJR5iSVbvHpzr2BePgGcvl+A9Hq6A6P886Wl/UuNfdROX3PT22PLy8Wl/6jRfGGyfU9gWT9IqT9glBNDQjb7fqk71Jy+DK3HWL56nRfAlXOShtA8Qev/zTgBJreMUtvcyNpW98PphAyQLE7AvbfqJ9/x5dOTTT7alNh2RTTxlzcnvEHRPwSmPqVibfKjqUQUhIa2iQ4qhfzBp6IjaeeEZY6VMplwm6qwr6Rp0J6FhZhZyzeLwulzOnBtaa/5paI54s+z6pYSib9PvKe3uRgNKwxR7o/Thqd1WtFH80o3gWaF+REeXwzpM9MODszaipvtiLci9br2BbNNcdFQREtCuf9X2RxG3nW9Jh2W/Wrvbo/39wbMaxVmmtRahFtq+MjhfZzN+ThCgRXk0JG/EjdOxaNjta5iZU962gUj3wr9yZNWVSUGrFMGkyIa6BRA3OZZ/M= - - secure: JXMLW6AmtlTVERT1e3XWaa6c2CrNKwlfEmcBBlMc11am2yeSp9yMEMd3ZnOI2CozeoHezBzdmTSqxm1uTuZT6U3lPrevU16ir/kdxzHa/K2NANAxQPPkhaSO0BjYEU03rAHm4n5HvutwRCNOuJFbE3XtPizSc6FQ3XOI9zq1RQQp2Dn5xXKbVBWEhzL+fHLx2AWfomprUU8O0a5TYY5xjaPYN9EBBieFhm5sI2N5/Sv95MSVbzfMa5VSNzHqwHeaZ8tt5L7Rv2otLLxUJblxRSiYkVyD1OZVU3YlrJGb+FkCFC8YDgz1RZ4mistCJtBG9/tRKztPBP93B/tb5g0wO2kZ6Mv71NHPXigWrXesBmu430GMvCnCYZgXVS4nT0cQHt7UyG0RU2LYwLzYr8Sn7StlEr5RL+TzDW28lHY4LK4+z+Gbk+ApxBinPZfpZCqvPYqvUiFBj0HOKxn6IC8ELZ3QHZd1XwkiPRTntfQ+lElg9M+aEvAKcN2FEE3o9RG6PG57xH7wIeQkeNMLSkLbREL3xXl3hWbgY7DOj7CyiokKyCAgM+1fpquIE7GKZdTBiAWl7z7y9iNYO9tDyLA08Cbp9nIU6llGFkEZ+QZWn5hkjYch36imEl7SBWYTANGsdNc1SNocWKK4pUhOycOokOS2fI7l5W/eBHTG/vlqUOw= - - secure: PotXkx2ykgYzVce32FZKUxkXazfiCZ0yqWLhosci27YnwOlCimH+XL97eM5enQFiU/YYI/ETYkhesuNuobTxFpXo383xIgt35rfD60Tp+sSmZTXOdveC2fnhqvm55/jysoMFN1nBMe/0VSOAFnMAzSX6qYvlsc9yC4LSTHiRyo7aev2iVKW5As1wmE9Cs+O0ee7azanckLXGPibIpijSf8D0IPu1w5CUR3/UBkL6rMQ3Ir8R8sAJpOtzW3O5Rx2F97swShhWa/gmk6zYu4RS+Q1Lr2MbVdlrrOgT/czkxsPIqIWmRP4L5nLS2L/wYtCiCatwUZm5LVP9bA6+vWbPagTOHm+2/VYqTn4QMiAbpRHZjLIQxQd067XOE/JsYrLUrCWquqEZ1XmT04gSI+rgvf0+oU0Wno19lgN0BArNOHG8GEPSugciB/I/tY6aV1jXiam5uRJc/tBgaQaRWiGRXKdFdB0BDbyKlcxxXq+1Fc6b//7qbeaioREj72z1aN37AKVfe+F+cx7gMX85OuFhyyX5ZV7S2i4fyrH4krlHTLWywdPVAMuggV0lITvSflhj8AE2U6A8yedAdt8I1UZDOkkns4HlCl8I7EXqCjGcLs2uYdiSTh1yzkKCYKSXpWH0nEUqr3j1cpghUJ3cjg1licQH2GzIIqlp7u/XTfody80= - - secure: CzDicgJ/x6ZaNWf7uqWpPsLRSC7gkQ6ssWUY1kRxtIYWbM7vEDGbq9MbuvnDOLAgs2tgkkXi5DzjfmrFWe5cK+OrOaqakZhrRf9yHmpgOFqlBXYutUMV4SOwGcUaTa+SVtryOqCBZkGtZaury5wIkHMz93gTKo1fhPU1xL5KTLg4xqhkfw2ppdSm+Zjj1M26GKd1/aKDIdTAmRs9ryiGb5vDnU7NEWzt2j+VT67a03cI7NDgncNknBoMrdbNYDQTnnsaIHhS6dRZYl6tfbYyV+ugKJMApLksrsnpXoKKQ58bYZHyDPmmwzHHaoULf+HojBTokz7Y9W0ATJ0Sip42tsfcLq5ij/XO2/Q/PnWOn0RneUCzzV/7H7+PNzJoN8dN97vS/8QLcLij/wbPQX3efpwUEdz2UbMy0Ybx8KgTS4MOoiDk+7dDHP9ksW6pGpto86iTBEX6dWyjbdMCGbD1SdSPH76XS1vizQ3+Fai1E/B8sjhxX1xGoAhhoxXgQ0mkgPEFfqKj/7+j4VC0EY+S6jZBaOoWR8AFt2eelJlFba+zk43DFKMK3aRMpqwrRjUo1QJvAd50pBOOuyqgrHVOLM6BIIe9TgMfUSnoMgUGgzyVWxYdb+0/5SNAozCoS1S+ZGBUkUHzq3gANU7soNMhsXmcNgxKKMhNG7ZIDaD52+0= - - secure: 0QRMNpI794/7ZRINcD6tuoPclBwzGTS5SwFMoaMvVJT2Kc4VdECyogHK/Kyg/f69vdAu6mLQ7v1rMlA98kPaftO1NZmQRfZXBKVGs8Id805VmvLzBeTlQm2/CeuWrBK1vQrgBlL9eyV5yDIbeAlUtNQ5xwKFK4S9Mu39sWbf3wIvwxReKH6/XvbIGjmDLF+eoKhfaBuf96WP8Ys76esn6r+rGNBjE0qwWZOcs7tTQI3GRK2qSPDP2FAksrO8u95jilJ+eN9Iy+FzBZVo4oJvsNpRSs/IJln+J02wZ6258b5jfFDFZZiC8J282HYmSJqH64ShxFywQGMJ8jmuQiuNEZ6XnZEzMRXWrKGJY1t/l8aI6T4UCIrlpASS58YC900xzzjhMbNiWaPAV4F27KD73DzePrtjrx6YlmWhtW2loON4v3UOD3C9J8+72LR/WjGrpCHaCy5tin0VxWv1JfCfBbcpvP/WzlnEbykFfs748FmnBe4pLUbRjmCuHsufBsdAn+IZgwhlEeMk+mZZK/8U+RoV5rxO0aQhILkcjUv6mBxuUfF4lxyHAdHHunD/LAPgFqi4Qy74Hgdz5z5wtJPRFf2mTjtysBFQbfSTk3EStZINPtKS9Yrq049lX0GCRO9nAZGoB89gWamEniT2FgeC6syTssB51/AuMDTSsf4qIOM= - - secure: GTDIQIzWqM+uIVttiJVjCljBgmBx9aJBLHqsRHjNVl0gZRP3OW6Az9tdGwrt7S2S0gw79cW26mYXfPaAQ141EGSeyUXLTG0c5Lw3OGYZmhUcPgvmi56QQ9JkremBgNo7RPZMS6GyDtHYoWjYSZQJhPa0fe83WTFGJQWZOXX+6Zm4x+wIYppi6yaEhKLGrcaoacxVHcKY31F/4HXg3EgSiAQaT+y3iitjyap4pToSgs6x5QW2vTOdgK8Qi4/CTSo3z4oNY590VbZZsP+i89UKgniar1+2SdSV2Ney1XJQtdsAhKlVY+M+5DaM9/y+dhs2ryYeuJlOP3ebCC/JKv7m1IOjRgkDIN7/ByaRk9ufU+mzY6XoWvXhpFuKO+u1LBjai/FvpV8aJPrmM3r4qSuQW6XI4TXsf3enrCJkljUhTzAo5X41rlrGy4rGHA8t0k8hFQK9eoO65dJth6HLasLuj4PaV5t6CJFAphnFPYqMo1EJfQ7tu6AILgpEg4Cnx2o+OYaXtKuILWQ6lluVef7HQ2l1A4pWPh734D9aY7s1W1ZsNhpD7mVuQE4M9qEOZyfZJPcLq41kYpjRfSCenidQZXqR3hpfElboFzGf2QFhgy8Cy6V+nYzqJBEPo9kLWj4K4u+KkCXborUSvIED9/n+CSUdRydMefMHqnSrtvOpAnA= - - secure: cMK2zu7UgtF56IRBVCIQOoSn2Wcrf7sc955z+lRcUy5dNkdObRDpLUl/ZKB095DIJ4HWQVhZ+ONoGQmaM6f5Mp0yJ628tM2A2rSlqjPVhnFpwYrFYk+T3mRMYi6Ldwmp9nGBYxI5FPlN1nuUDN0M1Oz3U9WGlctxuR9nwix8nY6Glpt8JTSKcGn8J9VZ6ZpwQ0qKKTWdme7X+LAJHZoBL3+W0UCSArr8OIe3bgSkN+ec+bduzr2+WB6NzuXgYs/58fGQ1nD5Uf1L7Bwy0Z9aNm5ENr8A2+Dx9UPNMpXUdoDGLwvGC2IUOV2YTcnHtqghpudfH+hB3kiI8z/whbgOrWGr1Icm2NgNZ9tbCS1QG2T7WlPTmvml9hsde1+zwCzDx3NUsALAwJf4f1Qi7lW1QnbCprQ8sctH6xTQh6nkHpt7/eFo0rTob+z3+HGcoEHol6gVhRXJcKNRjVxbpZg8yTXyO9viT6HpvqJFf/0O4rARXHpx/olEvBBUxXQl7h6wEDWynNol2W4WyWwzIYU8cVXr6nivQI6sfArXNFiyEv4J02d0RROnmSbnCVZlXM1vcNknAzsHLI/TS6SRqcg49NNTLWPuVaknVAQpdz9j8TgwtIx9xO7Mkl8rAXYKvwuNyz61R5f6o+y3AL+8ZwRCjLZ7tv3iWdpTVXHKg9O4ADI= - - secure: sMn4k+L9wspBTMYG0M8+n4ba1OY1ZtIG8zODm/dMHWPgodUwmC5E+HanKuTfo8URK7gYBOLpZ8OXzCTkOoCKqNYCKQihwaigAOx6sr1cC6TwFqppXhQGMe6Y9UlAaow2Xr10yIlZXMgKf6fJX3uXtdkXJ3XM7IqgB1zGCHACRFbR/aUzRjnachDV/XGmrMIfKi8MVJ1d6GK0VjIyIUFpAjDnfLf1q39xnq6vqeRIedB1wAdlLzFZ5tRNaPAK1j+a23VSwbqzAlg3cX2vBQ3b7IPeTU8a1hIoMh5IBP2ceEgwM2oWkvHmYzliVSn/SiWMYb7TRX/FxAwfqWws4uZ/mymOVQbCaBfK6RKF9UH/ldpAkA+AIxRLvQyN1UYWq2WweCmMiQtgmySBCzfmGrJMzRyZScW1kWzXOWTSzYqMEqU0xPk+vKYSwGHGctPqIsBrvhuow6dckHWssdWhGTtZIOb8bpePTOVGRCJcYPOUcZAX+VaJN2uT1wwUZ9vXiDKbXGxa5BuCqW1oOUC/fqSKVyrmxPXhA0TZUVDhDnzmcbHygvl5BlbpONIdUY/jYkM6LJpkkFivBsMSsEvVeEwUWT1kbo6apXpWLeD59zP6Hq/PAH+tXz0uer6SEcyo/M8XBUcWoZTniBjEI+WiCOnxcKeNLahxqZAbbnLJHl1OMF8= - - secure: ZjL66xRfoBX5/IJ+ClFkXMOQ50gqwR4GdLK+v2EfADHhtiO+Q/xjqDFybVhkz4ra4MognkWDX4+t/Td8viwIDBKvfmNxmVOUQiEtvUFdEDOVS3w6PacT5RHrcyL/7wUnF5ZKc4CH8idwxtr0lCmo3xBWzlxQgXU887Ro/pQwFDpscyTElYtMQMe3T3rLvBzaKRWyaNGAUvqAB6D632ZLB/xCvnoaUKXgfqyV8Em9x5jbTO/++/S1qrQiPBhGK6Ggwh/xBtmRxLTeNtEJPqF6ntp7yV9HMLd8vyk1xsscdFH6xJ7wRTgsM0VC7jB+NvXjAdQgPybHZYw5+5Xg8C+D7fHPZbCrnF299QvGF8AAChKNR1rkOAXuPdttDyhTKYbZHnRFniiDQju3PDfdnQ6aqKh4VUViPcliLsV5pYodgo09ubpBvrQm7v8esgZDBtCPJALJPmlpL3cU9fRVc10l4qTRxFuG9mNAb0uRICocmQNwYdBaSgKCZHSkcnfSxUglLPT5tqT3JfPMlRCDEzb2AlIrjIOnxv0MJVvo4xUybJK/00bEFWq68mUqCq3HXM4pdthE9fxpZyYAHmaQNE5YpRwMdD2aJHA7C83eggx0zMqE+S9O2L97io4ytiOs0rTYHDLkcBhds3TOvIqZHRNgOj4xkiB3g1KsaZpXpRzkDkY= -matrix: - include: - - python: '3.4' - env: NPM=Test - - python: '3.4' - env: UNIT=Test -notifications: - slack: - secure: DQyUTk6evwPpO0P5+OPhBSgl+fGEerOjBlBwQliXAkDaMKH3Cpi4cTQ3ETR/+3g6bGqPGK+QJ1R+6Ht+hJD7dNomyVIoQmvF0P5afJtpk/A3cDILe90t76ET9jc/iBjWeUFQdokFUJ7Gt1GGYtI6e5XcVu/Qc/xqCvFMtsBN6mnBFuvcQki+WkoIIPPawyhfryCNOLo5vvbi4SZ5dZI+M0MGq9HQjOyF1sdIgKuXhxjupmL+kVPVXAq9kiOANYFkwbNsP9j0BTYW5wFHpAztBqz3NT6EchgCdue8tgV4hC4rRFvM/bsA4qz/TJ5wRRLBRXtnNtPcyhAqTiU+wC6qWt++fjSEMGe4KYqtRRV1YuxQiTVHN84t77DILLNfMh/6uSs0KijwOT+Oazwd/UsN1zYOH93AM4FZ0h92yyb6j6JQ+DUk4WFel1Zr4kZzhtHSPGw+K4fxY0zIt1qpaDPXjtZHQ0+LwIIMtwMp5bBcwDn9d1ADnUhUAuuIN2hHaXrVy4vP2hIcd0LzezBqvc7JXimyd5yRgUeOCTrKGkAeSo8VA7XIj0ZmlpQRYKNTJP+gz4Y6C/RCxISnFDF/vcX+IsDdvreZXJMplE1Aqxf0uR6Zj8Sr8q+QWGKydv6ettlLZuqDuv0l/l/9qpzYfRqLSZcetGRVFHHcR9wuQiDyums= -before_install: -- SNO_PATH="snovault = git https://github.com/$USER/$SNO_REPO.git branch=" -- SNO_BRANCH=$(grep "${SNO_PATH}" buildout.cfg | sed "s@$SNO_PATH@@") -- SNO_STATUS=$(curl -s "https://api.travis-ci.org/$USER/$SNO_REPO.svg?branch=$SNO_BRANCH" | grep pass) -- | - if [ -z "$SNO_STATUS" ]; then - echo "Snovault branch build for $SNO_BRANCH is failing; exiting build" - travis_terminate - else - echo "Snovault branch $SNO_BRANCH is okay with build status: $SNO_STATUS" - fi -- echo $tibanna_deploy -- postgres --version -- initdb --version -- nvm install 4 -- node --version -- npm config set python /usr/bin/python2.7 -install: -- curl -O ${ES_DOWNLOAD_URL} -- sudo dpkg -i --force-confnew elasticsearch-${ES_VERSION}.deb -- python bootstrap.py --buildout-version 2.4.1 --setuptools-version 18.1 -- bin/buildout || (echo "Retrying buildout" && bin/buildout ) -- pip install awsebcli -- pip install coveralls -- pip install codacy-coverage -before_script: -- | - if test -n "$BROWSER"; then - CONNECT_URL=https://saucelabs.com/downloads/sc-4.3.8-linux.tar.gz - CONNECT_DOWNLOAD=sc.tar.gz - SC_READYFILE=sauce-connect-ready-$RANDOM - SC_LOGFILE=$HOME/sauce-connect.log - SC_PIDFILE=$HOME/sauce-connect.pid - curl $CONNECT_URL > $CONNECT_DOWNLOAD - mkdir sc - tar -zxf $CONNECT_DOWNLOAD --strip 1 --directory sc - sc/bin/sc --readyfile $SC_READYFILE \ - --logfile $SC_LOGFILE \ - --pidfile $SC_PIDFILE \ - --tunnel-identifier $TRAVIS_JOB_NUMBER \ - --user $SAUCE_USERNAME --api-key $SAUCE_ACCESS_KEY > /dev/null & - while test -f "$SC_PIDFILE" && test ! -f "$SC_READYFILE"; do sleep .5; done - fi -- sudo service elasticsearch stop -script: -- if test -n "$NPM"; then npm test; fi -- if test -n "$NPM"; then bin/test -v -v -m "not bdd and setone" --durations=40 --cov - src/encoded; fi -- if test -n "$UNIT"; then bin/test -v -v -m "working and not setone" --durations=40 - --cov src/encoded; fi -- | - if test -n "$BROWSER"; then - test -f "$SC_PIDFILE" && bin/test -v -v -m "bdd" --tb=short \ - --splinter-implicit-wait 10 \ - --splinter-webdriver remote \ - --splinter-remote-url "http://$SAUCE_USERNAME:$SAUCE_ACCESS_KEY@localhost:4445/wd/hub" \ - --splinter-socket-timeout 300 \ - --browser-arg tunnel-identifier "$TRAVIS_JOB_NUMBER" \ - --browser-arg-int build "$TRAVIS_BUILD_NUMBER" \ - --browser-arg-int idleTimeout 300 \ - --browser-arg name "$TRAVIS_REPO_SLUG $TRAVIS_BRANCH $TRAVIS_COMMIT" \ - --browser-arg browser "$BROWSER" - --cov src/encoded - fi -after_script: -- echo leader=$BUILD_LEADER status=$BUILD_AGGREGATE_STATUS -- | - if test -f "$SC_PIDFILE"; then - SAUCE_JOB_ID=`grep -m 1 /session/ "$HOME/sauce-connect.log" | cut -d / -f 7` - SAUCE_PASSED=`((TRAVIS_TEST_RESULT == 0)) && echo true || echo false` - curl -H "Content-Type:text/json" -s -X PUT -d "{\"passed\": $SAUCE_PASSED}" \ - "http://$SAUCE_USERNAME:$SAUCE_ACCESS_KEY@saucelabs.com/rest/v1/$SAUCE_USERNAME/jobs/$SAUCE_JOB_ID" > /dev/null - echo "Sauce test page https://saucelabs.com/tests/$SAUCE_JOB_ID" - kill $(cat "$SC_PIDFILE") - wait $(cat "$SC_PIDFILE") - fi -after_success: -- coveralls -- "if test -n \"$NPM\"; then \n cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js\n - \ cat ./coverage/lcov.info | ./node_modules/.bin/codacy-coverage\n rm -rf ./coverage\nfi\n" -- if test -n "$UNIT"; then python-codacy-coverage -r coverage.xml; fi -- python deploy/travis_after_all.py https://api.travis-ci.org -- export $(cat .to_export_back) >> /dev/null -- echo $TRAVIS_COMMIT_MESSAGE -- MSG_NO_WHITESPACE="$(echo -e "${TRAVIS_COMMIT_MSG}" | tr -d '[:space:]')" -- echo $MSG_NO_WHITESPACE -- echo $TRAVIS_PULL_REQUEST -- echo $TRAVIS_BRANCH -- echo $tibanna_deploy -- | - if [ "$BUILD_LEADER" = "YES" ]; then - if [ "$BUILD_AGGREGATE_STATUS" = "others_succeeded" ]; then - if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then - bin/aws s3 cp --recursive s3://encoded-4dn-conf-prod/.aws /home/travis/.aws - git remote add origin-travis https://${GH_TOKEN}@github.com/4dn-dcic/fourfront.git - git config --global user.email "travis@builder.com" - git config --global user.name "Travis" - if [ "$TRAVIS_BRANCH" = "master" ]; then - git config --replace-all remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*' - git fetch - if [ -z "$tibanna_deploy" ]; then - echo "Job succeeded! PUBLISHING to fourfront-mastertest environment..." - eb use fourfront-mastertest --debug - else - echo "Job succeeded! PUBLISHING to ${tibanna_deploy} environment..." - eb use ${tibanna_deploy} --debug - fi - travis_wait 30 python deploy/deploy_beanstalk.py - fi - if [ "$TRAVIS_BRANCH" = "production" ]; then - echo "Tests passed! PUBLISHING to ${tibanna_deploy} environment..." - eb use ${tibanna_deploy} --debug - travis_wait 30 python deploy/deploy_beanstalk.py --prod - fi - fi - fi - fi -after_failure: -- python deploy/travis_after_all.py https://api.travis-ci.org -- export $(cat .to_export_back) >> /dev/null -notification: - slack: - rooms: - secure: YuLIoKULHxT7ZY9nKVme0pVELk1HBS/nPjYYu4Bs48lOgJVK2I/OyvV9HVx2g7raK6/s4T7wTXG0EmljcakmKY78wsSyhR79fAZcV7+p4SobEi7nGHuHXVq0sp21ZeYiFJvD/5ospHX0hMudSicaANT66X7EyLTMdG/NGu5it6TPjcJWhxERXXz6Zbizm6j/wv2a64R0mMUGFSe1aWrFZv5ft+GgfupM2TPwX0PxNl2DamaGowJriKuU+svqIP7LXGuzYBxf9sj08gf3/S0e1r2bzg/ik4clzM2Lp5mcU27nLWh6TUr3hthFNsaKAFwIPS6Ay1Z8GeotAiZC538fyeM1zDvjz0Uu5fEuZnI89NVZ734NZfoFCEg4JcPDs8RHgzreR209zRgCVUh8AI7m7vMv302MNAjbG42FPlB1gDg9bukxbyytccm5/TuzAGLktG1jCWJk0uU34JsWZakF9fSqmFckppXZ1472cPQT+ityuszyUGEMgLvNxpZDpwMhywN7eMuQu64YD4ex1EvbidNEIdb0Z5UnpXaC2H+vxdBb+Qg58r/HN/S6OW8H+C/pZMuRMdm1AcgOjS9hROcRKgmwSD53Z/XXrT4GYZTW48jNOCFE2O+6RXUGqvQzcH8JG5blnTxoCV8dtb9GDbORQm7b80MOiidG8Xi8Jyq8oPM= diff --git a/add_route b/add_route deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/deploy/docker/local/Dockerfile b/deploy/docker/local/Dockerfile index f4e8397f3a..9f558e0c89 100644 --- a/deploy/docker/local/Dockerfile +++ b/deploy/docker/local/Dockerfile @@ -8,8 +8,8 @@ MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" # Build Arguments ARG CGAP_ENV_NAME ENV CGAP_ENV_NAME=${CGAP_ENV_NAME:-"cgap-docker-will-test"} -ARG RUN_TEST -ENV RUN_TEST=${RUN_TEST:-1} +#ARG RUN_TEST +#ENV RUN_TEST=${RUN_TEST:-1} # Configure (global) Env ENV CGAP_REPO=https://github.com/dbmi-bgm/cgap-portal.git @@ -25,7 +25,7 @@ ENV PYTHONFAULTHANDLER=1 \ PIP_DEFAULT_TIMEOUT=100 \ POETRY_VERSION=1.1.4 -# Install nginx (unused for local deployment as this runs as part of `make deploy1`) +# Install nginx COPY install_nginx.sh / RUN bash /install_nginx.sh diff --git a/deploy/docker/local/development.ini b/deploy/docker/local/development.ini index 9d365c34ca..377a3b6e9b 100644 --- a/deploy/docker/local/development.ini +++ b/deploy/docker/local/development.ini @@ -27,7 +27,7 @@ pyramid.debug_routematch = false pyramid.default_locale_name = en # this line determines which load function is used in load_data # most deployments use: "load_test_data = encoded.loadxl:load_test_data" -load_test_data = encoded.loadxl:load_local_data +load_test_data = encoded.loadxl:load_prod_data encoded_version = 100.200.300 snovault_version = 200.300.400 utils_version = 300.400.500 diff --git a/deploy/docker/local/entrypoint.sh b/deploy/docker/local/entrypoint.sh index 7ed9d5239b..83f54cdfe8 100644 --- a/deploy/docker/local/entrypoint.sh +++ b/deploy/docker/local/entrypoint.sh @@ -1,6 +1,6 @@ #!/bin/sh -if [ $RUN_TEST -eq "0" ]; then +if [ -z ${RUN_TEST+x} ]; then # Clear db/es since this is the local entry point poetry run clear-db-es-contents development.ini --app-name app --env $CGAP_ENV_NAME @@ -15,7 +15,7 @@ if [ $RUN_TEST -eq "0" ]; then service nginx start # Start application - make deploy2 + pserve development.ini --reload else diff --git a/deploy/docker/docker-compose.yml b/docker-compose.yml similarity index 66% rename from deploy/docker/docker-compose.yml rename to docker-compose.yml index c4e6e593f8..e815573b0f 100644 --- a/deploy/docker/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: # Connect to it in development.ini with # sqlalchemy.url = postgresql://postgres:postgres@db:5432/postgres db: - build: ./postgres + build: ./deploy/docker/postgres container_name: pg1 environment: POSTGRES_DB: postgres @@ -20,24 +20,24 @@ services: # Disabled for now as too compute intensive # To connect use: # elasticsearch.server = es:9200 -# es: -# build: ./elasticsearch -# container_name: es1 -# environment: -# - node.name=es01 -# - cluster.name=es-docker-cluster -# - bootstrap.memory_lock=true -# - "ES_JAVA_OPTS=-Xms512m -Xmx512m" -# ulimits: -# memlock: -# soft: -1 -# hard: -1 -# ports: -# - "9200:9200" + # es: + # build: ./elasticsearch + # container_name: es1 + # environment: + # - node.name=es01 + # - cluster.name=es-docker-cluster + # - bootstrap.memory_lock=true + # - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + # ulimits: + # memlock: + # soft: -1 + # hard: -1 + # ports: + # - "9200:9200" # Main Application Component app: - build: ./local + build: ./deploy/docker/local container_name: cgap command: "/home/cgap-admin/cgap-portal/entrypoint.sh" environment: @@ -45,6 +45,8 @@ services: AWS_SECRET_ACCESS_KEY: "$AWS_SECRET_ACCESS_KEY" ports: - "8000:8000" # nginx proxy port (note application traffic is not forwarded) + volumes: + - "./src:/home/cgap-admin/cgap-portal/src" depends_on: - db #- es diff --git a/magic_fix b/magic_fix deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/nginx.yml b/nginx.yml deleted file mode 100644 index b67777e149..0000000000 --- a/nginx.yml +++ /dev/null @@ -1,78 +0,0 @@ -#cloud-config - -# Launch instance with a network interface configured to the local IP addresses we references in the config. -# Specify availability zone as we have a different subnet in each one. -# $ aws --profile production ec2 run-instances --user-data file://nginx.yml --iam-instance-profile Name="proxy" --image-id ami-5189a661 --instance-type t2.medium --region us-west-2 --placement AvailabilityZone=us-west-2c --network-interfaces '[{"DeviceIndex": 0, "Groups": ["sg-4ae7be2f"], "SubnetId": "subnet-201d1766", "PrivateIpAddresses": [{"PrivateIpAddress": "172.31.2.70", "Primary": true}, {"PrivateIpAddress": "172.31.2.78", "Primary": false}, {"PrivateIpAddress": "172.31.2.79", "Primary": false}]}]' - -# Choose instance size based on network performance required. - -# After boot associate elastic ip addresses with the private ip addresses -# $ aws --profile production ec2 associate-address --region us-west-2 --public-ip 52.11.61.187 --private-ip-address 172.31.2.70 --allow-reassociation --instance-id -# $ aws --profile production ec2 associate-address --region us-west-2 --public-ip 52.88.20.43 --private-ip-address 172.31.2.78 --allow-reassociation --instance-id -# $ aws --profile production ec2 associate-address --region us-west-2 --public-ip 52.32.76.4 --private-ip-address 172.31.2.79 --allow-reassociation --instance-id - -apt_sources: -- source: ppa:nginx/stable - -bootcmd: -- cloud-init-per once ssh-users-ca echo "TrustedUserCAKeys /etc/ssh/users_ca.pub" >> /etc/ssh/sshd_config - -output: - all: '| tee -a /var/log/cloud-init-output.log' - -package_upgrade: true - -packages: -- awscli -- curl -- dnsmasq -- nginx-full -- ntp -- unattended-upgrades -- update-notifier-common - -power_state: - mode: reboot - -runcmd: -- aws --region us-east-1 s3 cp s3://encoded-conf-proxy/ssl.tgz ssl.tgz -- mkdir -p /etc/nginx/ssl -- tar -zxf ssl.tgz --directory /etc/nginx/ssl -# Generate a big prime number for DH SSL (takes a few minutes.) -- openssl dhparam 2048 -out /etc/nginx/ssl/dhparam.pem -- chmod 600 /etc/nginx/ssl/dhparam.pem -- curl -o /etc/nginx/nginx.conf https://raw.githubusercontent.com/ENCODE-DCC/encoded/master/encode-proxy-nginx.conf -# The final octets of the local IP addresses configured above match the public IPs -- sed -i.bak s/171.67.205./172.31.2./g /etc/nginx/nginx.conf - -write_files: -- path: /etc/apt/apt.conf.d/20auto-upgrades - content: | - APT::Periodic::Update-Package-Lists "1"; - APT::Periodic::Unattended-Upgrade "1"; - -- path: /etc/apt/apt.conf.d/50unattended-upgrades - content: | - Unattended-Upgrade::Allowed-Origins { - "${distro_id} ${distro_codename}-security"; - }; - Unattended-Upgrade::Automatic-Reboot "true"; - -- path: /etc/motd - content: | - ######################################### - ## Nginx proxy server ## - ## For demo instances: ## - ## ssh .instance.encodedcc.org ## - ######################################### - -- path: /etc/network/interfaces.d/eth0.cfg - content: | - # The primary network interface - auto eth0 - iface eth0 inet dhcp - post-up ip addr add 172.31.2.78/20 dev eth0 - post-up ip addr add 172.31.2.79/20 dev eth0 - -- path: /etc/ssh/users_ca.pub - content: ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAv/ymOcnN4LhM4NACc3Or116XXJ6KytuOgB/+1qNkOFBqBosrn7cmJ35rsoNHRgYNrCsRE9ch74RKsN6H72FtSJgBhGh/9oUK7Os6Fqt3/ZZXxgxIx6ubs/MTgrxrAnujiBxUXMXQhLKMriNMpo8mt4nGYVtLk9PBjiyfncaS8H9ZKoNio9dhP8bmTuYvioAI35dqKdSlVLyzr/XkZxia8Ki+pQ0N6uuiEwMR3ToM+LSp8wpFOOAiu4PEAujRW7us/+1hlpKWfn0J7/V3826joHE+I967Vg/+ikcVhF77JjK1nib879VgCWfmn1HPQosIpk4yJfVgGvRVI7I2nfBPVw== encoded@demo-l.encodedcc.org diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index ad769b40ac..0000000000 --- a/setup.cfg +++ /dev/null @@ -1,16 +0,0 @@ -[flake8] -max-line-length = 160 -[coverage:run] -branch = True -omit = - */*env/* - */site-packages/* - */.cache/* - */.git/* - */.idea/* - */*.egg-info/* - */encode_schemas/* - */encode_types/* - */tests/* - */docs/* - */commands/* diff --git a/test.cfg b/test.cfg deleted file mode 100644 index 7b3f6c01aa..0000000000 --- a/test.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[buildout] -extends = buildout.cfg diff --git a/testing.cfg b/testing.cfg deleted file mode 100644 index 7f802f1dd2..0000000000 --- a/testing.cfg +++ /dev/null @@ -1,8 +0,0 @@ -[buildout] -extends = buildout.cfg - -[production-ini] -create_tables = true -load_test_data = encoded.loadxl:load_prod_data - - From a58bc8bdadc665472ffa0ba04d0033ea1229b810 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Mon, 1 Mar 2021 14:25:16 -0500 Subject: [PATCH 021/120] provision a bunch more --- .gitignore | 2 + Makefile | 14 +++- README.rst | 17 +++++ deploy/docker/local/Dockerfile | 8 +-- deploy/docker/local/entrypoint.sh | 22 ++++--- deploy/docker/production/Dockerfile | 91 ++++++++++++++++++++++++++ deploy/docker/production/entrypoint.sh | 17 +++++ deploy/docker/production/nginx.conf | 43 ++++++++++++ docker-compose.yml | 17 ++++- docs/source/docker-local.rst | 15 +++-- docs/source/index.rst | 1 + 11 files changed, 225 insertions(+), 22 deletions(-) create mode 100644 deploy/docker/production/Dockerfile create mode 100644 deploy/docker/production/entrypoint.sh create mode 100644 deploy/docker/production/nginx.conf diff --git a/.gitignore b/.gitignore index 10d02188d8..a5420f0889 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,8 @@ beanstalk.cfg /src/encoded/static/css/style.css /src/encoded/static/scss/_variables.original.scss +/src-docker/* + /src/encoded/tests/data/temp-local-inserts/* !/src/encoded/tests/data/temp-local-inserts/.gitkeep diff --git a/Makefile b/Makefile index a1ca449539..9f5c0c4e49 100644 --- a/Makefile +++ b/Makefile @@ -150,10 +150,16 @@ update: # updates dependencies poetry update build-docker: - pushd deploy/docker/ && docker-compose build && popd + docker-compose build + +build-docker-clean: + docker-compose build --no-cache deploy-docker: - pushd deploy/docker/ && docker-compose up + docker-compose up + +deploy-docker-daemon: + docker-compose up -d help: @make info @@ -181,3 +187,7 @@ info: $(info - Use 'make test' to run tests with normal options we use on travis ('-m "working and not manual"').) $(info - Use 'make test-any' to run tests without marker constraints (i.e., with no '-m' option).) $(info - Use 'make update' to update dependencies (and the lock file).) + $(info - Use 'make build-docker' to build the local Docker image.) + $(info - Use 'make build-docker-clean' to build the local Docker image with no cache.) + $(info - Use 'make deploy-docker' start up the cluster - pserve output will follow if successful.) + $(info - Use 'make deploy-docker-daemon' will start the cluster in daemon mode.) diff --git a/README.rst b/README.rst index 84e5cf01ff..ed866d9a20 100644 --- a/README.rst +++ b/README.rst @@ -27,3 +27,20 @@ Welcome to CGAP! We are a team of scientists, clinicians, and developers who aim * `cgapwolf `_ for workflow development Be warned that features are under active development and may not be stable! Visit the production deployment for the best experience. For installation and more information on getting started, see our `documentation page `_. + +For information on how to run CGAP with Docker, see `here. <./docs/docker-setup.rst>`_ + +Navigating this Repository +^^^^^^^^^^^^^^^^^^^^^^^^^^ + * .ebextensions/ contains Beanstalk deployment scripts + * .github/ contains Github Action Workflows + * bin/ contains the few remaining executables + * deploy/docker contains Docker setups (see docker-compose.yml) + * docs/ contains documentation + * etc/ is deprecated/unused + * examples/ is deprecated/unused + * jest/ is deprecated/unused + * parts/ contains production WSGI entry points + * scripts/ contains misc scripts + * src/ where the code is + * src-docker/ mount point where container code is \ No newline at end of file diff --git a/deploy/docker/local/Dockerfile b/deploy/docker/local/Dockerfile index 9f558e0c89..4af3501eb9 100644 --- a/deploy/docker/local/Dockerfile +++ b/deploy/docker/local/Dockerfile @@ -38,13 +38,12 @@ RUN apt-get install -y ca-certificates nodejs npm # Configure CGAP User WORKDIR /home/nginx RUN mkdir -p /home/nginx/cgap-portal -RUN chown nginx /home/nginx/cgap-portal # Configure venv ENV VIRTUAL_ENV=/opt/venv RUN python -m venv /opt/venv ENV PATH="$VIRTUAL_ENV/bin:$PATH" -RUN chown nginx /opt/venv +RUN chown -R nginx:nginx /opt/venv # Copy to /home/nginx/cgap-portal RUN git clone $CGAP_REPO --branch $CGAP_BRANCH @@ -56,7 +55,7 @@ RUN pip install poetry==1.1.4 wheel==0.29.0 RUN poetry install RUN python setup_eb.py develop RUN make fix-dist-info -RUN npm install --no-fund --no-progress --python=/opt/venv/bin/python +RUN npm ci --no-fund --no-progress --python=/opt/venv/bin/python RUN npm run build RUN npm run build-scss RUN make aws-ip-ranges @@ -85,6 +84,7 @@ COPY development.ini test.ini COPY entrypoint.sh . RUN chmod +x entrypoint.sh EXPOSE 8000 -USER nginx +# Uncomment this to run as the unprivileged user +# USER nginx ENTRYPOINT ["/home/nginx/cgap-portal/entrypoint.sh"] diff --git a/deploy/docker/local/entrypoint.sh b/deploy/docker/local/entrypoint.sh index 83f54cdfe8..fbfe5546d8 100644 --- a/deploy/docker/local/entrypoint.sh +++ b/deploy/docker/local/entrypoint.sh @@ -1,15 +1,19 @@ #!/bin/sh -if [ -z ${RUN_TEST+x} ]; then +if [ -z ${TEST+x} ]; then - # Clear db/es since this is the local entry point - poetry run clear-db-es-contents development.ini --app-name app --env $CGAP_ENV_NAME + if [ ! -z ${LOAD+x} ]; then - # Create mapping - poetry run create-mapping-on-deploy development.ini --app-name app + # Clear db/es since this is the local entry point + poetry run clear-db-es-contents development.ini --app-name app --env $CGAP_ENV_NAME - # Load Data (based on development.ini, for now just master-inserts) - poetry run load-data development.ini --app-name app --prod + # Create mapping + poetry run create-mapping-on-deploy development.ini --app-name app + + # Load Data (based on development.ini, for now just master-inserts) + poetry run load-data development.ini --app-name app --prod + + fi # Start nginx proxy service nginx start @@ -19,6 +23,8 @@ if [ -z ${RUN_TEST+x} ]; then else - make test + echo "Not starting serving application" + echo "Enter the container with docker exec" + sleep 100000000 fi diff --git a/deploy/docker/production/Dockerfile b/deploy/docker/production/Dockerfile new file mode 100644 index 0000000000..e6accbb704 --- /dev/null +++ b/deploy/docker/production/Dockerfile @@ -0,0 +1,91 @@ +# CGAP-Portal Dockerfile + +# TODO: appropriately pin/verify this image +FROM python:3.6.12-buster + +MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" + +# Build Arguments +ARG CGAP_ENV_NAME +ENV CGAP_ENV_NAME=${CGAP_ENV_NAME:-"fourfront-cgapmastertest"} +ARG CGAP_BRANCH +ENV CGAP_BRANCH=${CGAP_BRANCH:-"master"} + +# Configure (global) Env +ENV CGAP_REPO=https://github.com/dbmi-bgm/cgap-portal.git +ENV NGINX_USER=nginx +ENV DEBIAN_FRONTEND=noninteractive +ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1 +ENV PYTHONFAULTHANDLER=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONHASHSEED=random \ + PIP_NO_CACHE_DIR=off \ + PIP_DISABLE_PIP_VERSION_CHECK=on \ + PIP_DEFAULT_TIMEOUT=100 \ + POETRY_VERSION=1.1.4 + +# Install nginx +COPY install_nginx.sh / +RUN bash /install_nginx.sh + +# Intall things needed for our system +RUN apt-get update +RUN apt-get install -y curl vim emacs postgresql-client net-tools +RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - +RUN apt-get install -y ca-certificates nodejs npm + +# Configure CGAP User +WORKDIR /home/nginx +RUN mkdir -p /home/nginx/cgap-portal + +# Configure venv +ENV VIRTUAL_ENV=/opt/venv +RUN python -m venv /opt/venv +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +RUN chown -R nginx:nginx /opt/venv + +# Copy to /home/nginx/cgap-portal +RUN git clone $CGAP_REPO --branch $CGAP_BRANCH + +# Build the application +WORKDIR /home/nginx/cgap-portal +RUN pip install --upgrade pip +RUN pip install poetry==1.1.4 wheel==0.29.0 +RUN poetry install +RUN python setup_eb.py develop +RUN make fix-dist-info +RUN npm ci --no-fund --no-progress --no-optional --no-audit --python=/opt/venv/bin/python +RUN npm run build +RUN npm run build-scss +RUN make aws-ip-ranges +RUN cat /dev/urandom | head -c 256 | base64 > session-secret.b64 + +# Copy config files in (down here for quick debugging) +# Remove default configuration from Nginx +RUN rm /etc/nginx/nginx.conf +RUN rm /etc/nginx/conf.d/default.conf +COPY nginx.conf /etc/nginx/nginx.conf +# give nginx user permissions +RUN chown -R nginx:nginx /var/cache/nginx && \ + chown -R nginx:nginx /var/log/nginx && \ + chown -R nginx:nginx /etc/nginx/conf.d +RUN touch /var/run/nginx.pid && \ + chown -R nginx:nginx /var/run/nginx.pid +RUN rm -f /var/log/nginx/* +RUN touch /var/log/nginx/access.log && \ + chown -R nginx:nginx /var/log/nginx/access.log +RUN touch /var/log/nginx/error.log && \ + chown -R nginx:nginx /var/log/nginx/error.log + +# Copy over ini file, entrypoint +# TODO ini file must be generated +COPY production.ini . +COPY entrypoint.sh . +RUN chmod +x entrypoint.sh +EXPOSE 8000 +# Container does not ever run as root +# This way even if someone gets access to the container they cannot +# read production.ini for example +USER nginx + +ENTRYPOINT ["/home/nginx/cgap-portal/entrypoint.sh"] diff --git a/deploy/docker/production/entrypoint.sh b/deploy/docker/production/entrypoint.sh new file mode 100644 index 0000000000..097f1a37ac --- /dev/null +++ b/deploy/docker/production/entrypoint.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +# Clear db/es since this is the local entry point +poetry run clear-db-es-contents production.ini --app-name app --env $CGAP_ENV_NAME + +# Create mapping +poetry run create-mapping-on-deploy production.ini --app-name app + +# Load Data (based on development.ini, for now just master-inserts) +poetry run load-data production.ini --app-name app --prod + +# Start nginx proxy +service nginx start + +# Start application +pserve production.ini + diff --git a/deploy/docker/production/nginx.conf b/deploy/docker/production/nginx.conf new file mode 100644 index 0000000000..39d5b1e276 --- /dev/null +++ b/deploy/docker/production/nginx.conf @@ -0,0 +1,43 @@ +# Production nginx setup +# For now, this looks exactly like the local deployment +# It certainly needs updating + +error_log /var/log/nginx/error.log warn; + +events { + worker_connections 2048; +} +http { + resolver 8.8.8.8; + upstream app { + server 0.0.0.0:6543; + keepalive 10; + } + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + server { + listen 8000; + access_log /var/log/nginx/access.log main; + location / { + # Normalize duplicate slashes + if ($request ~ ^(GET|HEAD)\s([^?]*)//(.*)\sHTTP/[0-9.]+$) { + return 301 $2/$3; + } + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_pass http://app; + proxy_set_header Connection ""; + } + location ~ ^/_proxy/(.*)$ { + internal; + proxy_set_header Authorization ""; + proxy_set_header Content-Type ""; + proxy_buffering off; + proxy_pass $1$is_args$args; + } + } +} diff --git a/docker-compose.yml b/docker-compose.yml index e815573b0f..5f23f21350 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,6 +36,11 @@ services: # - "9200:9200" # Main Application Component + # Options: + # * TEST - Just build (and not serve the app). Comment this to enable local deployment. + # * LOAD - Trigger a reload of the inserts in the database. + # After running once, comment this out so the container will serve without reloading inserts. + # Has no effect if TEST is specified. app: build: ./deploy/docker/local container_name: cgap @@ -43,10 +48,20 @@ services: environment: AWS_ACCESS_KEY_ID: "$AWS_ACCESS_KEY_ID" AWS_SECRET_ACCESS_KEY: "$AWS_SECRET_ACCESS_KEY" + TEST: "true" # default: run local deployment + LOAD: "true" # default: load the inserts (comment this out after first run) ports: - "8000:8000" # nginx proxy port (note application traffic is not forwarded) volumes: - - "./src:/home/cgap-admin/cgap-portal/src" + - 'src-docker:/home/nginx/cgap-portal/src' # mount source directory in container to mount point depends_on: - db #- es + +volumes: + src-docker: + driver: local + driver_opts: + type: none + o: bind + device: "./src-docker" diff --git a/docs/source/docker-local.rst b/docs/source/docker-local.rst index b26dadee6e..da238cae7c 100644 --- a/docs/source/docker-local.rst +++ b/docs/source/docker-local.rst @@ -24,7 +24,14 @@ and the client. To access the running container:: $ docker ps # will show running containers - $ docker exec -it /bin/bash + $ docker exec -it bash + +Container Development +--------------------- + +When the container is first built, a volume is created in the local repository location called ``src-docker``. +The ``src`` directory of the repository within the container is mounted to this location, allowing you to make +modifications to the source there and trigger rebuilds via bash as appropriate. Advanced Usage -------------- @@ -32,9 +39,3 @@ Advanced Usage There are several useful commands documented below that may be helpful when issues are encountered or changes need to be made. * ``docker-compose build --no-cache`` # will force a full rebuild of the entire image - -Configuration Notes -------------------- - -* TODO write me - diff --git a/docs/source/index.rst b/docs/source/index.rst index dbf1e7a8be..a8c1a01c15 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -41,6 +41,7 @@ Be warned that features are under active development and may not be stable! Visi infrastructure_overview dataflow_overview ingestion + docker-local *Data Model* From 91961f9adcc163441027de0cc56e6857c5b05cd5 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 2 Mar 2021 07:59:52 -0500 Subject: [PATCH 022/120] fix test db setup --- Makefile | 6 +++--- src/encoded/tests/conftest.py | 23 +++++++++++++++++++++-- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 9f5c0c4e49..15c10d63a0 100644 --- a/Makefile +++ b/Makefile @@ -150,16 +150,16 @@ update: # updates dependencies poetry update build-docker: - docker-compose build + docker-compose build build-docker-clean: - docker-compose build --no-cache + docker-compose build --no-cache deploy-docker: docker-compose up deploy-docker-daemon: - docker-compose up -d + docker-compose up -d help: @make info diff --git a/src/encoded/tests/conftest.py b/src/encoded/tests/conftest.py index 57c1118f7d..b776959731 100644 --- a/src/encoded/tests/conftest.py +++ b/src/encoded/tests/conftest.py @@ -13,6 +13,8 @@ from snovault import DBSESSION, ROOT, UPGRADER from snovault.elasticsearch import ELASTIC_SEARCH, create_mapping from snovault.util import generate_indexer_namespace_for_testing +from snovault.storage import Base +from snovault.app import configure_engine from .conftest_settings import make_app_settings_dictionary from .. import main from ..loadxl import load_all @@ -32,14 +34,31 @@ def autouse_external_tx(external_tx): pass +@pytest.yield_fixture(scope='session') +def docker_db_conn(): + engine_settings = { + 'sqlalchemy.url': 'postgres://postgres:postgres@db:5432/postgres', # for local deployment, use default postgres + } + engine = configure_engine(engine_settings) + conn = engine.connect() + tx = conn.begin() + try: + Base.metadata.create_all(bind=conn) + yield conn + finally: + tx.rollback() + conn.close() + engine.dispose() + + @pytest.fixture(scope='session') -def app_settings(request, wsgi_server_host_port, DBSession): +def app_settings(request, wsgi_server_host_port, docker_db_conn): settings = make_app_settings_dictionary() settings['auth0.audiences'] = 'http://%s:%s' % wsgi_server_host_port settings['sqlalchemy.url'] = 'postgres://postgres:postgres@db:5432/postgres' # add some here for file testing - settings[DBSESSION] = DBSession + settings[DBSESSION] = docker_db_conn # TODO fix return settings From 5069e475fefa6357b4a6cabb2f4fbab82d696050 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 2 Mar 2021 08:20:15 -0500 Subject: [PATCH 023/120] port/refactor more implicit fixtures from snovault --- src/encoded/tests/conftest.py | 159 ++++++++++++++++++++++++++++++++-- 1 file changed, 152 insertions(+), 7 deletions(-) diff --git a/src/encoded/tests/conftest.py b/src/encoded/tests/conftest.py index b776959731..394c59a735 100644 --- a/src/encoded/tests/conftest.py +++ b/src/encoded/tests/conftest.py @@ -29,13 +29,9 @@ """ -@pytest.fixture(autouse=True) -def autouse_external_tx(external_tx): - pass - - @pytest.yield_fixture(scope='session') def docker_db_conn(): + """ Equiavlent to 'conn' fixture in snovault, but with Docker url instead. """ engine_settings = { 'sqlalchemy.url': 'postgres://postgres:postgres@db:5432/postgres', # for local deployment, use default postgres } @@ -51,14 +47,163 @@ def docker_db_conn(): engine.dispose() +def external_tx(request, docker_db_conn): + """ Taken from snovault to use Docker conn """ + notice_pytest_fixtures(request) + tx = docker_db_conn.begin_nested() + yield tx + tx.rollback() + + +@pytest.fixture(autouse=True) +def autouse_external_tx(external_tx): + pass + + +@pytest.yield_fixture(scope='session') +def zsa_savepoints(docker_db_conn): + """ Place a savepoint at the start of the zope transaction + This means failed requests rollback to the db state when they began rather + than that at the start of the test. + + Taken from snovault, refactored to replace local 'conn' with Docker + """ + notice_pytest_fixtures(docker_db_conn) + @implementer(ISynchronizer) + class Savepoints(object): + def __init__(self, conn): + self.conn = conn + self.sp = None + self.state = None + + def beforeCompletion(self, transaction): + pass + + def afterCompletion(self, transaction): + # txn be aborted a second time in manager.begin() + if self.sp is None: + return + if self.state == 'commit': + self.state = 'completion' + self.sp.commit() + else: + self.state = 'abort' + self.sp.rollback() + self.sp = None + self.state = 'done' + + def newTransaction(self, transaction): + self.state = 'new' + self.sp = self.conn.begin_nested() + self.state = 'begun' + transaction.addBeforeCommitHook(self._registerCommit) + + def _registerCommit(self): + self.state = 'commit' + + zsa_savepoints = Savepoints(conn) + + transaction_management.manager.registerSynch(zsa_savepoints) + + yield zsa_savepoints + transaction_management.manager.unregisterSynch(zsa_savepoints) + + +@pytest.fixture(scope='session') +def _DBSession(docker_db_conn): + """ taken from snovault, replace conn with Docker """ + notice_pytest_fixtures(docker_db_conn) + # ``server`` thread must be in same scope + DBSession = sqlalchemy.orm.scoped_session(sqlalchemy.orm.sessionmaker(bind=docker_db_conn), scopefunc=lambda: 0) + zope.sqlalchemy.register(DBSession) + return DBSession + + +@pytest.fixture(scope='session') +def DBSession(_DBSession, zsa_savepoints, check_constraints): + """ taken directly from snovault """ + return _DBSession + + +@pytest.fixture +def transaction(request, external_tx, zsa_savepoints, check_constraints): + """ taken directly from snovault """ + notice_pytest_fixtures(request, external_tx, zsa_savepoints, check_constraints) + transaction_management.begin() + request.addfinalizer(transaction_management.abort) + return transaction_management + + +@pytest.fixture +def session(transaction, DBSession): + """ Returns a setup session + Depends on transaction as storage relies on some interaction there. + + Taken directly from snovault + """ + notice_pytest_fixtures(transaction, DBSession) + return DBSession() + + +@pytest.yield_fixture(scope='session') +def check_constraints(docker_db_conn, _DBSession): + """ + Check deferred constraints on zope transaction commit. + Deferred foreign key constraints are only checked at the outer transaction + boundary, not at a savepoint. With the Pyramid transaction bound to a + subtransaction check them manually. + + Taken from snovault and dropped in docker conn + """ + notice_pytest_fixtures(_DBSession) + + @implementer(ISynchronizer) + class CheckConstraints(object): + def __init__(self, conn): + self.conn = conn + self.state = None + + def beforeCompletion(self, transaction): + pass + + def afterCompletion(self, transaction): + pass + + def newTransaction(self, transaction): + + @transaction.addBeforeCommitHook + def set_constraints(): + self.state = 'checking' + session = _DBSession() + session.flush() + sp = self.conn.begin_nested() + try: + self.conn.execute('SET CONSTRAINTS ALL IMMEDIATE') + except: + sp.rollback() + raise + else: + self.conn.execute('SET CONSTRAINTS ALL DEFERRED') + finally: + sp.commit() + self.state = None + + check_constraints = CheckConstraints(docker_db_conn) + + transaction_management.manager.registerSynch(check_constraints) + + yield check_constraints + + transaction_management.manager.unregisterSynch(check_constraints) + + @pytest.fixture(scope='session') def app_settings(request, wsgi_server_host_port, docker_db_conn): settings = make_app_settings_dictionary() settings['auth0.audiences'] = 'http://%s:%s' % wsgi_server_host_port settings['sqlalchemy.url'] = 'postgres://postgres:postgres@db:5432/postgres' - # add some here for file testing - settings[DBSESSION] = docker_db_conn # TODO fix + settings[DBSESSION] = docker_db_conn # set DB connection to Docker return settings From 9f912901bdd4dcda019d6ef0712589c8fc6e1045 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 2 Mar 2021 08:27:26 -0500 Subject: [PATCH 024/120] another fix --- src/encoded/tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/encoded/tests/conftest.py b/src/encoded/tests/conftest.py index 394c59a735..9c3d9383d0 100644 --- a/src/encoded/tests/conftest.py +++ b/src/encoded/tests/conftest.py @@ -47,6 +47,7 @@ def docker_db_conn(): engine.dispose() +@pytest.yield_fixture(scope='session') def external_tx(request, docker_db_conn): """ Taken from snovault to use Docker conn """ notice_pytest_fixtures(request) From 8767da43c784f3c2dd51f2487a66ecab75f6405a Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 2 Mar 2021 08:42:49 -0500 Subject: [PATCH 025/120] remove snovault test import that infects the entire suite --- src/encoded/tests/test_ingestion_listener.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/encoded/tests/test_ingestion_listener.py b/src/encoded/tests/test_ingestion_listener.py index 5ebbf8e42c..47055fc35e 100644 --- a/src/encoded/tests/test_ingestion_listener.py +++ b/src/encoded/tests/test_ingestion_listener.py @@ -224,10 +224,10 @@ def mocked_should_remain_online(override=None): run(es_testapp, _queue_manager=queue_manager) # expected in this test since the source VCF is malformed -def test_test_port(): - - from snovault.tests.test_postgresql_fixture import SNOVAULT_DB_TEST_PORT - assert SNOVAULT_DB_TEST_PORT == 5440 +# def test_test_port(): +# XXX: pulls in auto-use fixtures from snovault that break Docker tests - Will 3/2/2021 +# from snovault.tests.test_postgresql_fixture import SNOVAULT_DB_TEST_PORT +# assert SNOVAULT_DB_TEST_PORT == 5440 @pytest.mark.parametrize('body, row', [ From 76a8478718f9b86cd2cfa783359dd64432c8a0a1 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 2 Mar 2021 12:14:53 -0500 Subject: [PATCH 026/120] update documentation, revert docker test setup --- docker-compose.yml | 12 +-- docs/source/docker-local.rst | 30 ++++--- src/encoded/tests/conftest.py | 165 +--------------------------------- 3 files changed, 21 insertions(+), 186 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5f23f21350..e845f15618 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,20 +48,10 @@ services: environment: AWS_ACCESS_KEY_ID: "$AWS_ACCESS_KEY_ID" AWS_SECRET_ACCESS_KEY: "$AWS_SECRET_ACCESS_KEY" - TEST: "true" # default: run local deployment + # TEST: "true" # default: run local deployment LOAD: "true" # default: load the inserts (comment this out after first run) ports: - "8000:8000" # nginx proxy port (note application traffic is not forwarded) - volumes: - - 'src-docker:/home/nginx/cgap-portal/src' # mount source directory in container to mount point depends_on: - db #- es - -volumes: - src-docker: - driver: local - driver_opts: - type: none - o: bind - device: "./src-docker" diff --git a/docs/source/docker-local.rst b/docs/source/docker-local.rst index da238cae7c..c866994e69 100644 --- a/docs/source/docker-local.rst +++ b/docs/source/docker-local.rst @@ -2,7 +2,12 @@ CGAP-Docker =========== It is now possible to run a local deployment of CGAP without installing any system level -dependencies other than Docker. Start by installing Docker:: +dependencies other than Docker. A few important notes on this setup. + +* Note this is not ideal for active development as you cannot run unit tests or edit source files in the container on the host machine (in your local editor). + + +Start by installing Docker:: $ brew install docker @@ -10,6 +15,7 @@ dependencies other than Docker. Start by installing Docker:: Prior to building the image, navigate to deploy/docker/local and open development.ini * Modify env.name and indexer.namespace - these values must be globally unique (feel free to just replace the name) +* Consider changing load_prod_data to load_local_data if you need to load more inserts There are two new Make targets that should be sufficient for normal use. To build the image locally, ensure your AWS keys are sourced and run:: @@ -24,18 +30,20 @@ and the client. To access the running container:: $ docker ps # will show running containers - $ docker exec -it bash + $ docker exec -it bash -Container Development ---------------------- -When the container is first built, a volume is created in the local repository location called ``src-docker``. -The ``src`` directory of the repository within the container is mounted to this location, allowing you to make -modifications to the source there and trigger rebuilds via bash as appropriate. -Advanced Usage --------------- +Docker Command Cheatsheet +^^^^^^^^^^^^^^^^^^^^^^^^^ -There are several useful commands documented below that may be helpful when issues are encountered or changes need to be made. +Below is a small list of useful Docker commands for advanced users:: -* ``docker-compose build --no-cache`` # will force a full rebuild of the entire image + $ docker-compose build # will trigger a build of the local cluster + $ docker-compose build --no-cache # will trigger a fresh build of the entire cluster + $ docker-compose down # will stop cluster + $ docker-compose down --volumes # will remove cluster volumes as well + $ docker-compose up # will start cluster and log all output to console + $ docker-compose up -d # will start cluster in background using existing containers + $ docker-compose up -d -V --build # trigger a rebuild/recreation of cluster containers + $ docker system prune # will cleanup unused Docker components - BE CAREFUL WITH THIS diff --git a/src/encoded/tests/conftest.py b/src/encoded/tests/conftest.py index 9c3d9383d0..b82670036a 100644 --- a/src/encoded/tests/conftest.py +++ b/src/encoded/tests/conftest.py @@ -15,6 +15,7 @@ from snovault.util import generate_indexer_namespace_for_testing from snovault.storage import Base from snovault.app import configure_engine +from dcicutils.qa_utils import notice_pytest_fixtures from .conftest_settings import make_app_settings_dictionary from .. import main from ..loadxl import load_all @@ -29,175 +30,11 @@ """ -@pytest.yield_fixture(scope='session') -def docker_db_conn(): - """ Equiavlent to 'conn' fixture in snovault, but with Docker url instead. """ - engine_settings = { - 'sqlalchemy.url': 'postgres://postgres:postgres@db:5432/postgres', # for local deployment, use default postgres - } - engine = configure_engine(engine_settings) - conn = engine.connect() - tx = conn.begin() - try: - Base.metadata.create_all(bind=conn) - yield conn - finally: - tx.rollback() - conn.close() - engine.dispose() - - -@pytest.yield_fixture(scope='session') -def external_tx(request, docker_db_conn): - """ Taken from snovault to use Docker conn """ - notice_pytest_fixtures(request) - tx = docker_db_conn.begin_nested() - yield tx - tx.rollback() - - @pytest.fixture(autouse=True) def autouse_external_tx(external_tx): pass -@pytest.yield_fixture(scope='session') -def zsa_savepoints(docker_db_conn): - """ Place a savepoint at the start of the zope transaction - This means failed requests rollback to the db state when they began rather - than that at the start of the test. - - Taken from snovault, refactored to replace local 'conn' with Docker - """ - notice_pytest_fixtures(docker_db_conn) - @implementer(ISynchronizer) - class Savepoints(object): - def __init__(self, conn): - self.conn = conn - self.sp = None - self.state = None - - def beforeCompletion(self, transaction): - pass - - def afterCompletion(self, transaction): - # txn be aborted a second time in manager.begin() - if self.sp is None: - return - if self.state == 'commit': - self.state = 'completion' - self.sp.commit() - else: - self.state = 'abort' - self.sp.rollback() - self.sp = None - self.state = 'done' - - def newTransaction(self, transaction): - self.state = 'new' - self.sp = self.conn.begin_nested() - self.state = 'begun' - transaction.addBeforeCommitHook(self._registerCommit) - - def _registerCommit(self): - self.state = 'commit' - - zsa_savepoints = Savepoints(conn) - - transaction_management.manager.registerSynch(zsa_savepoints) - - yield zsa_savepoints - transaction_management.manager.unregisterSynch(zsa_savepoints) - - -@pytest.fixture(scope='session') -def _DBSession(docker_db_conn): - """ taken from snovault, replace conn with Docker """ - notice_pytest_fixtures(docker_db_conn) - # ``server`` thread must be in same scope - DBSession = sqlalchemy.orm.scoped_session(sqlalchemy.orm.sessionmaker(bind=docker_db_conn), scopefunc=lambda: 0) - zope.sqlalchemy.register(DBSession) - return DBSession - - -@pytest.fixture(scope='session') -def DBSession(_DBSession, zsa_savepoints, check_constraints): - """ taken directly from snovault """ - return _DBSession - - -@pytest.fixture -def transaction(request, external_tx, zsa_savepoints, check_constraints): - """ taken directly from snovault """ - notice_pytest_fixtures(request, external_tx, zsa_savepoints, check_constraints) - transaction_management.begin() - request.addfinalizer(transaction_management.abort) - return transaction_management - - -@pytest.fixture -def session(transaction, DBSession): - """ Returns a setup session - Depends on transaction as storage relies on some interaction there. - - Taken directly from snovault - """ - notice_pytest_fixtures(transaction, DBSession) - return DBSession() - - -@pytest.yield_fixture(scope='session') -def check_constraints(docker_db_conn, _DBSession): - """ - Check deferred constraints on zope transaction commit. - Deferred foreign key constraints are only checked at the outer transaction - boundary, not at a savepoint. With the Pyramid transaction bound to a - subtransaction check them manually. - - Taken from snovault and dropped in docker conn - """ - notice_pytest_fixtures(_DBSession) - - @implementer(ISynchronizer) - class CheckConstraints(object): - def __init__(self, conn): - self.conn = conn - self.state = None - - def beforeCompletion(self, transaction): - pass - - def afterCompletion(self, transaction): - pass - - def newTransaction(self, transaction): - - @transaction.addBeforeCommitHook - def set_constraints(): - self.state = 'checking' - session = _DBSession() - session.flush() - sp = self.conn.begin_nested() - try: - self.conn.execute('SET CONSTRAINTS ALL IMMEDIATE') - except: - sp.rollback() - raise - else: - self.conn.execute('SET CONSTRAINTS ALL DEFERRED') - finally: - sp.commit() - self.state = None - - check_constraints = CheckConstraints(docker_db_conn) - - transaction_management.manager.registerSynch(check_constraints) - - yield check_constraints - - transaction_management.manager.unregisterSynch(check_constraints) - - @pytest.fixture(scope='session') def app_settings(request, wsgi_server_host_port, docker_db_conn): From aacf055a28517a42c27179ddd7f9517b481f972d Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 2 Mar 2021 12:42:47 -0500 Subject: [PATCH 027/120] fix readme link, more test fixes --- README.rst | 2 +- src/encoded/tests/conftest.py | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index ed866d9a20..6f211802d0 100644 --- a/README.rst +++ b/README.rst @@ -28,7 +28,7 @@ Welcome to CGAP! We are a team of scientists, clinicians, and developers who aim Be warned that features are under active development and may not be stable! Visit the production deployment for the best experience. For installation and more information on getting started, see our `documentation page `_. -For information on how to run CGAP with Docker, see `here. <./docs/docker-setup.rst>`_ +For information on how to run CGAP with Docker, see `here. <./docs/source/docker-setup.rst>`_ Navigating this Repository ^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/src/encoded/tests/conftest.py b/src/encoded/tests/conftest.py index b82670036a..28677c839f 100644 --- a/src/encoded/tests/conftest.py +++ b/src/encoded/tests/conftest.py @@ -36,12 +36,10 @@ def autouse_external_tx(external_tx): @pytest.fixture(scope='session') -def app_settings(request, wsgi_server_host_port, docker_db_conn): - +def app_settings(request, wsgi_server_host_port, DBSession): settings = make_app_settings_dictionary() settings['auth0.audiences'] = 'http://%s:%s' % wsgi_server_host_port - settings['sqlalchemy.url'] = 'postgres://postgres:postgres@db:5432/postgres' - settings[DBSESSION] = docker_db_conn # set DB connection to Docker + settings[DBSESSION] = DBSession # set DB connection to Docker return settings @@ -49,19 +47,20 @@ def app_settings(request, wsgi_server_host_port, docker_db_conn): @pytest.fixture(scope='session') -def es_app_settings(wsgi_server_host_port, aws_auth): +def es_app_settings(wsgi_server_host_port, elasticsearch_server, postgresql_server, aws_auth): settings = make_app_settings_dictionary() settings['create_tables'] = True settings['persona.audiences'] = 'http://%s:%s' % wsgi_server_host_port # 2-tuple such as: ('localhost', '5000') - settings['elasticsearch.server'] = 'search-cgap-testing-6-8-vo4mdkmkshvmyddc65ux7dtaou.us-east-1.es.amazonaws.com:443' - settings['sqlalchemy.url'] = 'postgres://postgres:postgres@db:5432/postgres' + settings['elasticsearch.server'] = elasticsearch_server + settings['sqlalchemy.url'] = postgresql_server settings['collection_datastore'] = 'elasticsearch' settings['item_datastore'] = 'elasticsearch' settings['indexer'] = True settings['indexer.namespace'] = INDEXER_NAMESPACE_FOR_TESTING # use aws auth to access elasticsearch - settings['elasticsearch.aws_auth'] = True + if aws_auth: + settings['elasticsearch.aws_auth'] = True return settings From 2276cc1baf0132e3eef2343a7294b5f926c36565 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 2 Mar 2021 13:09:49 -0500 Subject: [PATCH 028/120] final touches prior to PR --- .gitignore | 2 - deploy/docker/local/Dockerfile | 3 +- deploy/docker/local/Makefile | 14 -- ...development.ini => docker_development.ini} | 0 docker-compose.yml | 5 +- docs/source/docker-local.rst | 3 +- src/encoded/tests/conftest.py | 5 +- src/encoded/tests/test_fixtures.py | 176 +++++++++--------- src/encoded/tests/test_ingestion_listener.py | 7 +- 9 files changed, 98 insertions(+), 117 deletions(-) delete mode 100644 deploy/docker/local/Makefile rename deploy/docker/local/{development.ini => docker_development.ini} (100%) diff --git a/.gitignore b/.gitignore index a5420f0889..10d02188d8 100644 --- a/.gitignore +++ b/.gitignore @@ -36,8 +36,6 @@ beanstalk.cfg /src/encoded/static/css/style.css /src/encoded/static/scss/_variables.original.scss -/src-docker/* - /src/encoded/tests/data/temp-local-inserts/* !/src/encoded/tests/data/temp-local-inserts/.gitkeep diff --git a/deploy/docker/local/Dockerfile b/deploy/docker/local/Dockerfile index 4af3501eb9..431904146d 100644 --- a/deploy/docker/local/Dockerfile +++ b/deploy/docker/local/Dockerfile @@ -79,8 +79,7 @@ RUN touch /var/log/nginx/error.log && \ chown -R nginx:nginx /var/log/nginx/error.log # Copy over ini file, entrypoint -COPY development.ini . -COPY development.ini test.ini +COPY docker_development.ini development.ini COPY entrypoint.sh . RUN chmod +x entrypoint.sh EXPOSE 8000 diff --git a/deploy/docker/local/Makefile b/deploy/docker/local/Makefile deleted file mode 100644 index 1697fbf88a..0000000000 --- a/deploy/docker/local/Makefile +++ /dev/null @@ -1,14 +0,0 @@ -configure: - brew install docker - -build: - docker build --progress=plain -t cgap-docker:0.0.0b0 . - -enter: - docker run -v /tmp/snovault/pgdata:/tmp/snovault/pgdata -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -p 6543:6543 -it cgap-docker:0.0.0b0 bash - -info: - @: $(info Here are some 'make' options:) - $(info - Use 'make configure' to install Docker on OSX via Brew.) - $(info - Use 'make build' to build a local version of the Docker image.) - $(info - Use 'make enter' to start a bash session in the build container.) diff --git a/deploy/docker/local/development.ini b/deploy/docker/local/docker_development.ini similarity index 100% rename from deploy/docker/local/development.ini rename to deploy/docker/local/docker_development.ini diff --git a/docker-compose.yml b/docker-compose.yml index e845f15618..15a6e30db2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -46,8 +46,9 @@ services: container_name: cgap command: "/home/cgap-admin/cgap-portal/entrypoint.sh" environment: - AWS_ACCESS_KEY_ID: "$AWS_ACCESS_KEY_ID" - AWS_SECRET_ACCESS_KEY: "$AWS_SECRET_ACCESS_KEY" + # Default to caller's environment + AWS_ACCESS_KEY_ID: + AWS_SECRET_ACCESS_KEY: # TEST: "true" # default: run local deployment LOAD: "true" # default: load the inserts (comment this out after first run) ports: diff --git a/docs/source/docker-local.rst b/docs/source/docker-local.rst index c866994e69..f9965259ad 100644 --- a/docs/source/docker-local.rst +++ b/docs/source/docker-local.rst @@ -4,7 +4,8 @@ CGAP-Docker It is now possible to run a local deployment of CGAP without installing any system level dependencies other than Docker. A few important notes on this setup. -* Note this is not ideal for active development as you cannot run unit tests or edit source files in the container on the host machine (in your local editor). +* This is not ideal for active development as you cannot run unit tests or edit source files in the container on the host machine (in your local editor). +* VERY IMPORTANT: Do not upload the local deployment container image to any registry. Start by installing Docker:: diff --git a/src/encoded/tests/conftest.py b/src/encoded/tests/conftest.py index 28677c839f..4cdd98c6da 100644 --- a/src/encoded/tests/conftest.py +++ b/src/encoded/tests/conftest.py @@ -8,14 +8,11 @@ import pkg_resources from pyramid.request import apply_request_extensions -from pyramid.testing import DummyRequest, setUp, tearDown +from pyramid.testing import DummyRequest from pyramid.threadlocal import get_current_registry, manager as threadlocal_manager from snovault import DBSESSION, ROOT, UPGRADER from snovault.elasticsearch import ELASTIC_SEARCH, create_mapping from snovault.util import generate_indexer_namespace_for_testing -from snovault.storage import Base -from snovault.app import configure_engine -from dcicutils.qa_utils import notice_pytest_fixtures from .conftest_settings import make_app_settings_dictionary from .. import main from ..loadxl import load_all diff --git a/src/encoded/tests/test_fixtures.py b/src/encoded/tests/test_fixtures.py index 8a2cf08651..17479f76aa 100644 --- a/src/encoded/tests/test_fixtures.py +++ b/src/encoded/tests/test_fixtures.py @@ -7,94 +7,94 @@ pytestmark = [pytest.mark.setone, pytest.mark.working, pytest.mark.schema, pytest.mark.indexing] -# @pytest.yield_fixture(scope='session') -# def minitestdata(app, conn): -# tx = conn.begin_nested() -# -# environ = { -# 'HTTP_ACCEPT': 'application/json', -# 'REMOTE_USER': 'TEST', -# } -# testapp = TestApp(app, environ) -# -# item = { -# 'email': 'human@email.org', -# 'first_name': 'Homo', -# 'last_name': 'Sapien', -# } -# testapp.post_json('/user', item, status=201) -# -# yield -# tx.rollback() -# -# -# -# @pytest.yield_fixture(scope='session') -# def minitestdata2(app, conn): -# tx = conn.begin_nested() -# -# environ = { -# 'HTTP_ACCEPT': 'application/json', -# 'REMOTE_USER': 'TEST', -# } -# testapp = TestApp(app, environ) -# -# item = { -# 'email': 'human2@email.org', -# 'first_name': 'Homo', -# 'last_name': 'Erectus', -# } -# testapp.post_json('/user', item, status=201) -# -# yield -# tx.rollback() -# -# -# @pytest.mark.usefixtures('minitestdata') -# def test_fixtures1(testapp): -# """ This test is not really exhaustive. -# -# Still need to inspect the sql log to verify fixture correctness. -# """ -# res = testapp.get('/user').maybe_follow() -# items = res.json['@graph'] -# assert len(items) == 1 -# -# # Trigger an error -# item = {'foo': 'bar'} -# res = testapp.post_json('/user', item, status=422) -# assert res.json['errors'] -# -# res = testapp.get('/user').maybe_follow() -# items = res.json['@graph'] -# assert len(items) == 1 -# -# item = { -# 'email': 'human3@email.org', -# 'first_name': 'Homo', -# 'last_name': 'Habilis', -# } -# testapp.post_json('/user', item, status=201) -# -# res = testapp.get('/user').maybe_follow() -# items = res.json['@graph'] -# assert len(items) == 2 -# -# # Trigger an error -# item = {'foo': 'bar'} -# res = testapp.post_json('/user', item, status=422) -# assert res.json['errors'] -# -# res = testapp.get('/user').maybe_follow() -# items = res.json['@graph'] -# assert len(items) == 2 -# -# -# def test_fixtures2(minitestdata2, testapp): -# # http://stackoverflow.com/questions/15775601/mutually-exclusive-fixtures -# res = testapp.get('/user').maybe_follow() -# items = res.json['@graph'] -# assert len(items) == 1 +@pytest.yield_fixture(scope='session') +def minitestdata(app, conn): + tx = conn.begin_nested() + + environ = { + 'HTTP_ACCEPT': 'application/json', + 'REMOTE_USER': 'TEST', + } + testapp = TestApp(app, environ) + + item = { + 'email': 'human@email.org', + 'first_name': 'Homo', + 'last_name': 'Sapien', + } + testapp.post_json('/user', item, status=201) + + yield + tx.rollback() + + + +@pytest.yield_fixture(scope='session') +def minitestdata2(app, conn): + tx = conn.begin_nested() + + environ = { + 'HTTP_ACCEPT': 'application/json', + 'REMOTE_USER': 'TEST', + } + testapp = TestApp(app, environ) + + item = { + 'email': 'human2@email.org', + 'first_name': 'Homo', + 'last_name': 'Erectus', + } + testapp.post_json('/user', item, status=201) + + yield + tx.rollback() + + +@pytest.mark.usefixtures('minitestdata') +def test_fixtures1(testapp): + """ This test is not really exhaustive. + + Still need to inspect the sql log to verify fixture correctness. + """ + res = testapp.get('/user').maybe_follow() + items = res.json['@graph'] + assert len(items) == 1 + + # Trigger an error + item = {'foo': 'bar'} + res = testapp.post_json('/user', item, status=422) + assert res.json['errors'] + + res = testapp.get('/user').maybe_follow() + items = res.json['@graph'] + assert len(items) == 1 + + item = { + 'email': 'human3@email.org', + 'first_name': 'Homo', + 'last_name': 'Habilis', + } + testapp.post_json('/user', item, status=201) + + res = testapp.get('/user').maybe_follow() + items = res.json['@graph'] + assert len(items) == 2 + + # Trigger an error + item = {'foo': 'bar'} + res = testapp.post_json('/user', item, status=422) + assert res.json['errors'] + + res = testapp.get('/user').maybe_follow() + items = res.json['@graph'] + assert len(items) == 2 + + +def test_fixtures2(minitestdata2, testapp): + # http://stackoverflow.com/questions/15775601/mutually-exclusive-fixtures + res = testapp.get('/user').maybe_follow() + items = res.json['@graph'] + assert len(items) == 1 def test_order_complete(app): diff --git a/src/encoded/tests/test_ingestion_listener.py b/src/encoded/tests/test_ingestion_listener.py index 47055fc35e..fc04a9189d 100644 --- a/src/encoded/tests/test_ingestion_listener.py +++ b/src/encoded/tests/test_ingestion_listener.py @@ -224,10 +224,9 @@ def mocked_should_remain_online(override=None): run(es_testapp, _queue_manager=queue_manager) # expected in this test since the source VCF is malformed -# def test_test_port(): -# XXX: pulls in auto-use fixtures from snovault that break Docker tests - Will 3/2/2021 -# from snovault.tests.test_postgresql_fixture import SNOVAULT_DB_TEST_PORT -# assert SNOVAULT_DB_TEST_PORT == 5440 +def test_test_port(): + from snovault.tests.test_postgresql_fixture import SNOVAULT_DB_TEST_PORT + assert SNOVAULT_DB_TEST_PORT == 5440 @pytest.mark.parametrize('body, row', [ From 157d92f20ec95981e841bb00e1f81da55889849e Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 2 Mar 2021 13:18:03 -0500 Subject: [PATCH 029/120] small changes --- deploy/docker/local/Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/deploy/docker/local/Dockerfile b/deploy/docker/local/Dockerfile index 431904146d..eb5b0e9bbf 100644 --- a/deploy/docker/local/Dockerfile +++ b/deploy/docker/local/Dockerfile @@ -8,12 +8,13 @@ MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" # Build Arguments ARG CGAP_ENV_NAME ENV CGAP_ENV_NAME=${CGAP_ENV_NAME:-"cgap-docker-will-test"} -#ARG RUN_TEST -#ENV RUN_TEST=${RUN_TEST:-1} +# Repair the below after this is merged +#ARG CGAP_BRANCH +#ENV CGAP_BRANCH=${CGAP_BRANCH:-"master"} +ENV CGAP_BRANCH=c4_519 # Configure (global) Env ENV CGAP_REPO=https://github.com/dbmi-bgm/cgap-portal.git -ENV CGAP_BRANCH=c4_519 ENV NGINX_USER=nginx ENV DEBIAN_FRONTEND=noninteractive ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1 From b35208c1a48ca4aeba1978da00f0f29fe4e52fe3 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 2 Mar 2021 13:31:26 -0500 Subject: [PATCH 030/120] more small changes --- Makefile | 4 ++-- deploy/docker/local/entrypoint.sh | 2 +- docs/source/docker-local.rst | 10 ++++++---- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 15c10d63a0..6e9bb19241 100644 --- a/Makefile +++ b/Makefile @@ -156,10 +156,10 @@ build-docker-clean: docker-compose build --no-cache deploy-docker: - docker-compose up + docker-compose up -V deploy-docker-daemon: - docker-compose up -d + docker-compose up -d -V help: @make info diff --git a/deploy/docker/local/entrypoint.sh b/deploy/docker/local/entrypoint.sh index fbfe5546d8..fe66a258e9 100644 --- a/deploy/docker/local/entrypoint.sh +++ b/deploy/docker/local/entrypoint.sh @@ -19,7 +19,7 @@ if [ -z ${TEST+x} ]; then service nginx start # Start application - pserve development.ini --reload + make deploy2 else diff --git a/docs/source/docker-local.rst b/docs/source/docker-local.rst index f9965259ad..eda98dbf4e 100644 --- a/docs/source/docker-local.rst +++ b/docs/source/docker-local.rst @@ -5,7 +5,8 @@ It is now possible to run a local deployment of CGAP without installing any syst dependencies other than Docker. A few important notes on this setup. * This is not ideal for active development as you cannot run unit tests or edit source files in the container on the host machine (in your local editor). -* VERY IMPORTANT: Do not upload the local deployment container image to any registry. +* ElasticSearch is too compute intensive to virtualize on most machines. For this reason we use the CGAP test ES cluster for this deployment instead of spinning up an ES cluster in Docker. If you want to attempt to run containerized ES, see ``docker-compose.yml``. +* VERY IMPORTANT: Do not upload the local deployment container image to any registry. This utility is in beta - . Start by installing Docker:: @@ -13,7 +14,7 @@ Start by installing Docker:: $ brew install docker -Prior to building the image, navigate to deploy/docker/local and open development.ini +Prior to building the image, navigate to deploy/docker/local and open docker_development.ini * Modify env.name and indexer.namespace - these values must be globally unique (feel free to just replace the name) * Consider changing load_prod_data to load_local_data if you need to load more inserts @@ -26,7 +27,9 @@ AWS keys are sourced and run:: The first command will take awhile the first time you run it but should speed up after. Since it is doing a fresh rebuild every time it is a little slower than the old local deployment since it has to fully reinstall/rebuild both Python -and the client. +and the client. Because of this, it is recommended to continue active development using the existing installation setup. +Once the branch is ready for integrated testing, set the desired branch in ``docker-compose.yml`` and trigger a build. +When the app is brought online the behavior should be identical to that of the existing local deployment setup. To access the running container:: @@ -34,7 +37,6 @@ To access the running container:: $ docker exec -it bash - Docker Command Cheatsheet ^^^^^^^^^^^^^^^^^^^^^^^^^ From eb46b09781b2da84444db698652d3d8b96e62e40 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 2 Mar 2021 13:36:13 -0500 Subject: [PATCH 031/120] remove etc as well --- etc/encoded-apache.conf | 173 ---------------------------------------- etc/logging-apache.conf | 1 - 2 files changed, 174 deletions(-) delete mode 100644 etc/encoded-apache.conf delete mode 100644 etc/logging-apache.conf diff --git a/etc/encoded-apache.conf b/etc/encoded-apache.conf deleted file mode 100644 index 069d48dfd9..0000000000 --- a/etc/encoded-apache.conf +++ /dev/null @@ -1,173 +0,0 @@ -KeepAliveTimeout 75 - -# The socket directory must be readable by the daemon process user -WSGISocketPrefix /var/run/wsgi -WSGIDaemonProcess encoded user=encoded group=encoded processes=6 threads=1 display-name=encoded-app -# No need for embedded interpreters -WSGIRestrictEmbedded On -# Pass the authorization header so basic auth works -WSGIPassAuthorization On - -# Indexer. Configure first to avoid catchall '/' -WSGIDaemonProcess encoded-indexer user=encoded group=encoded processes=1 threads=1 display-name=encoded-indexer -WSGIScriptAlias /_indexer /srv/encoded/parts/production-indexer/wsgi process-group=encoded-indexer application-group=%{GLOBAL} - -# https://github.com/GrahamDumpleton/mod_wsgi/issues/2 -SetEnvIf Request_Method HEAD X_REQUEST_METHOD=HEAD - -LogFormat "%v:%p %h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\" %{X-Stats}o&server_time=%D" vhost_combined_stats - - - Order deny,allow - Allow from all - - Require all granted - - - - -# Specifying process-group and application-group here ensures processes are started on httpd start -WSGIScriptAlias / /srv/encoded/parts/production/wsgi process-group=encoded application-group=%{GLOBAL} - - - Order deny,allow - Allow from all - - Require all granted - - # Limit upload size to 500 MB (375MB before base64 encoding) - LimitRequestBody 524288000 - # Apache adds -gzip to outgoing ETag in mod_deflate, remove inbound. - # https://issues.apache.org/bugzilla/show_bug.cgi?id=39727 - RequestHeader edit If-Match -gzip\"$ \" - RequestHeader edit If-None-Match -gzip\"$ \" - - # CORS support - Header always set Access-Control-Allow-Origin "*" - Header always set Access-Control-Allow-Methods "GET, HEAD" - Header always set Access-Control-Allow-Headers "Accept, Origin, Range, X-Requested-With" - Header always set Access-Control-Expose-Headers: "Content-Length, Content-Range, Content-Type" - - # CORS preflight - RewriteCond %{REQUEST_METHOD} OPTIONS - RewriteRule ^ - [redirect=200,last] - - -# Serve static resources directly from Apache -Alias /static /srv/encoded/src/encoded/static -Alias /favicon.ico /srv/encoded/src/encoded/static/img/favicon.ico - - - Order deny,allow - Allow from all - - Require all granted - - - -# Compress JSON responses. -AddOutputFilterByType DEFLATE application/javascript application/json text/css text/html text/javascript - -# Source map type (to enable compression) - - ForceType application/json - - -RewriteEngine On - -# Exclude robots from all but production site -#RewriteCond %{HTTP_HOST} =www.encodeproject.org -#RewriteRule ^/robots\.txt$ /static/robots.txt [last,passthrough] -#RewriteRule ^/robots\.txt$ /static/dev-robots.txt [last,passthrough] - -# Google site verification -#RewriteRule ^/google[0-9a-f]+.html$ /static$0 [last,passthrough] - -# Proxy modencode comparative page - - ProxyPass http://cake.encodedcc.org/comparative - ProxyPassReverse http://cake.encodedcc.org/comparative - - -# Proxy internal redirects for file downloads -SSLProxyEngine On -RewriteCond %{ENV:REDIRECT_STATUS} . -RewriteRule ^/_proxy/(.+)$ $1 [proxy] - -# Forbid PUT/PATCH/POST to plain http -RewriteCond %{HTTP:X-Forwarded-Proto} =http -RewriteCond %{REQUEST_METHOD} !^(GET|HEAD)$ -RewriteCond %{HTTP_HOST} ^(www\.encodeproject\.org|test\.encodedcc\.org)$ -RewriteRule ^ - [forbidden] - -# Forbid basic auth to plain http -RewriteCond %{HTTP:X-Forwarded-Proto} =http -RewriteCond %{HTTP:Authorization} . -RewriteCond %{HTTP_HOST} ^(www\.encodeproject\.org|test\.encodedcc\.org)$ -RewriteRule ^ - [forbidden] - -ErrorDocument 403 "Forbidden. HTTPS required for authenticated access." - -# Redirect no-www to https://www.encodeproject.org -#RewriteCond %{HTTP_HOST} =encodeproject.org -#RewriteCond %{REQUEST_METHOD} ^(GET|HEAD)$ -#RewriteCond %{HTTP:Authorization} !. -#RewriteRule ^ https://www.encodeproject.org%{REQUEST_URI} [redirect=permanent,last,qsappend] - -# Redirect to https -#RewriteCond %{HTTP:X-Forwarded-Proto} =http -#RewriteCond %{HTTP_HOST} ^(www\.encodeproject\.org|test\.encodedcc\.org)$ -#RewriteRule ^ https://%{HTTP_HOST}%{REQUEST_URI} [redirect=permanent,last,qsappend] - -################### -# Portal redirects - -# Normalize index.html etc. -RewriteRule ^/ENCODE$ $0/ [nocase] -RewriteRule ^/encode/(.*)$ /ENCODE/$1 -RewriteRule ^/ENCODE/FAQ$ $0/ -RewriteRule ^(/ENCODE/.+)\.html$ $1 -RewriteRule ^(/ENCODE(/|/.+/))index$ $1 - -# Redirect -RewriteRule ^/ENCODE/$ /? [last,redirect=permanent] -RewriteRule ^/ENCODE/search$ /search/?type=experiment [last,redirect=permanent] -RewriteRule ^/ENCODE/dataSummary$ /search/?type=experiment [last,redirect=permanent] -RewriteRule ^/ENCODE/dataMatrix/encodeDataMatrixMouse$ /search/?type=experiment&replicates.library.biosample.donor.organism.scientific_name=Mus\ musculus [last,redirect=permanent] -RewriteRule ^/ENCODE/dataMatrix/encodeDataMatrixHuman$ /search/?type=experiment&replicates.library.biosample.donor.organism.scientific_name=Homo\ sapiens [last,redirect=permanent] -RewriteRule ^/ENCODE/dataMatrix/encodeChipMatrixHuman$ /search/?type=experiment&replicates.library.biosample.donor.organism.scientific_name=Homo\ sapiens&assay_term_name=ChIP-seq [last,redirect=permanent] -RewriteRule ^/ENCODE/dataMatrix/encodeDataSummaryHuman$ /search/?type=experiment&replicates.library.biosample.donor.organism.scientific_name=Homo\ sapiens [last,redirect=permanent] -RewriteRule ^/ENCODE/dataMatrix/encodeChipMatrixMouse$ /search/?type=experiment&replicates.library.biosample.donor.organism.scientific_name=Mus\ musculus&assay_term_name=ChIP-seq [last,redirect=permanent] -RewriteRule ^/ENCODE/dataMatrix/encodeDataSummaryMouse$ /search/?type=experiment&replicates.library.biosample.donor.organism.scientific_name=Mus\ musculus [last,redirect=permanent] -RewriteRule ^/ENCODE/terms$ /about/data-use-policy/? [last,redirect=permanent] -RewriteRule ^/ENCODE/cellTypes$ /search/?type=biosample&organism.scientific_name=Homo\ sapiens [last,redirect=permanent] -RewriteRule ^/ENCODE/cellTypesMouse$ /search/?type=biosample&organism.scientific_name=Mus\ musculus [last,redirect=permanent] -RewriteRule ^/ENCODE/antibodies$ /search/?type=antibody_approval [last,redirect=permanent] -RewriteRule ^/ENCODE/softwareTools$ /software/? [last,redirect=permanent] -RewriteRule ^/ENCODE/experiment_guidelines$ /about/experiment-guidelines/? [last,redirect=permanent] -RewriteRule ^/ENCODE/platform_characterization$ /data-standards/platform-characterization/? [last,redirect=permanent] -RewriteRule ^/ENCODE/qualityMetrics$ /data-standards/2012-quality-metrics/? [last,redirect=permanent] -RewriteRule ^/ENCODE/contributors$ /about/contributors/? [last,redirect=permanent] -RewriteRule ^/ENCODE/analysis$ /about/2012-integrative-analysis/? [last,redirect=permanent] -RewriteRule ^/ENCODE/pubsOther$ /publications/? [last,redirect=permanent] -RewriteRule ^/ENCODE/pubsEncode$ /publications/? [last,redirect=permanent] -RewriteRule ^/ENCODE/fileFormats$ /help/file-formats/? [last,redirect=permanent] -RewriteRule ^/ENCODE/contacts$ /help/contacts/? [last,redirect=permanent] -RewriteRule ^/ENCODE/FAQ/$ /tutorials/? [last,redirect=permanent] -RewriteRule ^/ENCODE/usageResources$ /tutorials/? [last,redirect=permanent] -RewriteRule ^/ENCODE/releaseLog$ /about/contributors/? [last,redirect=permanent] -RewriteRule ^/ENCODE/pilot$ /about/contributors/? [last,redirect=permanent] -RewriteRule ^/ENCODE/downloads$ /help/getting-started/? [last,redirect=permanent] -RewriteRule ^/ENCODE/downloadsMouse$ /help/getting-started/? [last,redirect=permanent] -RewriteRule ^/ENCODE/otherTerms$ /help/getting-started/? [last,redirect=permanent] -RewriteRule ^/ENCODE/integrativeAnalysis/VM$ http://encodedcc.stanford.edu/ftp/encodevm/? [last,redirect=permanent] -RewriteRule ^/encyclopedia/visualize http://genome.ucsc.edu/cgi-bin/hgTracks?db=hg19&hgt.customText=http://bib.umassmed.edu/~iyers/encode_elements/display/tracks.txt [last,redirect=permanent] - -# Fallback -RewriteRule ^/ENCODE/.*$ - [gone] - -# Redirect to genome browser -RewriteRule ^/cgi-bin/hgTracks$ http://genome.ucsc.edu/cgi-bin/hgTracks [last,redirect=permanent] -RewriteRule ^/cgi-bin/hgTables$ http://genome.ucsc.edu/cgi-bin/hgTables [last,redirect=permanent] -RewriteRule ^/cgi-bin/hgTrackUi$ http://genome.ucsc.edu/cgi-bin/hgTrackUi [last,redirect=permanent] -RewriteRule ^/cgi-bin/hgHubConnect$ http://genome.ucsc.edu/cgi-bin/hgHubConnect [last,redirect=permanent] diff --git a/etc/logging-apache.conf b/etc/logging-apache.conf deleted file mode 100644 index 2ad19759a8..0000000000 --- a/etc/logging-apache.conf +++ /dev/null @@ -1 +0,0 @@ -CustomLog ${APACHE_LOG_DIR}/access.log vhost_combined_stats From cbe240969a243ac37c7a194dce185de5cbbfef9d Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 2 Mar 2021 13:37:08 -0500 Subject: [PATCH 032/120] 2 more deletions --- CHANGES.rst | 6 ------ test.cfg.DISABLED | 2 -- 2 files changed, 8 deletions(-) delete mode 100644 CHANGES.rst delete mode 100644 test.cfg.DISABLED diff --git a/CHANGES.rst b/CHANGES.rst deleted file mode 100644 index 6b772de229..0000000000 --- a/CHANGES.rst +++ /dev/null @@ -1,6 +0,0 @@ -Changes -======= - -0.1 (unreleased) ----------------- - diff --git a/test.cfg.DISABLED b/test.cfg.DISABLED deleted file mode 100644 index 7b3f6c01aa..0000000000 --- a/test.cfg.DISABLED +++ /dev/null @@ -1,2 +0,0 @@ -[buildout] -extends = buildout.cfg From c6b7af3277fc29902f3ff85b09bee85bb5093940 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 2 Mar 2021 13:45:41 -0500 Subject: [PATCH 033/120] small changes to conftest --- src/encoded/tests/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/encoded/tests/conftest.py b/src/encoded/tests/conftest.py index 2c539795ba..78a68a57f3 100644 --- a/src/encoded/tests/conftest.py +++ b/src/encoded/tests/conftest.py @@ -41,7 +41,7 @@ def app_settings(request, wsgi_server_host_port, conn, DBSession): # noQA - We notice_pytest_fixtures(request, wsgi_server_host_port, conn, DBSession) settings = make_app_settings_dictionary() settings['auth0.audiences'] = 'http://%s:%s' % wsgi_server_host_port - settings[DBSESSION] = DBSession # set DB connection to Docker + settings[DBSESSION] = DBSession return settings @@ -62,7 +62,7 @@ def es_app_settings(wsgi_server_host_port, elasticsearch_server, postgresql_serv # use aws auth to access elasticsearch if aws_auth: - settings['elasticsearch.aws_auth'] = True + settings['elasticsearch.aws_auth'] = aws_auth return settings From eea3a7f779811ea53afb71952348a8942961b263 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 2 Mar 2021 13:51:43 -0500 Subject: [PATCH 034/120] fix setup_eb.py --- deploy/docker/elasticsearch/Dockerfile | 2 -- deploy/docker/local/Dockerfile | 8 ++++---- setup_eb.py | 3 +-- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/deploy/docker/elasticsearch/Dockerfile b/deploy/docker/elasticsearch/Dockerfile index 951db9d476..826ba8b72a 100644 --- a/deploy/docker/elasticsearch/Dockerfile +++ b/deploy/docker/elasticsearch/Dockerfile @@ -3,7 +3,5 @@ FROM docker.elastic.co/elasticsearch/elasticsearch:6.8.14 MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" ENV ELASTICSEARCH_VERSION="6.8.14" - -# Set port to 9200 ENV ELASTICSEARCH_SERVICE_PORT=9200 EXPOSE $ELASTICSEARCH_SERVICE_PORT diff --git a/deploy/docker/local/Dockerfile b/deploy/docker/local/Dockerfile index eb5b0e9bbf..992a1927df 100644 --- a/deploy/docker/local/Dockerfile +++ b/deploy/docker/local/Dockerfile @@ -30,7 +30,7 @@ ENV PYTHONFAULTHANDLER=1 \ COPY install_nginx.sh / RUN bash /install_nginx.sh -# Intall things needed for our system +# Install things needed for our system RUN apt-get update RUN apt-get install -y curl vim emacs postgresql-client net-tools RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - @@ -72,12 +72,12 @@ RUN chown -R nginx:nginx /var/cache/nginx && \ chown -R nginx:nginx /var/log/nginx && \ chown -R nginx:nginx /etc/nginx/conf.d RUN touch /var/run/nginx.pid && \ - chown -R nginx:nginx /var/run/nginx.pid + chown -R nginx:nginx /var/run/nginx.pid RUN rm -f /var/log/nginx/* RUN touch /var/log/nginx/access.log && \ - chown -R nginx:nginx /var/log/nginx/access.log + chown -R nginx:nginx /var/log/nginx/access.log RUN touch /var/log/nginx/error.log && \ - chown -R nginx:nginx /var/log/nginx/error.log + chown -R nginx:nginx /var/log/nginx/error.log # Copy over ini file, entrypoint COPY docker_development.ini development.ini diff --git a/setup_eb.py b/setup_eb.py index 86cdfa45c5..b67be46168 100644 --- a/setup_eb.py +++ b/setup_eb.py @@ -65,9 +65,8 @@ def entry_points(): PACKAGE_NAME = POETRY_DATA['name'] README = open(os.path.join(ROOT_DIR, 'README.rst')).read() -CHANGES = open(os.path.join(ROOT_DIR, 'CHANGES.rst')).read() DESCRIPTION = POETRY_DATA['description'] -LONG_DESCRIPTION = README + '\n\n' + CHANGES +LONG_DESCRIPTION = README AUTHOR, AUTHOR_EMAIL = author_and_email(POETRY_DATA['authors'][0]) URL = 'http://data.4dnucleome.org' LICENSE = 'MIT' From 4ec5321f67c898e815bc0275ae3777a49c56b082 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Wed, 3 Mar 2021 15:15:28 -0500 Subject: [PATCH 035/120] c4-519 first real pass at production setup, adapted for ecs --- deploy/docker/production/Dockerfile | 5 +- deploy/docker/production/assume_identity.py | 71 +++++++++++++++++++ deploy/docker/production/docker-compose.yml | 13 ++++ deploy/docker/production/entrypoint.sh | 4 ++ deploy/docker/production/install_nginx.sh | 77 +++++++++++++++++++++ docker-compose.yml | 2 +- src/encoded/ingestion/variant_utils.py | 1 + 7 files changed, 169 insertions(+), 4 deletions(-) create mode 100644 deploy/docker/production/assume_identity.py create mode 100644 deploy/docker/production/docker-compose.yml create mode 100644 deploy/docker/production/install_nginx.sh diff --git a/deploy/docker/production/Dockerfile b/deploy/docker/production/Dockerfile index e6accbb704..f2cdb6b802 100644 --- a/deploy/docker/production/Dockerfile +++ b/deploy/docker/production/Dockerfile @@ -77,11 +77,10 @@ RUN touch /var/log/nginx/access.log && \ RUN touch /var/log/nginx/error.log && \ chown -R nginx:nginx /var/log/nginx/error.log -# Copy over ini file, entrypoint -# TODO ini file must be generated -COPY production.ini . COPY entrypoint.sh . +COPY assume_identity.py . RUN chmod +x entrypoint.sh +RUN chmod +x assume_identity.py EXPOSE 8000 # Container does not ever run as root # This way even if someone gets access to the container they cannot diff --git a/deploy/docker/production/assume_identity.py b/deploy/docker/production/assume_identity.py new file mode 100644 index 0000000000..86a1b54467 --- /dev/null +++ b/deploy/docker/production/assume_identity.py @@ -0,0 +1,71 @@ +# This code snippet is adapated from AWS ECS Documentation +# The plan is 'create' environments by uploading Secrets to AWS Secrets Manager +# and retrieve the secrets upon deployment. +# The idea is the container runner will have an assumed IAM role granting access +# specifically to the secret + +import os +import json +import boto3 +from botocore.exceptions import ClientError +from dcicutils.beanstalk_utils import REGION +from dcicutils.qa_utils import override_environ +from dcicutils.deployment_utils import IniFileManager + + +class CGAPDockerIniFileManager(IniFileManager): + """ This runs at top level, so path is slightly different. """ + _MY_DIR = os.path.dirname(__file__) + TEMPLATE_DIR = os.path.join(_MY_DIR, "deploy/ini_files") + PYPROJECT_FILE_NAME = os.path.join(os.path.dirname(_MY_DIR), "pyproject.toml") + + +def assume_identity(): + """ This function makes a request to secrets manager for the identity passed to the container. + See documentation above. + """ + secret_name = os.environ.get('IDENTITY', 'dev/beanstalk/cgap-dev') + region_name = REGION # us-east-1 + + # XXX: We should refactor a SecretsManager wrapper into dcicutils + session = boto3.session.Session() + client = session.client( + service_name='secretsmanager', + region_name=region_name + ) + + try: + get_secret_value_response = client.get_secret_value( + SecretId=secret_name + ) + except ClientError as e: # leaving some useful debug info to help narrow issues + if e.response['Error']['Code'] == 'DecryptionFailureException': + # Secrets Manager can't decrypt the protected secret text using the provided KMS key. + raise e + elif e.response['Error']['Code'] == 'InternalServiceErrorException': + # An error occurred on the server side. + raise e + elif e.response['Error']['Code'] == 'InvalidParameterException': + # You provided an invalid value for a parameter. + raise e + elif e.response['Error']['Code'] == 'InvalidRequestException': + # You provided a parameter value that is not valid for the current state of the resource. + raise e + elif e.response['Error']['Code'] == 'ResourceNotFoundException': + raise e + else: + # Decrypts secret using the associated KMS CMK. + # Depending on whether the secret is a string or binary, one of these fields will be populated. + if 'SecretString' in get_secret_value_response: + identity = json.loads(get_secret_value_response['SecretString']) + else: + print('Got unexpected response structure from boto3') + exit(1) + + # build production.ini + with override_environ(identity): # noQA exit above + CGAPDockerIniFileManager.main() + + +if __name__ == '__main__': + assume_identity() diff --git a/deploy/docker/production/docker-compose.yml b/deploy/docker/production/docker-compose.yml new file mode 100644 index 0000000000..09e0235faf --- /dev/null +++ b/deploy/docker/production/docker-compose.yml @@ -0,0 +1,13 @@ +version: "3.8" + +services: + + # Production component + cgap-production: + build: . + container_name: cgap + command: "/home/nginx/cgap-portal/entrypoint.sh" + environment: + IDENTITY: + ports: + - "8000:8000" # nginx proxy port (note application traffic is not forwarded) diff --git a/deploy/docker/production/entrypoint.sh b/deploy/docker/production/entrypoint.sh index 097f1a37ac..705cc67699 100644 --- a/deploy/docker/production/entrypoint.sh +++ b/deploy/docker/production/entrypoint.sh @@ -1,5 +1,9 @@ #!/bin/sh +# Run assume_identity.py to access the desired deployment configuration from +# secrets manager - this builds production.ini +python assume_identity.py + # Clear db/es since this is the local entry point poetry run clear-db-es-contents production.ini --app-name app --env $CGAP_ENV_NAME diff --git a/deploy/docker/production/install_nginx.sh b/deploy/docker/production/install_nginx.sh new file mode 100644 index 0000000000..98e82d6187 --- /dev/null +++ b/deploy/docker/production/install_nginx.sh @@ -0,0 +1,77 @@ +# Copied from: https://github.com/nginxinc/docker-nginx/blob/594ce7a8bc26c85af88495ac94d5cd0096b306f7/mainline/buster/Dockerfile + +# Standard set up Nginx +export NGINX_VERSION=1.17.10 +export NJS_VERSION=0.3.9 +export PKG_RELEASE=1~buster + +# Securely provisions deps, installs Nginx, then removes all artifacts +set -x \ + && apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y gnupg1 ca-certificates \ + && \ + NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \ + found=''; \ + for server in \ + ha.pool.sks-keyservers.net \ + hkp://keyserver.ubuntu.com:80 \ + hkp://p80.pool.sks-keyservers.net:80 \ + pgp.mit.edu \ + ; do \ + echo "Fetching GPG key $NGINX_GPGKEY from $server"; \ + apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \ + done; \ + test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \ + apt-get remove --purge --auto-remove -y gnupg1 && rm -rf /var/lib/apt/lists/* \ + && dpkgArch="$(dpkg --print-architecture)" \ + && nginxPackages=" \ + nginx=${NGINX_VERSION}-${PKG_RELEASE} \ + nginx-module-xslt=${NGINX_VERSION}-${PKG_RELEASE} \ + nginx-module-geoip=${NGINX_VERSION}-${PKG_RELEASE} \ + nginx-module-image-filter=${NGINX_VERSION}-${PKG_RELEASE} \ + nginx-module-njs=${NGINX_VERSION}.${NJS_VERSION}-${PKG_RELEASE} \ + " \ + && case "$dpkgArch" in \ + amd64|i386) \ + echo "deb https://nginx.org/packages/mainline/debian/ buster nginx" >> /etc/apt/sources.list.d/nginx.list \ + && apt-get update \ + ;; \ + *) \ + echo "deb-src https://nginx.org/packages/mainline/debian/ buster nginx" >> /etc/apt/sources.list.d/nginx.list \ + \ + && tempDir="$(mktemp -d)" \ + && chmod 777 "$tempDir" \ + \ + && savedAptMark="$(apt-mark showmanual)" \ + \ + && apt-get update \ + && apt-get build-dep -y $nginxPackages \ + && ( \ + cd "$tempDir" \ + && DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" \ + apt-get source --compile $nginxPackages \ + ) \ + \ + && apt-mark showmanual | xargs apt-mark auto > /dev/null \ + && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; } \ + \ + && ls -lAFh "$tempDir" \ + && ( cd "$tempDir" && dpkg-scanpackages . > Packages ) \ + && grep '^Package: ' "$tempDir/Packages" \ + && echo "deb [ trusted=yes ] file://$tempDir ./" > /etc/apt/sources.list.d/temp.list \ + && apt-get -o Acquire::GzipIndexes=false update \ + ;; \ + esac \ + \ + && apt-get install --no-install-recommends --no-install-suggests -y \ + $nginxPackages \ + gettext-base \ + && apt-get remove --purge --auto-remove -y ca-certificates && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx.list \ + \ + && if [ -n "$tempDir" ]; then \ + apt-get purge -y --auto-remove \ + && rm -rf "$tempDir" /etc/apt/sources.list.d/temp.list; \ + fi + +# forward request and error logs to docker log collector +ln -sf /dev/stdout /var/log/nginx/access.log && ln -sf /dev/stderr /var/log/nginx/error.log diff --git a/docker-compose.yml b/docker-compose.yml index 15a6e30db2..940ef20e66 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,7 +44,7 @@ services: app: build: ./deploy/docker/local container_name: cgap - command: "/home/cgap-admin/cgap-portal/entrypoint.sh" + command: "/home/nginx/cgap-portal/entrypoint.sh" environment: # Default to caller's environment AWS_ACCESS_KEY_ID: diff --git a/src/encoded/ingestion/variant_utils.py b/src/encoded/ingestion/variant_utils.py index 208f0bee97..293faa42fe 100644 --- a/src/encoded/ingestion/variant_utils.py +++ b/src/encoded/ingestion/variant_utils.py @@ -97,6 +97,7 @@ def _post_or_patch_variant_sample(self, variant_sample): log.info('Exception encountered on variant_sample post (attempting patch): %s' % e) self.vapp.patch_json('/variant_sample/%s' % variant_sample['CALL_INFO'] + ':' + variant_sample['variant'], + variant_sample, status=200) def build_variant(self, record): From e3fd154e5fc7cecebe7b103266dc9a2c75a75dcd Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 6 Apr 2021 12:54:54 -0400 Subject: [PATCH 036/120] add ability to build prod image --- deploy/docker/production/Makefile | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 deploy/docker/production/Makefile diff --git a/deploy/docker/production/Makefile b/deploy/docker/production/Makefile new file mode 100644 index 0000000000..6fd5d25348 --- /dev/null +++ b/deploy/docker/production/Makefile @@ -0,0 +1,12 @@ +login: + aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 645819926742.dkr.ecr.us-east-1.amazonaws.com + +build: + docker build -t cgap-docker . + +tag: + docker tag cgap-docker:latest 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-docker:latest + +push: + docker push 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-docker:latest + From 01c8dc0a36d83e368b7c964f1e6471f992f6a1e2 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 29 Apr 2021 08:58:49 -0400 Subject: [PATCH 037/120] small changes --- deploy/docker/production/assume_identity.py | 9 ++++++++- deploy/docker/production/entrypoint.sh | 20 ++++++++++++-------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/deploy/docker/production/assume_identity.py b/deploy/docker/production/assume_identity.py index 86a1b54467..86935be581 100644 --- a/deploy/docker/production/assume_identity.py +++ b/deploy/docker/production/assume_identity.py @@ -7,12 +7,16 @@ import os import json import boto3 +import logging from botocore.exceptions import ClientError from dcicutils.beanstalk_utils import REGION from dcicutils.qa_utils import override_environ from dcicutils.deployment_utils import IniFileManager +logger = logging.getLogger(__file__) + + class CGAPDockerIniFileManager(IniFileManager): """ This runs at top level, so path is slightly different. """ _MY_DIR = os.path.dirname(__file__) @@ -58,13 +62,16 @@ def assume_identity(): # Depending on whether the secret is a string or binary, one of these fields will be populated. if 'SecretString' in get_secret_value_response: identity = json.loads(get_secret_value_response['SecretString']) + logger.error('got a secret') else: - print('Got unexpected response structure from boto3') + logger.error('Got unexpected response structure from boto3') exit(1) # build production.ini with override_environ(identity): # noQA exit above + logger.error('building the ini file') CGAPDockerIniFileManager.main() + logger.error('production.ini should be here now') if __name__ == '__main__': diff --git a/deploy/docker/production/entrypoint.sh b/deploy/docker/production/entrypoint.sh index 705cc67699..42c2c0e9db 100644 --- a/deploy/docker/production/entrypoint.sh +++ b/deploy/docker/production/entrypoint.sh @@ -1,17 +1,21 @@ #!/bin/sh +# Echo something at startup +echo "Starting up CGAP Portal" + # Run assume_identity.py to access the desired deployment configuration from # secrets manager - this builds production.ini python assume_identity.py -# Clear db/es since this is the local entry point -poetry run clear-db-es-contents production.ini --app-name app --env $CGAP_ENV_NAME - -# Create mapping -poetry run create-mapping-on-deploy production.ini --app-name app - -# Load Data (based on development.ini, for now just master-inserts) -poetry run load-data production.ini --app-name app --prod +# XXX: this is deployment stuff, doesn't need to be run at this time +## Clear db/es since this is the local entry point +#poetry run clear-db-es-contents production.ini --app-name app --env $CGAP_ENV_NAME +# +## Create mapping +#poetry run create-mapping-on-deploy production.ini --app-name app +# +## Load Data (based on development.ini, for now just master-inserts) +#poetry run load-data production.ini --app-name app --prod # Start nginx proxy service nginx start From 0fdcd573799db3d684dfb8725a8fe81388a47165 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 29 Apr 2021 09:44:47 -0400 Subject: [PATCH 038/120] point to cgap-mastertest (current ecr), test os.system logging --- deploy/docker/production/Makefile | 6 +++--- deploy/docker/production/assume_identity.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/deploy/docker/production/Makefile b/deploy/docker/production/Makefile index 6fd5d25348..5cecd6b194 100644 --- a/deploy/docker/production/Makefile +++ b/deploy/docker/production/Makefile @@ -2,11 +2,11 @@ login: aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 645819926742.dkr.ecr.us-east-1.amazonaws.com build: - docker build -t cgap-docker . + docker build -t cgap-mastertest . tag: - docker tag cgap-docker:latest 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-docker:latest + docker tag cgap-mastertest:latest 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest push: - docker push 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-docker:latest + docker push 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest diff --git a/deploy/docker/production/assume_identity.py b/deploy/docker/production/assume_identity.py index 86935be581..374a49d0a9 100644 --- a/deploy/docker/production/assume_identity.py +++ b/deploy/docker/production/assume_identity.py @@ -62,16 +62,16 @@ def assume_identity(): # Depending on whether the secret is a string or binary, one of these fields will be populated. if 'SecretString' in get_secret_value_response: identity = json.loads(get_secret_value_response['SecretString']) - logger.error('got a secret') + os.system('echo got a secret') else: - logger.error('Got unexpected response structure from boto3') + os.system('echo Got unexpected response structure from boto3') exit(1) # build production.ini with override_environ(identity): # noQA exit above - logger.error('building the ini file') + os.system('echo building the ini file') CGAPDockerIniFileManager.main() - logger.error('production.ini should be here now') + os.system('echo production.ini should be here now') if __name__ == '__main__': From c9425e6072ba41f5df2532a4ba9a16ee1c5205fe Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 29 Apr 2021 10:10:50 -0400 Subject: [PATCH 039/120] add watchtower for log forwarding to cloudwatch --- deploy/docker/production/assume_identity.py | 3 +++ poetry.lock | 17 ++++++++++++++++- pyproject.toml | 1 + 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/deploy/docker/production/assume_identity.py b/deploy/docker/production/assume_identity.py index 374a49d0a9..843a2299f3 100644 --- a/deploy/docker/production/assume_identity.py +++ b/deploy/docker/production/assume_identity.py @@ -8,13 +8,16 @@ import json import boto3 import logging +import watchtower from botocore.exceptions import ClientError from dcicutils.beanstalk_utils import REGION from dcicutils.qa_utils import override_environ from dcicutils.deployment_utils import IniFileManager +logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__file__) +logger.addHandler(watchtower.CloudWatchLogHandler()) class CGAPDockerIniFileManager(IniFileManager): diff --git a/poetry.lock b/poetry.lock index d875fb228f..47bd05718e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1928,6 +1928,17 @@ python-versions = "*" docs = ["Sphinx (>=1.8.1)", "docutils", "pylons-sphinx-themes (>=1.0.9)"] testing = ["nose", "coverage"] +[[package]] +name = "watchtower" +version = "1.0.6" +description = "Python CloudWatch Logging" +category = "main" +optional = false +python-versions = ">=3.5" + +[package.dependencies] +boto3 = ">=1.9.253,<2" + [[package]] name = "webob" version = "1.8.7" @@ -2073,7 +2084,7 @@ test = ["zope.testing"] [metadata] lock-version = "1.1" python-versions = ">=3.6.1,<3.7" -content-hash = "abc0588a86d0a588f0a5c30c913ff9d12621bea9bfaad336a0921433110f4d76" +content-hash = "f757ac1b0d257e70d7fc609424468b780f99229bba95d4625531c5863ed25b2d" [metadata.files] apipkg = [ @@ -3011,6 +3022,10 @@ waitress = [ {file = "waitress-1.2.0-py2.py3-none-any.whl", hash = "sha256:8b8c8686f628a635b9747e3014a0ab19cf9cf95c5c36eb3331ae355a462ee602"}, {file = "waitress-1.2.0.tar.gz", hash = "sha256:e624c829656ffc99b33d661072b2814885ae92835cf835ee8ab283ddb7c915b9"}, ] +watchtower = [ + {file = "watchtower-1.0.6-py3-none-any.whl", hash = "sha256:2859275df4ad71b005b983613dd64cabbda61f9fdd3db7600753fc465090119d"}, + {file = "watchtower-1.0.6.tar.gz", hash = "sha256:5eb5d78e730e1016e166b14a79a02d1b939cf1a58f2d559ff4f7c6f953284ebf"}, +] webob = [ {file = "WebOb-1.8.7-py2.py3-none-any.whl", hash = "sha256:73aae30359291c14fa3b956f8b5ca31960e420c28c1bec002547fb04928cf89b"}, {file = "WebOb-1.8.7.tar.gz", hash = "sha256:b64ef5141be559cfade448f044fa45c2260351edcb6a8ef6b7e00c7dcef0c323"}, diff --git a/pyproject.toml b/pyproject.toml index b454991a6d..86a0525a32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,6 +116,7 @@ WSGIProxy2 = "0.4.2" sentry-sdk = "^0.16.5" granite-suite = "0.1.11b0" openpyxl = "^3.0.7" +watchtower = "^1.0.6" [tool.poetry.dev-dependencies] # PyCharm says boto3-stubs contains useful type hints From f8a3fe1a18ec08ba4c0dc11d6137be23c7574365 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 29 Apr 2021 10:50:54 -0400 Subject: [PATCH 040/120] invoke assume_identity with poetry...? --- deploy/docker/production/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/docker/production/entrypoint.sh b/deploy/docker/production/entrypoint.sh index 42c2c0e9db..a7204c26b0 100644 --- a/deploy/docker/production/entrypoint.sh +++ b/deploy/docker/production/entrypoint.sh @@ -5,7 +5,7 @@ echo "Starting up CGAP Portal" # Run assume_identity.py to access the desired deployment configuration from # secrets manager - this builds production.ini -python assume_identity.py +poetry run assume_identity.py # XXX: this is deployment stuff, doesn't need to be run at this time ## Clear db/es since this is the local entry point From 1c2387dd903179e72e4443d4afdb11bbdefe0ca2 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 29 Apr 2021 10:55:08 -0400 Subject: [PATCH 041/120] invoke python directly? --- deploy/docker/production/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/docker/production/entrypoint.sh b/deploy/docker/production/entrypoint.sh index a7204c26b0..f5d4eeff7e 100644 --- a/deploy/docker/production/entrypoint.sh +++ b/deploy/docker/production/entrypoint.sh @@ -5,7 +5,7 @@ echo "Starting up CGAP Portal" # Run assume_identity.py to access the desired deployment configuration from # secrets manager - this builds production.ini -poetry run assume_identity.py +poetry run python -m assume_identity.py # XXX: this is deployment stuff, doesn't need to be run at this time ## Clear db/es since this is the local entry point From 0d7d57dedc7c5af086c7488fca83a096add1b09e Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 29 Apr 2021 11:39:17 -0400 Subject: [PATCH 042/120] repair branch, pass boto3 session to logger --- deploy/docker/production/Dockerfile | 4 ++-- deploy/docker/production/assume_identity.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/deploy/docker/production/Dockerfile b/deploy/docker/production/Dockerfile index f2cdb6b802..0032a0399c 100644 --- a/deploy/docker/production/Dockerfile +++ b/deploy/docker/production/Dockerfile @@ -7,9 +7,9 @@ MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" # Build Arguments ARG CGAP_ENV_NAME -ENV CGAP_ENV_NAME=${CGAP_ENV_NAME:-"fourfront-cgapmastertest"} +ENV CGAP_ENV_NAME=${CGAP_ENV_NAME:-"cgap-mastertest"} ARG CGAP_BRANCH -ENV CGAP_BRANCH=${CGAP_BRANCH:-"master"} +ENV CGAP_BRANCH=${CGAP_BRANCH:-"c4_519"} # Configure (global) Env ENV CGAP_REPO=https://github.com/dbmi-bgm/cgap-portal.git diff --git a/deploy/docker/production/assume_identity.py b/deploy/docker/production/assume_identity.py index 843a2299f3..33f31ec27d 100644 --- a/deploy/docker/production/assume_identity.py +++ b/deploy/docker/production/assume_identity.py @@ -17,7 +17,6 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__file__) -logger.addHandler(watchtower.CloudWatchLogHandler()) class CGAPDockerIniFileManager(IniFileManager): @@ -36,6 +35,8 @@ def assume_identity(): # XXX: We should refactor a SecretsManager wrapper into dcicutils session = boto3.session.Session() + # configure watchtower handler from session + logger.addHandler(watchtower.CloudWatchLogHandler(boto3_session=session)) client = session.client( service_name='secretsmanager', region_name=region_name From 1b5b7c28c102b9db7ffef18807dd55206ef5932b Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 29 Apr 2021 12:34:12 -0400 Subject: [PATCH 043/120] pass to python correctly --- deploy/docker/production/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/docker/production/entrypoint.sh b/deploy/docker/production/entrypoint.sh index f5d4eeff7e..1f974375c5 100644 --- a/deploy/docker/production/entrypoint.sh +++ b/deploy/docker/production/entrypoint.sh @@ -5,7 +5,7 @@ echo "Starting up CGAP Portal" # Run assume_identity.py to access the desired deployment configuration from # secrets manager - this builds production.ini -poetry run python -m assume_identity.py +poetry run python -m assume_identity # XXX: this is deployment stuff, doesn't need to be run at this time ## Clear db/es since this is the local entry point From 548ab463858ca0b0014e38c7ed9f3b4b3a3eda3e Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 29 Apr 2021 12:42:15 -0400 Subject: [PATCH 044/120] pass region to session --- deploy/docker/production/assume_identity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/docker/production/assume_identity.py b/deploy/docker/production/assume_identity.py index 33f31ec27d..fc8c4fc114 100644 --- a/deploy/docker/production/assume_identity.py +++ b/deploy/docker/production/assume_identity.py @@ -34,7 +34,7 @@ def assume_identity(): region_name = REGION # us-east-1 # XXX: We should refactor a SecretsManager wrapper into dcicutils - session = boto3.session.Session() + session = boto3.session.Session(region_name=region_name) # configure watchtower handler from session logger.addHandler(watchtower.CloudWatchLogHandler(boto3_session=session)) client = session.client( From 585cc549627a4e255391266de57d746a9e38161a Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 29 Apr 2021 13:00:25 -0400 Subject: [PATCH 045/120] force some output --- deploy/docker/production/assume_identity.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/deploy/docker/production/assume_identity.py b/deploy/docker/production/assume_identity.py index fc8c4fc114..9a5ba63eed 100644 --- a/deploy/docker/production/assume_identity.py +++ b/deploy/docker/production/assume_identity.py @@ -66,16 +66,15 @@ def assume_identity(): # Depending on whether the secret is a string or binary, one of these fields will be populated. if 'SecretString' in get_secret_value_response: identity = json.loads(get_secret_value_response['SecretString']) - os.system('echo got a secret') + raise Exception('got a secret') else: - os.system('echo Got unexpected response structure from boto3') - exit(1) + raise Exception('Got unexpected response structure from boto3') # build production.ini with override_environ(identity): # noQA exit above - os.system('echo building the ini file') + raise Exception('building the ini file') CGAPDockerIniFileManager.main() - os.system('echo production.ini should be here now') + logger.error('production.ini should be here now') if __name__ == '__main__': From 9e6d112bc8992edef7e6a12dcb342873b3df5408 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 29 Apr 2021 13:59:14 -0400 Subject: [PATCH 046/120] small fixes --- deploy/docker/production/assume_identity.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/deploy/docker/production/assume_identity.py b/deploy/docker/production/assume_identity.py index 9a5ba63eed..efedbbf01e 100644 --- a/deploy/docker/production/assume_identity.py +++ b/deploy/docker/production/assume_identity.py @@ -61,20 +61,19 @@ def assume_identity(): raise e elif e.response['Error']['Code'] == 'ResourceNotFoundException': raise e + else: + raise e else: # Decrypts secret using the associated KMS CMK. # Depending on whether the secret is a string or binary, one of these fields will be populated. if 'SecretString' in get_secret_value_response: identity = json.loads(get_secret_value_response['SecretString']) - raise Exception('got a secret') else: raise Exception('Got unexpected response structure from boto3') # build production.ini - with override_environ(identity): # noQA exit above - raise Exception('building the ini file') + with override_environ(**identity): # noQA exit above CGAPDockerIniFileManager.main() - logger.error('production.ini should be here now') if __name__ == '__main__': From 89580e5d506b8cd169abfa5430df5fd75b214434 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 29 Apr 2021 14:13:28 -0400 Subject: [PATCH 047/120] add mastertest ini --- deploy/docker/production/Dockerfile | 6 ++ deploy/docker/production/mastertest.ini | 84 +++++++++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 deploy/docker/production/mastertest.ini diff --git a/deploy/docker/production/Dockerfile b/deploy/docker/production/Dockerfile index 0032a0399c..62f5f87eb2 100644 --- a/deploy/docker/production/Dockerfile +++ b/deploy/docker/production/Dockerfile @@ -77,6 +77,12 @@ RUN touch /var/log/nginx/access.log && \ RUN touch /var/log/nginx/error.log && \ chown -R nginx:nginx /var/log/nginx/error.log +# Provide base ini file +# will be picked up by IniFileManager +# *.ini must match the env name in secrets manager! +# For now, this is mastertest. - Will 04/29/21 +COPY mastertest.ini deploy/ini_files/. + COPY entrypoint.sh . COPY assume_identity.py . RUN chmod +x entrypoint.sh diff --git a/deploy/docker/production/mastertest.ini b/deploy/docker/production/mastertest.ini new file mode 100644 index 0000000000..ab8db883ef --- /dev/null +++ b/deploy/docker/production/mastertest.ini @@ -0,0 +1,84 @@ +[app:app] +use = config:base.ini#app +session.secret = %(here)s/session-secret.b64 +file_upload_bucket = ${ENCODED_FILES_BUCKET} +file_wfout_bucket = ${ENCODED_WFOUT_BUCKET} +blob_bucket = ${ENCODED_BLOBS_BUCKET} +system_bucket = ${ENCODED_SYSTEM_BUCKET} +metadata_bundles_bucket = ${ENCODED_METADATA_BUNDLE_BUCKET} +sentry_dsn = ${SENTRY_DSN} +accession_factory = encoded.server_defaults.enc_accession +elasticsearch.server = ${ES_SERVER} +snovault.app_version = ${APP_VERSION} +env.name = cgap-mastertest +encoded_version = ${PROJECT_VERSION} +eb_app_version = ${APP_VERSION} +snovault_version = ${SNOVAULT_VERSION} +utils_version = ${UTILS_VERSION} +mpindexer = true +indexer = ${INDEXER} +indexer.namespace = cgap-mastertest +index_server = ${INDEX_SERVER} +elasticsearch.aws_auth = true +production = true +load_test_data = encoded.loadxl:load_prod_data +sqlalchemy.url = postgresql://${RDS_USERNAME}:${RDS_PASSWORD}@${RDS_HOSTNAME}:${RDS_PORT}/${RDS_DB_NAME} + +[composite:indexer] +use = config:base.ini#indexer + +[composite:ingester] +use = config:base.ini#ingester + +[pipeline:main] +pipeline = + config:base.ini#memlimit + egg:PasteDeploy#prefix + app + +[pipeline:debug] +pipeline = + egg:repoze.debug#pdbpm + app +set pyramid.includes = + pyramid_translogger + +[server:main] +use = egg:waitress#main +host = 0.0.0.0 +port = 6543 +threads = 1 + +[loggers] +keys = root, encoded, encoded_listener + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console + +[logger_encoded] +level = WARN +handlers = console +qualname = encoded +propagate = 0 + +[logger_encoded_listener] +level = INFO +handlers = console +qualname = snovault.elasticsearch.es_index_listener +propagate = 0 + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(message)s \ No newline at end of file From 5d242d32447d8c57b908c7a731f4364dd0f22360 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 29 Apr 2021 14:33:20 -0400 Subject: [PATCH 048/120] fix merge conflict --- deploy/docker/production/Dockerfile | 8 +++++--- deploy/docker/production/assume_identity.py | 10 ++++++---- src/encoded/ingestion/variant_utils.py | 5 +---- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/deploy/docker/production/Dockerfile b/deploy/docker/production/Dockerfile index 62f5f87eb2..0308a27124 100644 --- a/deploy/docker/production/Dockerfile +++ b/deploy/docker/production/Dockerfile @@ -82,15 +82,17 @@ RUN touch /var/log/nginx/error.log && \ # *.ini must match the env name in secrets manager! # For now, this is mastertest. - Will 04/29/21 COPY mastertest.ini deploy/ini_files/. +RUN touch production.ini +RUN chown nginx:nginx production.ini COPY entrypoint.sh . COPY assume_identity.py . RUN chmod +x entrypoint.sh RUN chmod +x assume_identity.py EXPOSE 8000 -# Container does not ever run as root -# This way even if someone gets access to the container they cannot -# read production.ini for example + +# Container does not run as root USER nginx +WORKDIR /home/nginx/cgap-portal ENTRYPOINT ["/home/nginx/cgap-portal/entrypoint.sh"] diff --git a/deploy/docker/production/assume_identity.py b/deploy/docker/production/assume_identity.py index efedbbf01e..cfbd372d2e 100644 --- a/deploy/docker/production/assume_identity.py +++ b/deploy/docker/production/assume_identity.py @@ -17,13 +17,14 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__file__) +_MY_DIR = os.path.dirname(__file__) class CGAPDockerIniFileManager(IniFileManager): """ This runs at top level, so path is slightly different. """ - _MY_DIR = os.path.dirname(__file__) - TEMPLATE_DIR = os.path.join(_MY_DIR, "deploy/ini_files") - PYPROJECT_FILE_NAME = os.path.join(os.path.dirname(_MY_DIR), "pyproject.toml") + # should work but doesn't (missing cgap-portal): os.path.join(os.path.dirname(_MY_DIR), "pyproject.toml") + TEMPLATE_DIR = '/home/nginx/cgap-portal/deploy/ini_files' + PYPROJECT_FILE_NAME = '/home/nginx/cgap-portal/pyproject.toml' def assume_identity(): @@ -72,7 +73,8 @@ def assume_identity(): raise Exception('Got unexpected response structure from boto3') # build production.ini - with override_environ(**identity): # noQA exit above + with override_environ(**identity): + CGAPDockerIniFileManager.main() diff --git a/src/encoded/ingestion/variant_utils.py b/src/encoded/ingestion/variant_utils.py index cc22d88835..e8207cae10 100644 --- a/src/encoded/ingestion/variant_utils.py +++ b/src/encoded/ingestion/variant_utils.py @@ -101,12 +101,9 @@ def _post_or_patch_variant_sample(self, variant_sample, variant_uuid): except Exception as e: # noqa exceptions thrown by the above call are not reported correctly log.info('Exception encountered on variant_sample post (attempting patch): %s' % e) self.vapp.patch_json('/variant_sample/%s' % -<<<<<<< HEAD - variant_sample['CALL_INFO'] + ':' + variant_sample['variant'], -======= + build_variant_sample_annotation_id(variant_sample['CALL_INFO'], variant_uuid, self.file), ->>>>>>> master variant_sample, status=200) From 77c1c77033d033323510fa72ca9054c81d14897e Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 29 Apr 2021 16:25:55 -0400 Subject: [PATCH 049/120] add composite flag for main --- deploy/docker/production/assume_identity.py | 4 ++++ deploy/docker/production/mastertest.ini | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/deploy/docker/production/assume_identity.py b/deploy/docker/production/assume_identity.py index cfbd372d2e..989c1c1a8c 100644 --- a/deploy/docker/production/assume_identity.py +++ b/deploy/docker/production/assume_identity.py @@ -23,6 +23,10 @@ class CGAPDockerIniFileManager(IniFileManager): """ This runs at top level, so path is slightly different. """ # should work but doesn't (missing cgap-portal): os.path.join(os.path.dirname(_MY_DIR), "pyproject.toml") + # TODO: repair via + # expected = + # actual = + # assert actual == expected, "The actual value %s was not what we expected, %s." % (actual, expected) TEMPLATE_DIR = '/home/nginx/cgap-portal/deploy/ini_files' PYPROJECT_FILE_NAME = '/home/nginx/cgap-portal/pyproject.toml' diff --git a/deploy/docker/production/mastertest.ini b/deploy/docker/production/mastertest.ini index ab8db883ef..c50830c8e6 100644 --- a/deploy/docker/production/mastertest.ini +++ b/deploy/docker/production/mastertest.ini @@ -24,6 +24,11 @@ production = true load_test_data = encoded.loadxl:load_prod_data sqlalchemy.url = postgresql://${RDS_USERNAME}:${RDS_PASSWORD}@${RDS_HOSTNAME}:${RDS_PORT}/${RDS_DB_NAME} +[composite:main] +use = egg:rutter#urlmap +/ = debug +/_indexer = indexer + [composite:indexer] use = config:base.ini#indexer From 75ff6bb1d6b02b2e852813402c60f63c21dd49e7 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 29 Apr 2021 16:30:29 -0400 Subject: [PATCH 050/120] remove composite app --- deploy/docker/production/mastertest.ini | 5 ----- 1 file changed, 5 deletions(-) diff --git a/deploy/docker/production/mastertest.ini b/deploy/docker/production/mastertest.ini index c50830c8e6..ab8db883ef 100644 --- a/deploy/docker/production/mastertest.ini +++ b/deploy/docker/production/mastertest.ini @@ -24,11 +24,6 @@ production = true load_test_data = encoded.loadxl:load_prod_data sqlalchemy.url = postgresql://${RDS_USERNAME}:${RDS_PASSWORD}@${RDS_HOSTNAME}:${RDS_PORT}/${RDS_DB_NAME} -[composite:main] -use = egg:rutter#urlmap -/ = debug -/_indexer = indexer - [composite:indexer] use = config:base.ini#indexer From 260cdd97224fc6e0ab08dec9d4edc366e7b695b0 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Fri, 30 Apr 2021 16:59:54 -0400 Subject: [PATCH 051/120] provision indexer --- deploy/docker/production/Dockerfile_indexer | 100 ++++++++++++++++++ deploy/docker/production/Makefile | 12 +-- deploy/docker/production/entrypoint.sh | 3 +- .../docker/production/entrypoint_indexer.sh | 10 ++ 4 files changed, 117 insertions(+), 8 deletions(-) create mode 100644 deploy/docker/production/Dockerfile_indexer create mode 100644 deploy/docker/production/entrypoint_indexer.sh diff --git a/deploy/docker/production/Dockerfile_indexer b/deploy/docker/production/Dockerfile_indexer new file mode 100644 index 0000000000..ca6495033b --- /dev/null +++ b/deploy/docker/production/Dockerfile_indexer @@ -0,0 +1,100 @@ +# CGAP-Portal Indexer Dockerfile +# TODO: figure out how to manage this better (no duplication) + +# TODO: appropriately pin/verify this image +FROM python:3.6.12-buster + +MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" + +# Build Arguments +ARG CGAP_ENV_NAME +ENV CGAP_ENV_NAME=${CGAP_ENV_NAME:-"cgap-mastertest"} +ARG CGAP_BRANCH +ENV CGAP_BRANCH=${CGAP_BRANCH:-"c4_519"} + +# Configure (global) Env +ENV CGAP_REPO=https://github.com/dbmi-bgm/cgap-portal.git +ENV NGINX_USER=nginx +ENV DEBIAN_FRONTEND=noninteractive +ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1 +ENV PYTHONFAULTHANDLER=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONHASHSEED=random \ + PIP_NO_CACHE_DIR=off \ + PIP_DISABLE_PIP_VERSION_CHECK=on \ + PIP_DEFAULT_TIMEOUT=100 \ + POETRY_VERSION=1.1.4 + +# Install nginx +COPY install_nginx.sh / +RUN bash /install_nginx.sh + +# Intall things needed for our system +RUN apt-get update +RUN apt-get install -y curl vim emacs postgresql-client net-tools +RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - +RUN apt-get install -y ca-certificates nodejs npm + +# Configure CGAP User +WORKDIR /home/nginx +RUN mkdir -p /home/nginx/cgap-portal + +# Configure venv +ENV VIRTUAL_ENV=/opt/venv +RUN python -m venv /opt/venv +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +RUN chown -R nginx:nginx /opt/venv + +# Copy to /home/nginx/cgap-portal +RUN git clone $CGAP_REPO --branch $CGAP_BRANCH + +# Build the application +WORKDIR /home/nginx/cgap-portal +RUN pip install --upgrade pip +RUN pip install poetry==1.1.4 wheel==0.29.0 +RUN poetry install +RUN python setup_eb.py develop +RUN make fix-dist-info +RUN npm ci --no-fund --no-progress --no-optional --no-audit --python=/opt/venv/bin/python +RUN npm run build +RUN npm run build-scss +RUN make aws-ip-ranges +RUN cat /dev/urandom | head -c 256 | base64 > session-secret.b64 + +# Copy config files in (down here for quick debugging) +# Remove default configuration from Nginx +RUN rm /etc/nginx/nginx.conf +RUN rm /etc/nginx/conf.d/default.conf +COPY nginx.conf /etc/nginx/nginx.conf +# give nginx user permissions +RUN chown -R nginx:nginx /var/cache/nginx && \ + chown -R nginx:nginx /var/log/nginx && \ + chown -R nginx:nginx /etc/nginx/conf.d +RUN touch /var/run/nginx.pid && \ + chown -R nginx:nginx /var/run/nginx.pid +RUN rm -f /var/log/nginx/* +RUN touch /var/log/nginx/access.log && \ + chown -R nginx:nginx /var/log/nginx/access.log +RUN touch /var/log/nginx/error.log && \ + chown -R nginx:nginx /var/log/nginx/error.log + +# Provide base ini file +# will be picked up by IniFileManager +# *.ini must match the env name in secrets manager! +# For now, this is mastertest. - Will 04/29/21 +COPY mastertest.ini deploy/ini_files/. +RUN touch production.ini +RUN chown nginx:nginx production.ini + +COPY entrypoint.sh . +COPY entrypoint_indexer.sh entrypoint.sh +COPY assume_identity.py . +RUN chmod +x entrypoint.sh +RUN chmod +x assume_identity.py +EXPOSE 8000 + +# Container does not run as root +USER nginx +WORKDIR /home/nginx/cgap-portal + +ENTRYPOINT ["/home/nginx/cgap-portal/entrypoint.sh"] diff --git a/deploy/docker/production/Makefile b/deploy/docker/production/Makefile index 5cecd6b194..d7b74c3d6a 100644 --- a/deploy/docker/production/Makefile +++ b/deploy/docker/production/Makefile @@ -1,12 +1,12 @@ login: aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 645819926742.dkr.ecr.us-east-1.amazonaws.com -build: - docker build -t cgap-mastertest . - -tag: +wsgi: + docker build -t cgap-mastertest:latest . docker tag cgap-mastertest:latest 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest - -push: docker push 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest +indexer: + docker build -t cgap-mastertest:latest-indexer -f Dockerfile_indexer . + docker tag cgap-mastertest:latest-indexer 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-indexer + docker push 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-indexer diff --git a/deploy/docker/production/entrypoint.sh b/deploy/docker/production/entrypoint.sh index 1f974375c5..1d0589c5e3 100644 --- a/deploy/docker/production/entrypoint.sh +++ b/deploy/docker/production/entrypoint.sh @@ -1,7 +1,6 @@ #!/bin/sh -# Echo something at startup -echo "Starting up CGAP Portal" +echo "Starting up CGAP-Portal WSGI" # Run assume_identity.py to access the desired deployment configuration from # secrets manager - this builds production.ini diff --git a/deploy/docker/production/entrypoint_indexer.sh b/deploy/docker/production/entrypoint_indexer.sh new file mode 100644 index 0000000000..e221c8b8ae --- /dev/null +++ b/deploy/docker/production/entrypoint_indexer.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +echo "Starting up CGAP-Portal Indexer" + +# Run assume_identity.py to access the desired deployment configuration from +# secrets manager - this builds production.ini +poetry run python -m assume_identity + +# Start indexer listener +es-index-listener production.ini --app-name app --verbose From 4cc7d71d735f39c165b008d20a1d1a2da22fbb21 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 4 May 2021 13:48:42 -0400 Subject: [PATCH 052/120] bring in indexer fix --- poetry.lock | 709 ++++++++++++++++++++++++------------------------- pyproject.toml | 2 +- 2 files changed, 352 insertions(+), 359 deletions(-) diff --git a/poetry.lock b/poetry.lock index 47bd05718e..1d94d339a9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -54,18 +54,18 @@ wrapt = "*" [[package]] name = "awscli" -version = "1.19.32" +version = "1.19.53" description = "Universal Command Line Environment for AWS." category = "main" optional = false python-versions = ">= 2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" [package.dependencies] -botocore = "1.20.32" +botocore = "1.20.53" colorama = ">=0.2.5,<0.4.4" docutils = ">=0.10,<0.16" PyYAML = ">=3.10,<5.5" -rsa = ">=3.1.2,<=4.5.0" +rsa = {version = ">=3.1.2,<4.8", markers = "python_version > \"2.7\""} s3transfer = ">=0.3.0,<0.4.0" [[package]] @@ -93,7 +93,7 @@ lxml = ["lxml"] [[package]] name = "bitarray" -version = "1.7.1" +version = "2.0.1" description = "efficient arrays of booleans -- C extension" category = "main" optional = false @@ -109,21 +109,21 @@ python-versions = "*" [[package]] name = "boto3" -version = "1.17.32" +version = "1.17.53" description = "The AWS SDK for Python" category = "main" optional = false python-versions = ">= 2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" [package.dependencies] -botocore = ">=1.20.32,<1.21.0" +botocore = ">=1.20.53,<1.21.0" jmespath = ">=0.7.1,<1.0.0" s3transfer = ">=0.3.0,<0.4.0" [[package]] name = "boto3-stubs" -version = "1.17.31.0" -description = "Type annotations for boto3 1.17.31, generated by mypy-boto3-buider 4.4.0" +version = "1.17.64.2" +description = "Type annotations for boto3 1.17.64, generated by mypy-boto3-buider 4.10.0" category = "dev" optional = false python-versions = ">=3.6" @@ -132,270 +132,277 @@ python-versions = ">=3.6" typing-extensions = {version = "*", markers = "python_version < \"3.8\""} [package.extras] -accessanalyzer = ["mypy-boto3-accessanalyzer (==1.17.31.0)"] -acm = ["mypy-boto3-acm (==1.17.31.0)"] -acm-pca = ["mypy-boto3-acm-pca (==1.17.31.0)"] -alexaforbusiness = ["mypy-boto3-alexaforbusiness (==1.17.31.0)"] -all = ["mypy-boto3-accessanalyzer (==1.17.31.0)", "mypy-boto3-acm (==1.17.31.0)", "mypy-boto3-acm-pca (==1.17.31.0)", "mypy-boto3-alexaforbusiness (==1.17.31.0)", "mypy-boto3-amp (==1.17.31.0)", "mypy-boto3-amplify (==1.17.31.0)", "mypy-boto3-amplifybackend (==1.17.31.0)", "mypy-boto3-apigateway (==1.17.31.0)", "mypy-boto3-apigatewaymanagementapi (==1.17.31.0)", "mypy-boto3-apigatewayv2 (==1.17.31.0)", "mypy-boto3-appconfig (==1.17.31.0)", "mypy-boto3-appflow (==1.17.31.0)", "mypy-boto3-appintegrations (==1.17.31.0)", "mypy-boto3-application-autoscaling (==1.17.31.0)", "mypy-boto3-application-insights (==1.17.31.0)", "mypy-boto3-appmesh (==1.17.31.0)", "mypy-boto3-appstream (==1.17.31.0)", "mypy-boto3-appsync (==1.17.31.0)", "mypy-boto3-athena (==1.17.31.0)", "mypy-boto3-auditmanager (==1.17.31.0)", "mypy-boto3-autoscaling (==1.17.31.0)", "mypy-boto3-autoscaling-plans (==1.17.31.0)", "mypy-boto3-backup (==1.17.31.0)", "mypy-boto3-batch (==1.17.31.0)", "mypy-boto3-braket (==1.17.31.0)", "mypy-boto3-budgets (==1.17.31.0)", "mypy-boto3-ce (==1.17.31.0)", "mypy-boto3-chime (==1.17.31.0)", "mypy-boto3-cloud9 (==1.17.31.0)", "mypy-boto3-clouddirectory (==1.17.31.0)", "mypy-boto3-cloudformation (==1.17.31.0)", "mypy-boto3-cloudfront (==1.17.31.0)", "mypy-boto3-cloudhsm (==1.17.31.0)", "mypy-boto3-cloudhsmv2 (==1.17.31.0)", "mypy-boto3-cloudsearch (==1.17.31.0)", "mypy-boto3-cloudsearchdomain (==1.17.31.0)", "mypy-boto3-cloudtrail (==1.17.31.0)", "mypy-boto3-cloudwatch (==1.17.31.0)", "mypy-boto3-codeartifact (==1.17.31.0)", "mypy-boto3-codebuild (==1.17.31.0)", "mypy-boto3-codecommit (==1.17.31.0)", "mypy-boto3-codedeploy (==1.17.31.0)", "mypy-boto3-codeguru-reviewer (==1.17.31.0)", "mypy-boto3-codeguruprofiler (==1.17.31.0)", "mypy-boto3-codepipeline (==1.17.31.0)", "mypy-boto3-codestar (==1.17.31.0)", "mypy-boto3-codestar-connections (==1.17.31.0)", "mypy-boto3-codestar-notifications (==1.17.31.0)", "mypy-boto3-cognito-identity (==1.17.31.0)", "mypy-boto3-cognito-idp (==1.17.31.0)", "mypy-boto3-cognito-sync (==1.17.31.0)", "mypy-boto3-comprehend (==1.17.31.0)", "mypy-boto3-comprehendmedical (==1.17.31.0)", "mypy-boto3-compute-optimizer (==1.17.31.0)", "mypy-boto3-config (==1.17.31.0)", "mypy-boto3-connect (==1.17.31.0)", "mypy-boto3-connect-contact-lens (==1.17.31.0)", "mypy-boto3-connectparticipant (==1.17.31.0)", "mypy-boto3-cur (==1.17.31.0)", "mypy-boto3-customer-profiles (==1.17.31.0)", "mypy-boto3-databrew (==1.17.31.0)", "mypy-boto3-dataexchange (==1.17.31.0)", "mypy-boto3-datapipeline (==1.17.31.0)", "mypy-boto3-datasync (==1.17.31.0)", "mypy-boto3-dax (==1.17.31.0)", "mypy-boto3-detective (==1.17.31.0)", "mypy-boto3-devicefarm (==1.17.31.0)", "mypy-boto3-devops-guru (==1.17.31.0)", "mypy-boto3-directconnect (==1.17.31.0)", "mypy-boto3-discovery (==1.17.31.0)", "mypy-boto3-dlm (==1.17.31.0)", "mypy-boto3-dms (==1.17.31.0)", "mypy-boto3-docdb (==1.17.31.0)", "mypy-boto3-ds (==1.17.31.0)", "mypy-boto3-dynamodb (==1.17.31.0)", "mypy-boto3-dynamodbstreams (==1.17.31.0)", "mypy-boto3-ebs (==1.17.31.0)", "mypy-boto3-ec2 (==1.17.31.0)", "mypy-boto3-ec2-instance-connect (==1.17.31.0)", "mypy-boto3-ecr (==1.17.31.0)", "mypy-boto3-ecr-public (==1.17.31.0)", "mypy-boto3-ecs (==1.17.31.0)", "mypy-boto3-efs (==1.17.31.0)", "mypy-boto3-eks (==1.17.31.0)", "mypy-boto3-elastic-inference (==1.17.31.0)", "mypy-boto3-elasticache (==1.17.31.0)", "mypy-boto3-elasticbeanstalk (==1.17.31.0)", "mypy-boto3-elastictranscoder (==1.17.31.0)", "mypy-boto3-elb (==1.17.31.0)", "mypy-boto3-elbv2 (==1.17.31.0)", "mypy-boto3-emr (==1.17.31.0)", "mypy-boto3-emr-containers (==1.17.31.0)", "mypy-boto3-es (==1.17.31.0)", "mypy-boto3-events (==1.17.31.0)", "mypy-boto3-firehose (==1.17.31.0)", "mypy-boto3-fms (==1.17.31.0)", "mypy-boto3-forecast (==1.17.31.0)", "mypy-boto3-forecastquery (==1.17.31.0)", "mypy-boto3-frauddetector (==1.17.31.0)", "mypy-boto3-fsx (==1.17.31.0)", "mypy-boto3-gamelift (==1.17.31.0)", "mypy-boto3-glacier (==1.17.31.0)", "mypy-boto3-globalaccelerator (==1.17.31.0)", "mypy-boto3-glue (==1.17.31.0)", "mypy-boto3-greengrass (==1.17.31.0)", "mypy-boto3-greengrassv2 (==1.17.31.0)", "mypy-boto3-groundstation (==1.17.31.0)", "mypy-boto3-guardduty (==1.17.31.0)", "mypy-boto3-health (==1.17.31.0)", "mypy-boto3-healthlake (==1.17.31.0)", "mypy-boto3-honeycode (==1.17.31.0)", "mypy-boto3-iam (==1.17.31.0)", "mypy-boto3-identitystore (==1.17.31.0)", "mypy-boto3-imagebuilder (==1.17.31.0)", "mypy-boto3-importexport (==1.17.31.0)", "mypy-boto3-inspector (==1.17.31.0)", "mypy-boto3-iot (==1.17.31.0)", "mypy-boto3-iot-data (==1.17.31.0)", "mypy-boto3-iot-jobs-data (==1.17.31.0)", "mypy-boto3-iot1click-devices (==1.17.31.0)", "mypy-boto3-iot1click-projects (==1.17.31.0)", "mypy-boto3-iotanalytics (==1.17.31.0)", "mypy-boto3-iotdeviceadvisor (==1.17.31.0)", "mypy-boto3-iotevents (==1.17.31.0)", "mypy-boto3-iotevents-data (==1.17.31.0)", "mypy-boto3-iotfleethub (==1.17.31.0)", "mypy-boto3-iotsecuretunneling (==1.17.31.0)", "mypy-boto3-iotsitewise (==1.17.31.0)", "mypy-boto3-iotthingsgraph (==1.17.31.0)", "mypy-boto3-iotwireless (==1.17.31.0)", "mypy-boto3-ivs (==1.17.31.0)", "mypy-boto3-kafka (==1.17.31.0)", "mypy-boto3-kendra (==1.17.31.0)", "mypy-boto3-kinesis (==1.17.31.0)", "mypy-boto3-kinesis-video-archived-media (==1.17.31.0)", "mypy-boto3-kinesis-video-media (==1.17.31.0)", "mypy-boto3-kinesis-video-signaling (==1.17.31.0)", "mypy-boto3-kinesisanalytics (==1.17.31.0)", "mypy-boto3-kinesisanalyticsv2 (==1.17.31.0)", "mypy-boto3-kinesisvideo (==1.17.31.0)", "mypy-boto3-kms (==1.17.31.0)", "mypy-boto3-lakeformation (==1.17.31.0)", "mypy-boto3-lambda (==1.17.31.0)", "mypy-boto3-lex-models (==1.17.31.0)", "mypy-boto3-lex-runtime (==1.17.31.0)", "mypy-boto3-lexv2-models (==1.17.31.0)", "mypy-boto3-lexv2-runtime (==1.17.31.0)", "mypy-boto3-license-manager (==1.17.31.0)", "mypy-boto3-lightsail (==1.17.31.0)", "mypy-boto3-location (==1.17.31.0)", "mypy-boto3-logs (==1.17.31.0)", "mypy-boto3-lookoutvision (==1.17.31.0)", "mypy-boto3-machinelearning (==1.17.31.0)", "mypy-boto3-macie (==1.17.31.0)", "mypy-boto3-macie2 (==1.17.31.0)", "mypy-boto3-managedblockchain (==1.17.31.0)", "mypy-boto3-marketplace-catalog (==1.17.31.0)", "mypy-boto3-marketplace-entitlement (==1.17.31.0)", "mypy-boto3-marketplacecommerceanalytics (==1.17.31.0)", "mypy-boto3-mediaconnect (==1.17.31.0)", "mypy-boto3-mediaconvert (==1.17.31.0)", "mypy-boto3-medialive (==1.17.31.0)", "mypy-boto3-mediapackage (==1.17.31.0)", "mypy-boto3-mediapackage-vod (==1.17.31.0)", "mypy-boto3-mediastore (==1.17.31.0)", "mypy-boto3-mediastore-data (==1.17.31.0)", "mypy-boto3-mediatailor (==1.17.31.0)", "mypy-boto3-meteringmarketplace (==1.17.31.0)", "mypy-boto3-mgh (==1.17.31.0)", "mypy-boto3-migrationhub-config (==1.17.31.0)", "mypy-boto3-mobile (==1.17.31.0)", "mypy-boto3-mq (==1.17.31.0)", "mypy-boto3-mturk (==1.17.31.0)", "mypy-boto3-mwaa (==1.17.31.0)", "mypy-boto3-neptune (==1.17.31.0)", "mypy-boto3-network-firewall (==1.17.31.0)", "mypy-boto3-networkmanager (==1.17.31.0)", "mypy-boto3-opsworks (==1.17.31.0)", "mypy-boto3-opsworkscm (==1.17.31.0)", "mypy-boto3-organizations (==1.17.31.0)", "mypy-boto3-outposts (==1.17.31.0)", "mypy-boto3-personalize (==1.17.31.0)", "mypy-boto3-personalize-events (==1.17.31.0)", "mypy-boto3-personalize-runtime (==1.17.31.0)", "mypy-boto3-pi (==1.17.31.0)", "mypy-boto3-pinpoint (==1.17.31.0)", "mypy-boto3-pinpoint-email (==1.17.31.0)", "mypy-boto3-pinpoint-sms-voice (==1.17.31.0)", "mypy-boto3-polly (==1.17.31.0)", "mypy-boto3-pricing (==1.17.31.0)", "mypy-boto3-qldb (==1.17.31.0)", "mypy-boto3-qldb-session (==1.17.31.0)", "mypy-boto3-quicksight (==1.17.31.0)", "mypy-boto3-ram (==1.17.31.0)", "mypy-boto3-rds (==1.17.31.0)", "mypy-boto3-rds-data (==1.17.31.0)", "mypy-boto3-redshift (==1.17.31.0)", "mypy-boto3-redshift-data (==1.17.31.0)", "mypy-boto3-rekognition (==1.17.31.0)", "mypy-boto3-resource-groups (==1.17.31.0)", "mypy-boto3-resourcegroupstaggingapi (==1.17.31.0)", "mypy-boto3-robomaker (==1.17.31.0)", "mypy-boto3-route53 (==1.17.31.0)", "mypy-boto3-route53domains (==1.17.31.0)", "mypy-boto3-route53resolver (==1.17.31.0)", "mypy-boto3-s3 (==1.17.31.0)", "mypy-boto3-s3control (==1.17.31.0)", "mypy-boto3-s3outposts (==1.17.31.0)", "mypy-boto3-sagemaker (==1.17.31.0)", "mypy-boto3-sagemaker-a2i-runtime (==1.17.31.0)", "mypy-boto3-sagemaker-edge (==1.17.31.0)", "mypy-boto3-sagemaker-featurestore-runtime (==1.17.31.0)", "mypy-boto3-sagemaker-runtime (==1.17.31.0)", "mypy-boto3-savingsplans (==1.17.31.0)", "mypy-boto3-schemas (==1.17.31.0)", "mypy-boto3-sdb (==1.17.31.0)", "mypy-boto3-secretsmanager (==1.17.31.0)", "mypy-boto3-securityhub (==1.17.31.0)", "mypy-boto3-serverlessrepo (==1.17.31.0)", "mypy-boto3-service-quotas (==1.17.31.0)", "mypy-boto3-servicecatalog (==1.17.31.0)", "mypy-boto3-servicecatalog-appregistry (==1.17.31.0)", "mypy-boto3-servicediscovery (==1.17.31.0)", "mypy-boto3-ses (==1.17.31.0)", "mypy-boto3-sesv2 (==1.17.31.0)", "mypy-boto3-shield (==1.17.31.0)", "mypy-boto3-signer (==1.17.31.0)", "mypy-boto3-sms (==1.17.31.0)", "mypy-boto3-sms-voice (==1.17.31.0)", "mypy-boto3-snowball (==1.17.31.0)", "mypy-boto3-sns (==1.17.31.0)", "mypy-boto3-sqs (==1.17.31.0)", "mypy-boto3-ssm (==1.17.31.0)", "mypy-boto3-sso (==1.17.31.0)", "mypy-boto3-sso-admin (==1.17.31.0)", "mypy-boto3-sso-oidc (==1.17.31.0)", "mypy-boto3-stepfunctions (==1.17.31.0)", "mypy-boto3-storagegateway (==1.17.31.0)", "mypy-boto3-sts (==1.17.31.0)", "mypy-boto3-support (==1.17.31.0)", "mypy-boto3-swf (==1.17.31.0)", "mypy-boto3-synthetics (==1.17.31.0)", "mypy-boto3-textract (==1.17.31.0)", "mypy-boto3-timestream-query (==1.17.31.0)", "mypy-boto3-timestream-write (==1.17.31.0)", "mypy-boto3-transcribe (==1.17.31.0)", "mypy-boto3-transfer (==1.17.31.0)", "mypy-boto3-translate (==1.17.31.0)", "mypy-boto3-waf (==1.17.31.0)", "mypy-boto3-waf-regional (==1.17.31.0)", "mypy-boto3-wafv2 (==1.17.31.0)", "mypy-boto3-wellarchitected (==1.17.31.0)", "mypy-boto3-workdocs (==1.17.31.0)", "mypy-boto3-worklink (==1.17.31.0)", "mypy-boto3-workmail (==1.17.31.0)", "mypy-boto3-workmailmessageflow (==1.17.31.0)", "mypy-boto3-workspaces (==1.17.31.0)", "mypy-boto3-xray (==1.17.31.0)"] -amp = ["mypy-boto3-amp (==1.17.31.0)"] -amplify = ["mypy-boto3-amplify (==1.17.31.0)"] -amplifybackend = ["mypy-boto3-amplifybackend (==1.17.31.0)"] -apigateway = ["mypy-boto3-apigateway (==1.17.31.0)"] -apigatewaymanagementapi = ["mypy-boto3-apigatewaymanagementapi (==1.17.31.0)"] -apigatewayv2 = ["mypy-boto3-apigatewayv2 (==1.17.31.0)"] -appconfig = ["mypy-boto3-appconfig (==1.17.31.0)"] -appflow = ["mypy-boto3-appflow (==1.17.31.0)"] -appintegrations = ["mypy-boto3-appintegrations (==1.17.31.0)"] -application-autoscaling = ["mypy-boto3-application-autoscaling (==1.17.31.0)"] -application-insights = ["mypy-boto3-application-insights (==1.17.31.0)"] -appmesh = ["mypy-boto3-appmesh (==1.17.31.0)"] -appstream = ["mypy-boto3-appstream (==1.17.31.0)"] -appsync = ["mypy-boto3-appsync (==1.17.31.0)"] -athena = ["mypy-boto3-athena (==1.17.31.0)"] -auditmanager = ["mypy-boto3-auditmanager (==1.17.31.0)"] -autoscaling = ["mypy-boto3-autoscaling (==1.17.31.0)"] -autoscaling-plans = ["mypy-boto3-autoscaling-plans (==1.17.31.0)"] -backup = ["mypy-boto3-backup (==1.17.31.0)"] -batch = ["mypy-boto3-batch (==1.17.31.0)"] -braket = ["mypy-boto3-braket (==1.17.31.0)"] -budgets = ["mypy-boto3-budgets (==1.17.31.0)"] -ce = ["mypy-boto3-ce (==1.17.31.0)"] -chime = ["mypy-boto3-chime (==1.17.31.0)"] -cloud9 = ["mypy-boto3-cloud9 (==1.17.31.0)"] -clouddirectory = ["mypy-boto3-clouddirectory (==1.17.31.0)"] -cloudformation = ["mypy-boto3-cloudformation (==1.17.31.0)"] -cloudfront = ["mypy-boto3-cloudfront (==1.17.31.0)"] -cloudhsm = ["mypy-boto3-cloudhsm (==1.17.31.0)"] -cloudhsmv2 = ["mypy-boto3-cloudhsmv2 (==1.17.31.0)"] -cloudsearch = ["mypy-boto3-cloudsearch (==1.17.31.0)"] -cloudsearchdomain = ["mypy-boto3-cloudsearchdomain (==1.17.31.0)"] -cloudtrail = ["mypy-boto3-cloudtrail (==1.17.31.0)"] -cloudwatch = ["mypy-boto3-cloudwatch (==1.17.31.0)"] -codeartifact = ["mypy-boto3-codeartifact (==1.17.31.0)"] -codebuild = ["mypy-boto3-codebuild (==1.17.31.0)"] -codecommit = ["mypy-boto3-codecommit (==1.17.31.0)"] -codedeploy = ["mypy-boto3-codedeploy (==1.17.31.0)"] -codeguru-reviewer = ["mypy-boto3-codeguru-reviewer (==1.17.31.0)"] -codeguruprofiler = ["mypy-boto3-codeguruprofiler (==1.17.31.0)"] -codepipeline = ["mypy-boto3-codepipeline (==1.17.31.0)"] -codestar = ["mypy-boto3-codestar (==1.17.31.0)"] -codestar-connections = ["mypy-boto3-codestar-connections (==1.17.31.0)"] -codestar-notifications = ["mypy-boto3-codestar-notifications (==1.17.31.0)"] -cognito-identity = ["mypy-boto3-cognito-identity (==1.17.31.0)"] -cognito-idp = ["mypy-boto3-cognito-idp (==1.17.31.0)"] -cognito-sync = ["mypy-boto3-cognito-sync (==1.17.31.0)"] -comprehend = ["mypy-boto3-comprehend (==1.17.31.0)"] -comprehendmedical = ["mypy-boto3-comprehendmedical (==1.17.31.0)"] -compute-optimizer = ["mypy-boto3-compute-optimizer (==1.17.31.0)"] -config = ["mypy-boto3-config (==1.17.31.0)"] -connect = ["mypy-boto3-connect (==1.17.31.0)"] -connect-contact-lens = ["mypy-boto3-connect-contact-lens (==1.17.31.0)"] -connectparticipant = ["mypy-boto3-connectparticipant (==1.17.31.0)"] -cur = ["mypy-boto3-cur (==1.17.31.0)"] -customer-profiles = ["mypy-boto3-customer-profiles (==1.17.31.0)"] -databrew = ["mypy-boto3-databrew (==1.17.31.0)"] -dataexchange = ["mypy-boto3-dataexchange (==1.17.31.0)"] -datapipeline = ["mypy-boto3-datapipeline (==1.17.31.0)"] -datasync = ["mypy-boto3-datasync (==1.17.31.0)"] -dax = ["mypy-boto3-dax (==1.17.31.0)"] -detective = ["mypy-boto3-detective (==1.17.31.0)"] -devicefarm = ["mypy-boto3-devicefarm (==1.17.31.0)"] -devops-guru = ["mypy-boto3-devops-guru (==1.17.31.0)"] -directconnect = ["mypy-boto3-directconnect (==1.17.31.0)"] -discovery = ["mypy-boto3-discovery (==1.17.31.0)"] -dlm = ["mypy-boto3-dlm (==1.17.31.0)"] -dms = ["mypy-boto3-dms (==1.17.31.0)"] -docdb = ["mypy-boto3-docdb (==1.17.31.0)"] -ds = ["mypy-boto3-ds (==1.17.31.0)"] -dynamodb = ["mypy-boto3-dynamodb (==1.17.31.0)"] -dynamodbstreams = ["mypy-boto3-dynamodbstreams (==1.17.31.0)"] -ebs = ["mypy-boto3-ebs (==1.17.31.0)"] -ec2 = ["mypy-boto3-ec2 (==1.17.31.0)"] -ec2-instance-connect = ["mypy-boto3-ec2-instance-connect (==1.17.31.0)"] -ecr = ["mypy-boto3-ecr (==1.17.31.0)"] -ecr-public = ["mypy-boto3-ecr-public (==1.17.31.0)"] -ecs = ["mypy-boto3-ecs (==1.17.31.0)"] -efs = ["mypy-boto3-efs (==1.17.31.0)"] -eks = ["mypy-boto3-eks (==1.17.31.0)"] -elastic-inference = ["mypy-boto3-elastic-inference (==1.17.31.0)"] -elasticache = ["mypy-boto3-elasticache (==1.17.31.0)"] -elasticbeanstalk = ["mypy-boto3-elasticbeanstalk (==1.17.31.0)"] -elastictranscoder = ["mypy-boto3-elastictranscoder (==1.17.31.0)"] -elb = ["mypy-boto3-elb (==1.17.31.0)"] -elbv2 = ["mypy-boto3-elbv2 (==1.17.31.0)"] -emr = ["mypy-boto3-emr (==1.17.31.0)"] -emr-containers = ["mypy-boto3-emr-containers (==1.17.31.0)"] -es = ["mypy-boto3-es (==1.17.31.0)"] -essential = ["mypy-boto3-cloudformation (==1.17.31.0)", "mypy-boto3-dynamodb (==1.17.31.0)", "mypy-boto3-ec2 (==1.17.31.0)", "mypy-boto3-lambda (==1.17.31.0)", "mypy-boto3-rds (==1.17.31.0)", "mypy-boto3-s3 (==1.17.31.0)", "mypy-boto3-sqs (==1.17.31.0)"] -events = ["mypy-boto3-events (==1.17.31.0)"] -firehose = ["mypy-boto3-firehose (==1.17.31.0)"] -fms = ["mypy-boto3-fms (==1.17.31.0)"] -forecast = ["mypy-boto3-forecast (==1.17.31.0)"] -forecastquery = ["mypy-boto3-forecastquery (==1.17.31.0)"] -frauddetector = ["mypy-boto3-frauddetector (==1.17.31.0)"] -fsx = ["mypy-boto3-fsx (==1.17.31.0)"] -gamelift = ["mypy-boto3-gamelift (==1.17.31.0)"] -glacier = ["mypy-boto3-glacier (==1.17.31.0)"] -globalaccelerator = ["mypy-boto3-globalaccelerator (==1.17.31.0)"] -glue = ["mypy-boto3-glue (==1.17.31.0)"] -greengrass = ["mypy-boto3-greengrass (==1.17.31.0)"] -greengrassv2 = ["mypy-boto3-greengrassv2 (==1.17.31.0)"] -groundstation = ["mypy-boto3-groundstation (==1.17.31.0)"] -guardduty = ["mypy-boto3-guardduty (==1.17.31.0)"] -health = ["mypy-boto3-health (==1.17.31.0)"] -healthlake = ["mypy-boto3-healthlake (==1.17.31.0)"] -honeycode = ["mypy-boto3-honeycode (==1.17.31.0)"] -iam = ["mypy-boto3-iam (==1.17.31.0)"] -identitystore = ["mypy-boto3-identitystore (==1.17.31.0)"] -imagebuilder = ["mypy-boto3-imagebuilder (==1.17.31.0)"] -importexport = ["mypy-boto3-importexport (==1.17.31.0)"] -inspector = ["mypy-boto3-inspector (==1.17.31.0)"] -iot = ["mypy-boto3-iot (==1.17.31.0)"] -iot-data = ["mypy-boto3-iot-data (==1.17.31.0)"] -iot-jobs-data = ["mypy-boto3-iot-jobs-data (==1.17.31.0)"] -iot1click-devices = ["mypy-boto3-iot1click-devices (==1.17.31.0)"] -iot1click-projects = ["mypy-boto3-iot1click-projects (==1.17.31.0)"] -iotanalytics = ["mypy-boto3-iotanalytics (==1.17.31.0)"] -iotdeviceadvisor = ["mypy-boto3-iotdeviceadvisor (==1.17.31.0)"] -iotevents = ["mypy-boto3-iotevents (==1.17.31.0)"] -iotevents-data = ["mypy-boto3-iotevents-data (==1.17.31.0)"] -iotfleethub = ["mypy-boto3-iotfleethub (==1.17.31.0)"] -iotsecuretunneling = ["mypy-boto3-iotsecuretunneling (==1.17.31.0)"] -iotsitewise = ["mypy-boto3-iotsitewise (==1.17.31.0)"] -iotthingsgraph = ["mypy-boto3-iotthingsgraph (==1.17.31.0)"] -iotwireless = ["mypy-boto3-iotwireless (==1.17.31.0)"] -ivs = ["mypy-boto3-ivs (==1.17.31.0)"] -kafka = ["mypy-boto3-kafka (==1.17.31.0)"] -kendra = ["mypy-boto3-kendra (==1.17.31.0)"] -kinesis = ["mypy-boto3-kinesis (==1.17.31.0)"] -kinesis-video-archived-media = ["mypy-boto3-kinesis-video-archived-media (==1.17.31.0)"] -kinesis-video-media = ["mypy-boto3-kinesis-video-media (==1.17.31.0)"] -kinesis-video-signaling = ["mypy-boto3-kinesis-video-signaling (==1.17.31.0)"] -kinesisanalytics = ["mypy-boto3-kinesisanalytics (==1.17.31.0)"] -kinesisanalyticsv2 = ["mypy-boto3-kinesisanalyticsv2 (==1.17.31.0)"] -kinesisvideo = ["mypy-boto3-kinesisvideo (==1.17.31.0)"] -kms = ["mypy-boto3-kms (==1.17.31.0)"] -lakeformation = ["mypy-boto3-lakeformation (==1.17.31.0)"] -lambda = ["mypy-boto3-lambda (==1.17.31.0)"] -lex-models = ["mypy-boto3-lex-models (==1.17.31.0)"] -lex-runtime = ["mypy-boto3-lex-runtime (==1.17.31.0)"] -lexv2-models = ["mypy-boto3-lexv2-models (==1.17.31.0)"] -lexv2-runtime = ["mypy-boto3-lexv2-runtime (==1.17.31.0)"] -license-manager = ["mypy-boto3-license-manager (==1.17.31.0)"] -lightsail = ["mypy-boto3-lightsail (==1.17.31.0)"] -location = ["mypy-boto3-location (==1.17.31.0)"] -logs = ["mypy-boto3-logs (==1.17.31.0)"] -lookoutvision = ["mypy-boto3-lookoutvision (==1.17.31.0)"] -machinelearning = ["mypy-boto3-machinelearning (==1.17.31.0)"] -macie = ["mypy-boto3-macie (==1.17.31.0)"] -macie2 = ["mypy-boto3-macie2 (==1.17.31.0)"] -managedblockchain = ["mypy-boto3-managedblockchain (==1.17.31.0)"] -marketplace-catalog = ["mypy-boto3-marketplace-catalog (==1.17.31.0)"] -marketplace-entitlement = ["mypy-boto3-marketplace-entitlement (==1.17.31.0)"] -marketplacecommerceanalytics = ["mypy-boto3-marketplacecommerceanalytics (==1.17.31.0)"] -mediaconnect = ["mypy-boto3-mediaconnect (==1.17.31.0)"] -mediaconvert = ["mypy-boto3-mediaconvert (==1.17.31.0)"] -medialive = ["mypy-boto3-medialive (==1.17.31.0)"] -mediapackage = ["mypy-boto3-mediapackage (==1.17.31.0)"] -mediapackage-vod = ["mypy-boto3-mediapackage-vod (==1.17.31.0)"] -mediastore = ["mypy-boto3-mediastore (==1.17.31.0)"] -mediastore-data = ["mypy-boto3-mediastore-data (==1.17.31.0)"] -mediatailor = ["mypy-boto3-mediatailor (==1.17.31.0)"] -meteringmarketplace = ["mypy-boto3-meteringmarketplace (==1.17.31.0)"] -mgh = ["mypy-boto3-mgh (==1.17.31.0)"] -migrationhub-config = ["mypy-boto3-migrationhub-config (==1.17.31.0)"] -mobile = ["mypy-boto3-mobile (==1.17.31.0)"] -mq = ["mypy-boto3-mq (==1.17.31.0)"] -mturk = ["mypy-boto3-mturk (==1.17.31.0)"] -mwaa = ["mypy-boto3-mwaa (==1.17.31.0)"] -neptune = ["mypy-boto3-neptune (==1.17.31.0)"] -network-firewall = ["mypy-boto3-network-firewall (==1.17.31.0)"] -networkmanager = ["mypy-boto3-networkmanager (==1.17.31.0)"] -opsworks = ["mypy-boto3-opsworks (==1.17.31.0)"] -opsworkscm = ["mypy-boto3-opsworkscm (==1.17.31.0)"] -organizations = ["mypy-boto3-organizations (==1.17.31.0)"] -outposts = ["mypy-boto3-outposts (==1.17.31.0)"] -personalize = ["mypy-boto3-personalize (==1.17.31.0)"] -personalize-events = ["mypy-boto3-personalize-events (==1.17.31.0)"] -personalize-runtime = ["mypy-boto3-personalize-runtime (==1.17.31.0)"] -pi = ["mypy-boto3-pi (==1.17.31.0)"] -pinpoint = ["mypy-boto3-pinpoint (==1.17.31.0)"] -pinpoint-email = ["mypy-boto3-pinpoint-email (==1.17.31.0)"] -pinpoint-sms-voice = ["mypy-boto3-pinpoint-sms-voice (==1.17.31.0)"] -polly = ["mypy-boto3-polly (==1.17.31.0)"] -pricing = ["mypy-boto3-pricing (==1.17.31.0)"] -qldb = ["mypy-boto3-qldb (==1.17.31.0)"] -qldb-session = ["mypy-boto3-qldb-session (==1.17.31.0)"] -quicksight = ["mypy-boto3-quicksight (==1.17.31.0)"] -ram = ["mypy-boto3-ram (==1.17.31.0)"] -rds = ["mypy-boto3-rds (==1.17.31.0)"] -rds-data = ["mypy-boto3-rds-data (==1.17.31.0)"] -redshift = ["mypy-boto3-redshift (==1.17.31.0)"] -redshift-data = ["mypy-boto3-redshift-data (==1.17.31.0)"] -rekognition = ["mypy-boto3-rekognition (==1.17.31.0)"] -resource-groups = ["mypy-boto3-resource-groups (==1.17.31.0)"] -resourcegroupstaggingapi = ["mypy-boto3-resourcegroupstaggingapi (==1.17.31.0)"] -robomaker = ["mypy-boto3-robomaker (==1.17.31.0)"] -route53 = ["mypy-boto3-route53 (==1.17.31.0)"] -route53domains = ["mypy-boto3-route53domains (==1.17.31.0)"] -route53resolver = ["mypy-boto3-route53resolver (==1.17.31.0)"] -s3 = ["mypy-boto3-s3 (==1.17.31.0)"] -s3control = ["mypy-boto3-s3control (==1.17.31.0)"] -s3outposts = ["mypy-boto3-s3outposts (==1.17.31.0)"] -sagemaker = ["mypy-boto3-sagemaker (==1.17.31.0)"] -sagemaker-a2i-runtime = ["mypy-boto3-sagemaker-a2i-runtime (==1.17.31.0)"] -sagemaker-edge = ["mypy-boto3-sagemaker-edge (==1.17.31.0)"] -sagemaker-featurestore-runtime = ["mypy-boto3-sagemaker-featurestore-runtime (==1.17.31.0)"] -sagemaker-runtime = ["mypy-boto3-sagemaker-runtime (==1.17.31.0)"] -savingsplans = ["mypy-boto3-savingsplans (==1.17.31.0)"] -schemas = ["mypy-boto3-schemas (==1.17.31.0)"] -sdb = ["mypy-boto3-sdb (==1.17.31.0)"] -secretsmanager = ["mypy-boto3-secretsmanager (==1.17.31.0)"] -securityhub = ["mypy-boto3-securityhub (==1.17.31.0)"] -serverlessrepo = ["mypy-boto3-serverlessrepo (==1.17.31.0)"] -service-quotas = ["mypy-boto3-service-quotas (==1.17.31.0)"] -servicecatalog = ["mypy-boto3-servicecatalog (==1.17.31.0)"] -servicecatalog-appregistry = ["mypy-boto3-servicecatalog-appregistry (==1.17.31.0)"] -servicediscovery = ["mypy-boto3-servicediscovery (==1.17.31.0)"] -ses = ["mypy-boto3-ses (==1.17.31.0)"] -sesv2 = ["mypy-boto3-sesv2 (==1.17.31.0)"] -shield = ["mypy-boto3-shield (==1.17.31.0)"] -signer = ["mypy-boto3-signer (==1.17.31.0)"] -sms = ["mypy-boto3-sms (==1.17.31.0)"] -sms-voice = ["mypy-boto3-sms-voice (==1.17.31.0)"] -snowball = ["mypy-boto3-snowball (==1.17.31.0)"] -sns = ["mypy-boto3-sns (==1.17.31.0)"] -sqs = ["mypy-boto3-sqs (==1.17.31.0)"] -ssm = ["mypy-boto3-ssm (==1.17.31.0)"] -sso = ["mypy-boto3-sso (==1.17.31.0)"] -sso-admin = ["mypy-boto3-sso-admin (==1.17.31.0)"] -sso-oidc = ["mypy-boto3-sso-oidc (==1.17.31.0)"] -stepfunctions = ["mypy-boto3-stepfunctions (==1.17.31.0)"] -storagegateway = ["mypy-boto3-storagegateway (==1.17.31.0)"] -sts = ["mypy-boto3-sts (==1.17.31.0)"] -support = ["mypy-boto3-support (==1.17.31.0)"] -swf = ["mypy-boto3-swf (==1.17.31.0)"] -synthetics = ["mypy-boto3-synthetics (==1.17.31.0)"] -textract = ["mypy-boto3-textract (==1.17.31.0)"] -timestream-query = ["mypy-boto3-timestream-query (==1.17.31.0)"] -timestream-write = ["mypy-boto3-timestream-write (==1.17.31.0)"] -transcribe = ["mypy-boto3-transcribe (==1.17.31.0)"] -transfer = ["mypy-boto3-transfer (==1.17.31.0)"] -translate = ["mypy-boto3-translate (==1.17.31.0)"] -waf = ["mypy-boto3-waf (==1.17.31.0)"] -waf-regional = ["mypy-boto3-waf-regional (==1.17.31.0)"] -wafv2 = ["mypy-boto3-wafv2 (==1.17.31.0)"] -wellarchitected = ["mypy-boto3-wellarchitected (==1.17.31.0)"] -workdocs = ["mypy-boto3-workdocs (==1.17.31.0)"] -worklink = ["mypy-boto3-worklink (==1.17.31.0)"] -workmail = ["mypy-boto3-workmail (==1.17.31.0)"] -workmailmessageflow = ["mypy-boto3-workmailmessageflow (==1.17.31.0)"] -workspaces = ["mypy-boto3-workspaces (==1.17.31.0)"] -xray = ["mypy-boto3-xray (==1.17.31.0)"] +accessanalyzer = ["mypy-boto3-accessanalyzer (==1.17.64.2)"] +acm = ["mypy-boto3-acm (==1.17.64.2)"] +acm-pca = ["mypy-boto3-acm-pca (==1.17.64.2)"] +alexaforbusiness = ["mypy-boto3-alexaforbusiness (==1.17.64.2)"] +all = ["mypy-boto3-accessanalyzer (==1.17.64.2)", "mypy-boto3-acm (==1.17.64.2)", "mypy-boto3-acm-pca (==1.17.64.2)", "mypy-boto3-alexaforbusiness (==1.17.64.2)", "mypy-boto3-amp (==1.17.64.2)", "mypy-boto3-amplify (==1.17.64.2)", "mypy-boto3-amplifybackend (==1.17.64.2)", "mypy-boto3-apigateway (==1.17.64.2)", "mypy-boto3-apigatewaymanagementapi (==1.17.64.2)", "mypy-boto3-apigatewayv2 (==1.17.64.2)", "mypy-boto3-appconfig (==1.17.64.2)", "mypy-boto3-appflow (==1.17.64.2)", "mypy-boto3-appintegrations (==1.17.64.2)", "mypy-boto3-application-autoscaling (==1.17.64.2)", "mypy-boto3-application-insights (==1.17.64.2)", "mypy-boto3-appmesh (==1.17.64.2)", "mypy-boto3-appstream (==1.17.64.2)", "mypy-boto3-appsync (==1.17.64.2)", "mypy-boto3-athena (==1.17.64.2)", "mypy-boto3-auditmanager (==1.17.64.2)", "mypy-boto3-autoscaling (==1.17.64.2)", "mypy-boto3-autoscaling-plans (==1.17.64.2)", "mypy-boto3-backup (==1.17.64.2)", "mypy-boto3-batch (==1.17.64.2)", "mypy-boto3-braket (==1.17.64.2)", "mypy-boto3-budgets (==1.17.64.2)", "mypy-boto3-ce (==1.17.64.2)", "mypy-boto3-chime (==1.17.64.2)", "mypy-boto3-cloud9 (==1.17.64.2)", "mypy-boto3-clouddirectory (==1.17.64.2)", "mypy-boto3-cloudformation (==1.17.64.2)", "mypy-boto3-cloudfront (==1.17.64.2)", "mypy-boto3-cloudhsm (==1.17.64.2)", "mypy-boto3-cloudhsmv2 (==1.17.64.2)", "mypy-boto3-cloudsearch (==1.17.64.2)", "mypy-boto3-cloudsearchdomain (==1.17.64.2)", "mypy-boto3-cloudtrail (==1.17.64.2)", "mypy-boto3-cloudwatch (==1.17.64.2)", "mypy-boto3-codeartifact (==1.17.64.2)", "mypy-boto3-codebuild (==1.17.64.2)", "mypy-boto3-codecommit (==1.17.64.2)", "mypy-boto3-codedeploy (==1.17.64.2)", "mypy-boto3-codeguru-reviewer (==1.17.64.2)", "mypy-boto3-codeguruprofiler (==1.17.64.2)", "mypy-boto3-codepipeline (==1.17.64.2)", "mypy-boto3-codestar (==1.17.64.2)", "mypy-boto3-codestar-connections (==1.17.64.2)", "mypy-boto3-codestar-notifications (==1.17.64.2)", "mypy-boto3-cognito-identity (==1.17.64.2)", "mypy-boto3-cognito-idp (==1.17.64.2)", "mypy-boto3-cognito-sync (==1.17.64.2)", "mypy-boto3-comprehend (==1.17.64.2)", "mypy-boto3-comprehendmedical (==1.17.64.2)", "mypy-boto3-compute-optimizer (==1.17.64.2)", "mypy-boto3-config (==1.17.64.2)", "mypy-boto3-connect (==1.17.64.2)", "mypy-boto3-connect-contact-lens (==1.17.64.2)", "mypy-boto3-connectparticipant (==1.17.64.2)", "mypy-boto3-cur (==1.17.64.2)", "mypy-boto3-customer-profiles (==1.17.64.2)", "mypy-boto3-databrew (==1.17.64.2)", "mypy-boto3-dataexchange (==1.17.64.2)", "mypy-boto3-datapipeline (==1.17.64.2)", "mypy-boto3-datasync (==1.17.64.2)", "mypy-boto3-dax (==1.17.64.2)", "mypy-boto3-detective (==1.17.64.2)", "mypy-boto3-devicefarm (==1.17.64.2)", "mypy-boto3-devops-guru (==1.17.64.2)", "mypy-boto3-directconnect (==1.17.64.2)", "mypy-boto3-discovery (==1.17.64.2)", "mypy-boto3-dlm (==1.17.64.2)", "mypy-boto3-dms (==1.17.64.2)", "mypy-boto3-docdb (==1.17.64.2)", "mypy-boto3-ds (==1.17.64.2)", "mypy-boto3-dynamodb (==1.17.64.2)", "mypy-boto3-dynamodbstreams (==1.17.64.2)", "mypy-boto3-ebs (==1.17.64.2)", "mypy-boto3-ec2 (==1.17.64.2)", "mypy-boto3-ec2-instance-connect (==1.17.64.2)", "mypy-boto3-ecr (==1.17.64.2)", "mypy-boto3-ecr-public (==1.17.64.2)", "mypy-boto3-ecs (==1.17.64.2)", "mypy-boto3-efs (==1.17.64.2)", "mypy-boto3-eks (==1.17.64.2)", "mypy-boto3-elastic-inference (==1.17.64.2)", "mypy-boto3-elasticache (==1.17.64.2)", "mypy-boto3-elasticbeanstalk (==1.17.64.2)", "mypy-boto3-elastictranscoder (==1.17.64.2)", "mypy-boto3-elb (==1.17.64.2)", "mypy-boto3-elbv2 (==1.17.64.2)", "mypy-boto3-emr (==1.17.64.2)", "mypy-boto3-emr-containers (==1.17.64.2)", "mypy-boto3-es (==1.17.64.2)", "mypy-boto3-events (==1.17.64.2)", "mypy-boto3-finspace (==1.17.64.2)", "mypy-boto3-finspace-data (==1.17.64.2)", "mypy-boto3-firehose (==1.17.64.2)", "mypy-boto3-fis (==1.17.64.2)", "mypy-boto3-fms (==1.17.64.2)", "mypy-boto3-forecast (==1.17.64.2)", "mypy-boto3-forecastquery (==1.17.64.2)", "mypy-boto3-frauddetector (==1.17.64.2)", "mypy-boto3-fsx (==1.17.64.2)", "mypy-boto3-gamelift (==1.17.64.2)", "mypy-boto3-glacier (==1.17.64.2)", "mypy-boto3-globalaccelerator (==1.17.64.2)", "mypy-boto3-glue (==1.17.64.2)", "mypy-boto3-greengrass (==1.17.64.2)", "mypy-boto3-greengrassv2 (==1.17.64.2)", "mypy-boto3-groundstation (==1.17.64.2)", "mypy-boto3-guardduty (==1.17.64.2)", "mypy-boto3-health (==1.17.64.2)", "mypy-boto3-healthlake (==1.17.64.2)", "mypy-boto3-honeycode (==1.17.64.2)", "mypy-boto3-iam (==1.17.64.2)", "mypy-boto3-identitystore (==1.17.64.2)", "mypy-boto3-imagebuilder (==1.17.64.2)", "mypy-boto3-importexport (==1.17.64.2)", "mypy-boto3-inspector (==1.17.64.2)", "mypy-boto3-iot (==1.17.64.2)", "mypy-boto3-iot-data (==1.17.64.2)", "mypy-boto3-iot-jobs-data (==1.17.64.2)", "mypy-boto3-iot1click-devices (==1.17.64.2)", "mypy-boto3-iot1click-projects (==1.17.64.2)", "mypy-boto3-iotanalytics (==1.17.64.2)", "mypy-boto3-iotdeviceadvisor (==1.17.64.2)", "mypy-boto3-iotevents (==1.17.64.2)", "mypy-boto3-iotevents-data (==1.17.64.2)", "mypy-boto3-iotfleethub (==1.17.64.2)", "mypy-boto3-iotsecuretunneling (==1.17.64.2)", "mypy-boto3-iotsitewise (==1.17.64.2)", "mypy-boto3-iotthingsgraph (==1.17.64.2)", "mypy-boto3-iotwireless (==1.17.64.2)", "mypy-boto3-ivs (==1.17.64.2)", "mypy-boto3-kafka (==1.17.64.2)", "mypy-boto3-kendra (==1.17.64.2)", "mypy-boto3-kinesis (==1.17.64.2)", "mypy-boto3-kinesis-video-archived-media (==1.17.64.2)", "mypy-boto3-kinesis-video-media (==1.17.64.2)", "mypy-boto3-kinesis-video-signaling (==1.17.64.2)", "mypy-boto3-kinesisanalytics (==1.17.64.2)", "mypy-boto3-kinesisanalyticsv2 (==1.17.64.2)", "mypy-boto3-kinesisvideo (==1.17.64.2)", "mypy-boto3-kms (==1.17.64.2)", "mypy-boto3-lakeformation (==1.17.64.2)", "mypy-boto3-lambda (==1.17.64.2)", "mypy-boto3-lex-models (==1.17.64.2)", "mypy-boto3-lex-runtime (==1.17.64.2)", "mypy-boto3-lexv2-models (==1.17.64.2)", "mypy-boto3-lexv2-runtime (==1.17.64.2)", "mypy-boto3-license-manager (==1.17.64.2)", "mypy-boto3-lightsail (==1.17.64.2)", "mypy-boto3-location (==1.17.64.2)", "mypy-boto3-logs (==1.17.64.2)", "mypy-boto3-lookoutequipment (==1.17.64.2)", "mypy-boto3-lookoutmetrics (==1.17.64.2)", "mypy-boto3-lookoutvision (==1.17.64.2)", "mypy-boto3-machinelearning (==1.17.64.2)", "mypy-boto3-macie (==1.17.64.2)", "mypy-boto3-macie2 (==1.17.64.2)", "mypy-boto3-managedblockchain (==1.17.64.2)", "mypy-boto3-marketplace-catalog (==1.17.64.2)", "mypy-boto3-marketplace-entitlement (==1.17.64.2)", "mypy-boto3-marketplacecommerceanalytics (==1.17.64.2)", "mypy-boto3-mediaconnect (==1.17.64.2)", "mypy-boto3-mediaconvert (==1.17.64.2)", "mypy-boto3-medialive (==1.17.64.2)", "mypy-boto3-mediapackage (==1.17.64.2)", "mypy-boto3-mediapackage-vod (==1.17.64.2)", "mypy-boto3-mediastore (==1.17.64.2)", "mypy-boto3-mediastore-data (==1.17.64.2)", "mypy-boto3-mediatailor (==1.17.64.2)", "mypy-boto3-meteringmarketplace (==1.17.64.2)", "mypy-boto3-mgh (==1.17.64.2)", "mypy-boto3-mgn (==1.17.64.2)", "mypy-boto3-migrationhub-config (==1.17.64.2)", "mypy-boto3-mobile (==1.17.64.2)", "mypy-boto3-mq (==1.17.64.2)", "mypy-boto3-mturk (==1.17.64.2)", "mypy-boto3-mwaa (==1.17.64.2)", "mypy-boto3-neptune (==1.17.64.2)", "mypy-boto3-network-firewall (==1.17.64.2)", "mypy-boto3-networkmanager (==1.17.64.2)", "mypy-boto3-nimble (==1.17.64.2)", "mypy-boto3-opsworks (==1.17.64.2)", "mypy-boto3-opsworkscm (==1.17.64.2)", "mypy-boto3-organizations (==1.17.64.2)", "mypy-boto3-outposts (==1.17.64.2)", "mypy-boto3-personalize (==1.17.64.2)", "mypy-boto3-personalize-events (==1.17.64.2)", "mypy-boto3-personalize-runtime (==1.17.64.2)", "mypy-boto3-pi (==1.17.64.2)", "mypy-boto3-pinpoint (==1.17.64.2)", "mypy-boto3-pinpoint-email (==1.17.64.2)", "mypy-boto3-pinpoint-sms-voice (==1.17.64.2)", "mypy-boto3-polly (==1.17.64.2)", "mypy-boto3-pricing (==1.17.64.2)", "mypy-boto3-qldb (==1.17.64.2)", "mypy-boto3-qldb-session (==1.17.64.2)", "mypy-boto3-quicksight (==1.17.64.2)", "mypy-boto3-ram (==1.17.64.2)", "mypy-boto3-rds (==1.17.64.2)", "mypy-boto3-rds-data (==1.17.64.2)", "mypy-boto3-redshift (==1.17.64.2)", "mypy-boto3-redshift-data (==1.17.64.2)", "mypy-boto3-rekognition (==1.17.64.2)", "mypy-boto3-resource-groups (==1.17.64.2)", "mypy-boto3-resourcegroupstaggingapi (==1.17.64.2)", "mypy-boto3-robomaker (==1.17.64.2)", "mypy-boto3-route53 (==1.17.64.2)", "mypy-boto3-route53domains (==1.17.64.2)", "mypy-boto3-route53resolver (==1.17.64.2)", "mypy-boto3-s3 (==1.17.64.2)", "mypy-boto3-s3control (==1.17.64.2)", "mypy-boto3-s3outposts (==1.17.64.2)", "mypy-boto3-sagemaker (==1.17.64.2)", "mypy-boto3-sagemaker-a2i-runtime (==1.17.64.2)", "mypy-boto3-sagemaker-edge (==1.17.64.2)", "mypy-boto3-sagemaker-featurestore-runtime (==1.17.64.2)", "mypy-boto3-sagemaker-runtime (==1.17.64.2)", "mypy-boto3-savingsplans (==1.17.64.2)", "mypy-boto3-schemas (==1.17.64.2)", "mypy-boto3-sdb (==1.17.64.2)", "mypy-boto3-secretsmanager (==1.17.64.2)", "mypy-boto3-securityhub (==1.17.64.2)", "mypy-boto3-serverlessrepo (==1.17.64.2)", "mypy-boto3-service-quotas (==1.17.64.2)", "mypy-boto3-servicecatalog (==1.17.64.2)", "mypy-boto3-servicecatalog-appregistry (==1.17.64.2)", "mypy-boto3-servicediscovery (==1.17.64.2)", "mypy-boto3-ses (==1.17.64.2)", "mypy-boto3-sesv2 (==1.17.64.2)", "mypy-boto3-shield (==1.17.64.2)", "mypy-boto3-signer (==1.17.64.2)", "mypy-boto3-sms (==1.17.64.2)", "mypy-boto3-sms-voice (==1.17.64.2)", "mypy-boto3-snowball (==1.17.64.2)", "mypy-boto3-sns (==1.17.64.2)", "mypy-boto3-sqs (==1.17.64.2)", "mypy-boto3-ssm (==1.17.64.2)", "mypy-boto3-sso (==1.17.64.2)", "mypy-boto3-sso-admin (==1.17.64.2)", "mypy-boto3-sso-oidc (==1.17.64.2)", "mypy-boto3-stepfunctions (==1.17.64.2)", "mypy-boto3-storagegateway (==1.17.64.2)", "mypy-boto3-sts (==1.17.64.2)", "mypy-boto3-support (==1.17.64.2)", "mypy-boto3-swf (==1.17.64.2)", "mypy-boto3-synthetics (==1.17.64.2)", "mypy-boto3-textract (==1.17.64.2)", "mypy-boto3-timestream-query (==1.17.64.2)", "mypy-boto3-timestream-write (==1.17.64.2)", "mypy-boto3-transcribe (==1.17.64.2)", "mypy-boto3-transfer (==1.17.64.2)", "mypy-boto3-translate (==1.17.64.2)", "mypy-boto3-waf (==1.17.64.2)", "mypy-boto3-waf-regional (==1.17.64.2)", "mypy-boto3-wafv2 (==1.17.64.2)", "mypy-boto3-wellarchitected (==1.17.64.2)", "mypy-boto3-workdocs (==1.17.64.2)", "mypy-boto3-worklink (==1.17.64.2)", "mypy-boto3-workmail (==1.17.64.2)", "mypy-boto3-workmailmessageflow (==1.17.64.2)", "mypy-boto3-workspaces (==1.17.64.2)", "mypy-boto3-xray (==1.17.64.2)"] +amp = ["mypy-boto3-amp (==1.17.64.2)"] +amplify = ["mypy-boto3-amplify (==1.17.64.2)"] +amplifybackend = ["mypy-boto3-amplifybackend (==1.17.64.2)"] +apigateway = ["mypy-boto3-apigateway (==1.17.64.2)"] +apigatewaymanagementapi = ["mypy-boto3-apigatewaymanagementapi (==1.17.64.2)"] +apigatewayv2 = ["mypy-boto3-apigatewayv2 (==1.17.64.2)"] +appconfig = ["mypy-boto3-appconfig (==1.17.64.2)"] +appflow = ["mypy-boto3-appflow (==1.17.64.2)"] +appintegrations = ["mypy-boto3-appintegrations (==1.17.64.2)"] +application-autoscaling = ["mypy-boto3-application-autoscaling (==1.17.64.2)"] +application-insights = ["mypy-boto3-application-insights (==1.17.64.2)"] +appmesh = ["mypy-boto3-appmesh (==1.17.64.2)"] +appstream = ["mypy-boto3-appstream (==1.17.64.2)"] +appsync = ["mypy-boto3-appsync (==1.17.64.2)"] +athena = ["mypy-boto3-athena (==1.17.64.2)"] +auditmanager = ["mypy-boto3-auditmanager (==1.17.64.2)"] +autoscaling = ["mypy-boto3-autoscaling (==1.17.64.2)"] +autoscaling-plans = ["mypy-boto3-autoscaling-plans (==1.17.64.2)"] +backup = ["mypy-boto3-backup (==1.17.64.2)"] +batch = ["mypy-boto3-batch (==1.17.64.2)"] +braket = ["mypy-boto3-braket (==1.17.64.2)"] +budgets = ["mypy-boto3-budgets (==1.17.64.2)"] +ce = ["mypy-boto3-ce (==1.17.64.2)"] +chime = ["mypy-boto3-chime (==1.17.64.2)"] +cloud9 = ["mypy-boto3-cloud9 (==1.17.64.2)"] +clouddirectory = ["mypy-boto3-clouddirectory (==1.17.64.2)"] +cloudformation = ["mypy-boto3-cloudformation (==1.17.64.2)"] +cloudfront = ["mypy-boto3-cloudfront (==1.17.64.2)"] +cloudhsm = ["mypy-boto3-cloudhsm (==1.17.64.2)"] +cloudhsmv2 = ["mypy-boto3-cloudhsmv2 (==1.17.64.2)"] +cloudsearch = ["mypy-boto3-cloudsearch (==1.17.64.2)"] +cloudsearchdomain = ["mypy-boto3-cloudsearchdomain (==1.17.64.2)"] +cloudtrail = ["mypy-boto3-cloudtrail (==1.17.64.2)"] +cloudwatch = ["mypy-boto3-cloudwatch (==1.17.64.2)"] +codeartifact = ["mypy-boto3-codeartifact (==1.17.64.2)"] +codebuild = ["mypy-boto3-codebuild (==1.17.64.2)"] +codecommit = ["mypy-boto3-codecommit (==1.17.64.2)"] +codedeploy = ["mypy-boto3-codedeploy (==1.17.64.2)"] +codeguru-reviewer = ["mypy-boto3-codeguru-reviewer (==1.17.64.2)"] +codeguruprofiler = ["mypy-boto3-codeguruprofiler (==1.17.64.2)"] +codepipeline = ["mypy-boto3-codepipeline (==1.17.64.2)"] +codestar = ["mypy-boto3-codestar (==1.17.64.2)"] +codestar-connections = ["mypy-boto3-codestar-connections (==1.17.64.2)"] +codestar-notifications = ["mypy-boto3-codestar-notifications (==1.17.64.2)"] +cognito-identity = ["mypy-boto3-cognito-identity (==1.17.64.2)"] +cognito-idp = ["mypy-boto3-cognito-idp (==1.17.64.2)"] +cognito-sync = ["mypy-boto3-cognito-sync (==1.17.64.2)"] +comprehend = ["mypy-boto3-comprehend (==1.17.64.2)"] +comprehendmedical = ["mypy-boto3-comprehendmedical (==1.17.64.2)"] +compute-optimizer = ["mypy-boto3-compute-optimizer (==1.17.64.2)"] +config = ["mypy-boto3-config (==1.17.64.2)"] +connect = ["mypy-boto3-connect (==1.17.64.2)"] +connect-contact-lens = ["mypy-boto3-connect-contact-lens (==1.17.64.2)"] +connectparticipant = ["mypy-boto3-connectparticipant (==1.17.64.2)"] +cur = ["mypy-boto3-cur (==1.17.64.2)"] +customer-profiles = ["mypy-boto3-customer-profiles (==1.17.64.2)"] +databrew = ["mypy-boto3-databrew (==1.17.64.2)"] +dataexchange = ["mypy-boto3-dataexchange (==1.17.64.2)"] +datapipeline = ["mypy-boto3-datapipeline (==1.17.64.2)"] +datasync = ["mypy-boto3-datasync (==1.17.64.2)"] +dax = ["mypy-boto3-dax (==1.17.64.2)"] +detective = ["mypy-boto3-detective (==1.17.64.2)"] +devicefarm = ["mypy-boto3-devicefarm (==1.17.64.2)"] +devops-guru = ["mypy-boto3-devops-guru (==1.17.64.2)"] +directconnect = ["mypy-boto3-directconnect (==1.17.64.2)"] +discovery = ["mypy-boto3-discovery (==1.17.64.2)"] +dlm = ["mypy-boto3-dlm (==1.17.64.2)"] +dms = ["mypy-boto3-dms (==1.17.64.2)"] +docdb = ["mypy-boto3-docdb (==1.17.64.2)"] +ds = ["mypy-boto3-ds (==1.17.64.2)"] +dynamodb = ["mypy-boto3-dynamodb (==1.17.64.2)"] +dynamodbstreams = ["mypy-boto3-dynamodbstreams (==1.17.64.2)"] +ebs = ["mypy-boto3-ebs (==1.17.64.2)"] +ec2 = ["mypy-boto3-ec2 (==1.17.64.2)"] +ec2-instance-connect = ["mypy-boto3-ec2-instance-connect (==1.17.64.2)"] +ecr = ["mypy-boto3-ecr (==1.17.64.2)"] +ecr-public = ["mypy-boto3-ecr-public (==1.17.64.2)"] +ecs = ["mypy-boto3-ecs (==1.17.64.2)"] +efs = ["mypy-boto3-efs (==1.17.64.2)"] +eks = ["mypy-boto3-eks (==1.17.64.2)"] +elastic-inference = ["mypy-boto3-elastic-inference (==1.17.64.2)"] +elasticache = ["mypy-boto3-elasticache (==1.17.64.2)"] +elasticbeanstalk = ["mypy-boto3-elasticbeanstalk (==1.17.64.2)"] +elastictranscoder = ["mypy-boto3-elastictranscoder (==1.17.64.2)"] +elb = ["mypy-boto3-elb (==1.17.64.2)"] +elbv2 = ["mypy-boto3-elbv2 (==1.17.64.2)"] +emr = ["mypy-boto3-emr (==1.17.64.2)"] +emr-containers = ["mypy-boto3-emr-containers (==1.17.64.2)"] +es = ["mypy-boto3-es (==1.17.64.2)"] +essential = ["mypy-boto3-cloudformation (==1.17.64.2)", "mypy-boto3-dynamodb (==1.17.64.2)", "mypy-boto3-ec2 (==1.17.64.2)", "mypy-boto3-lambda (==1.17.64.2)", "mypy-boto3-rds (==1.17.64.2)", "mypy-boto3-s3 (==1.17.64.2)", "mypy-boto3-sqs (==1.17.64.2)"] +events = ["mypy-boto3-events (==1.17.64.2)"] +finspace = ["mypy-boto3-finspace (==1.17.64.2)"] +finspace-data = ["mypy-boto3-finspace-data (==1.17.64.2)"] +firehose = ["mypy-boto3-firehose (==1.17.64.2)"] +fis = ["mypy-boto3-fis (==1.17.64.2)"] +fms = ["mypy-boto3-fms (==1.17.64.2)"] +forecast = ["mypy-boto3-forecast (==1.17.64.2)"] +forecastquery = ["mypy-boto3-forecastquery (==1.17.64.2)"] +frauddetector = ["mypy-boto3-frauddetector (==1.17.64.2)"] +fsx = ["mypy-boto3-fsx (==1.17.64.2)"] +gamelift = ["mypy-boto3-gamelift (==1.17.64.2)"] +glacier = ["mypy-boto3-glacier (==1.17.64.2)"] +globalaccelerator = ["mypy-boto3-globalaccelerator (==1.17.64.2)"] +glue = ["mypy-boto3-glue (==1.17.64.2)"] +greengrass = ["mypy-boto3-greengrass (==1.17.64.2)"] +greengrassv2 = ["mypy-boto3-greengrassv2 (==1.17.64.2)"] +groundstation = ["mypy-boto3-groundstation (==1.17.64.2)"] +guardduty = ["mypy-boto3-guardduty (==1.17.64.2)"] +health = ["mypy-boto3-health (==1.17.64.2)"] +healthlake = ["mypy-boto3-healthlake (==1.17.64.2)"] +honeycode = ["mypy-boto3-honeycode (==1.17.64.2)"] +iam = ["mypy-boto3-iam (==1.17.64.2)"] +identitystore = ["mypy-boto3-identitystore (==1.17.64.2)"] +imagebuilder = ["mypy-boto3-imagebuilder (==1.17.64.2)"] +importexport = ["mypy-boto3-importexport (==1.17.64.2)"] +inspector = ["mypy-boto3-inspector (==1.17.64.2)"] +iot = ["mypy-boto3-iot (==1.17.64.2)"] +iot-data = ["mypy-boto3-iot-data (==1.17.64.2)"] +iot-jobs-data = ["mypy-boto3-iot-jobs-data (==1.17.64.2)"] +iot1click-devices = ["mypy-boto3-iot1click-devices (==1.17.64.2)"] +iot1click-projects = ["mypy-boto3-iot1click-projects (==1.17.64.2)"] +iotanalytics = ["mypy-boto3-iotanalytics (==1.17.64.2)"] +iotdeviceadvisor = ["mypy-boto3-iotdeviceadvisor (==1.17.64.2)"] +iotevents = ["mypy-boto3-iotevents (==1.17.64.2)"] +iotevents-data = ["mypy-boto3-iotevents-data (==1.17.64.2)"] +iotfleethub = ["mypy-boto3-iotfleethub (==1.17.64.2)"] +iotsecuretunneling = ["mypy-boto3-iotsecuretunneling (==1.17.64.2)"] +iotsitewise = ["mypy-boto3-iotsitewise (==1.17.64.2)"] +iotthingsgraph = ["mypy-boto3-iotthingsgraph (==1.17.64.2)"] +iotwireless = ["mypy-boto3-iotwireless (==1.17.64.2)"] +ivs = ["mypy-boto3-ivs (==1.17.64.2)"] +kafka = ["mypy-boto3-kafka (==1.17.64.2)"] +kendra = ["mypy-boto3-kendra (==1.17.64.2)"] +kinesis = ["mypy-boto3-kinesis (==1.17.64.2)"] +kinesis-video-archived-media = ["mypy-boto3-kinesis-video-archived-media (==1.17.64.2)"] +kinesis-video-media = ["mypy-boto3-kinesis-video-media (==1.17.64.2)"] +kinesis-video-signaling = ["mypy-boto3-kinesis-video-signaling (==1.17.64.2)"] +kinesisanalytics = ["mypy-boto3-kinesisanalytics (==1.17.64.2)"] +kinesisanalyticsv2 = ["mypy-boto3-kinesisanalyticsv2 (==1.17.64.2)"] +kinesisvideo = ["mypy-boto3-kinesisvideo (==1.17.64.2)"] +kms = ["mypy-boto3-kms (==1.17.64.2)"] +lakeformation = ["mypy-boto3-lakeformation (==1.17.64.2)"] +lambda = ["mypy-boto3-lambda (==1.17.64.2)"] +lex-models = ["mypy-boto3-lex-models (==1.17.64.2)"] +lex-runtime = ["mypy-boto3-lex-runtime (==1.17.64.2)"] +lexv2-models = ["mypy-boto3-lexv2-models (==1.17.64.2)"] +lexv2-runtime = ["mypy-boto3-lexv2-runtime (==1.17.64.2)"] +license-manager = ["mypy-boto3-license-manager (==1.17.64.2)"] +lightsail = ["mypy-boto3-lightsail (==1.17.64.2)"] +location = ["mypy-boto3-location (==1.17.64.2)"] +logs = ["mypy-boto3-logs (==1.17.64.2)"] +lookoutequipment = ["mypy-boto3-lookoutequipment (==1.17.64.2)"] +lookoutmetrics = ["mypy-boto3-lookoutmetrics (==1.17.64.2)"] +lookoutvision = ["mypy-boto3-lookoutvision (==1.17.64.2)"] +machinelearning = ["mypy-boto3-machinelearning (==1.17.64.2)"] +macie = ["mypy-boto3-macie (==1.17.64.2)"] +macie2 = ["mypy-boto3-macie2 (==1.17.64.2)"] +managedblockchain = ["mypy-boto3-managedblockchain (==1.17.64.2)"] +marketplace-catalog = ["mypy-boto3-marketplace-catalog (==1.17.64.2)"] +marketplace-entitlement = ["mypy-boto3-marketplace-entitlement (==1.17.64.2)"] +marketplacecommerceanalytics = ["mypy-boto3-marketplacecommerceanalytics (==1.17.64.2)"] +mediaconnect = ["mypy-boto3-mediaconnect (==1.17.64.2)"] +mediaconvert = ["mypy-boto3-mediaconvert (==1.17.64.2)"] +medialive = ["mypy-boto3-medialive (==1.17.64.2)"] +mediapackage = ["mypy-boto3-mediapackage (==1.17.64.2)"] +mediapackage-vod = ["mypy-boto3-mediapackage-vod (==1.17.64.2)"] +mediastore = ["mypy-boto3-mediastore (==1.17.64.2)"] +mediastore-data = ["mypy-boto3-mediastore-data (==1.17.64.2)"] +mediatailor = ["mypy-boto3-mediatailor (==1.17.64.2)"] +meteringmarketplace = ["mypy-boto3-meteringmarketplace (==1.17.64.2)"] +mgh = ["mypy-boto3-mgh (==1.17.64.2)"] +mgn = ["mypy-boto3-mgn (==1.17.64.2)"] +migrationhub-config = ["mypy-boto3-migrationhub-config (==1.17.64.2)"] +mobile = ["mypy-boto3-mobile (==1.17.64.2)"] +mq = ["mypy-boto3-mq (==1.17.64.2)"] +mturk = ["mypy-boto3-mturk (==1.17.64.2)"] +mwaa = ["mypy-boto3-mwaa (==1.17.64.2)"] +neptune = ["mypy-boto3-neptune (==1.17.64.2)"] +network-firewall = ["mypy-boto3-network-firewall (==1.17.64.2)"] +networkmanager = ["mypy-boto3-networkmanager (==1.17.64.2)"] +nimble = ["mypy-boto3-nimble (==1.17.64.2)"] +opsworks = ["mypy-boto3-opsworks (==1.17.64.2)"] +opsworkscm = ["mypy-boto3-opsworkscm (==1.17.64.2)"] +organizations = ["mypy-boto3-organizations (==1.17.64.2)"] +outposts = ["mypy-boto3-outposts (==1.17.64.2)"] +personalize = ["mypy-boto3-personalize (==1.17.64.2)"] +personalize-events = ["mypy-boto3-personalize-events (==1.17.64.2)"] +personalize-runtime = ["mypy-boto3-personalize-runtime (==1.17.64.2)"] +pi = ["mypy-boto3-pi (==1.17.64.2)"] +pinpoint = ["mypy-boto3-pinpoint (==1.17.64.2)"] +pinpoint-email = ["mypy-boto3-pinpoint-email (==1.17.64.2)"] +pinpoint-sms-voice = ["mypy-boto3-pinpoint-sms-voice (==1.17.64.2)"] +polly = ["mypy-boto3-polly (==1.17.64.2)"] +pricing = ["mypy-boto3-pricing (==1.17.64.2)"] +qldb = ["mypy-boto3-qldb (==1.17.64.2)"] +qldb-session = ["mypy-boto3-qldb-session (==1.17.64.2)"] +quicksight = ["mypy-boto3-quicksight (==1.17.64.2)"] +ram = ["mypy-boto3-ram (==1.17.64.2)"] +rds = ["mypy-boto3-rds (==1.17.64.2)"] +rds-data = ["mypy-boto3-rds-data (==1.17.64.2)"] +redshift = ["mypy-boto3-redshift (==1.17.64.2)"] +redshift-data = ["mypy-boto3-redshift-data (==1.17.64.2)"] +rekognition = ["mypy-boto3-rekognition (==1.17.64.2)"] +resource-groups = ["mypy-boto3-resource-groups (==1.17.64.2)"] +resourcegroupstaggingapi = ["mypy-boto3-resourcegroupstaggingapi (==1.17.64.2)"] +robomaker = ["mypy-boto3-robomaker (==1.17.64.2)"] +route53 = ["mypy-boto3-route53 (==1.17.64.2)"] +route53domains = ["mypy-boto3-route53domains (==1.17.64.2)"] +route53resolver = ["mypy-boto3-route53resolver (==1.17.64.2)"] +s3 = ["mypy-boto3-s3 (==1.17.64.2)"] +s3control = ["mypy-boto3-s3control (==1.17.64.2)"] +s3outposts = ["mypy-boto3-s3outposts (==1.17.64.2)"] +sagemaker = ["mypy-boto3-sagemaker (==1.17.64.2)"] +sagemaker-a2i-runtime = ["mypy-boto3-sagemaker-a2i-runtime (==1.17.64.2)"] +sagemaker-edge = ["mypy-boto3-sagemaker-edge (==1.17.64.2)"] +sagemaker-featurestore-runtime = ["mypy-boto3-sagemaker-featurestore-runtime (==1.17.64.2)"] +sagemaker-runtime = ["mypy-boto3-sagemaker-runtime (==1.17.64.2)"] +savingsplans = ["mypy-boto3-savingsplans (==1.17.64.2)"] +schemas = ["mypy-boto3-schemas (==1.17.64.2)"] +sdb = ["mypy-boto3-sdb (==1.17.64.2)"] +secretsmanager = ["mypy-boto3-secretsmanager (==1.17.64.2)"] +securityhub = ["mypy-boto3-securityhub (==1.17.64.2)"] +serverlessrepo = ["mypy-boto3-serverlessrepo (==1.17.64.2)"] +service-quotas = ["mypy-boto3-service-quotas (==1.17.64.2)"] +servicecatalog = ["mypy-boto3-servicecatalog (==1.17.64.2)"] +servicecatalog-appregistry = ["mypy-boto3-servicecatalog-appregistry (==1.17.64.2)"] +servicediscovery = ["mypy-boto3-servicediscovery (==1.17.64.2)"] +ses = ["mypy-boto3-ses (==1.17.64.2)"] +sesv2 = ["mypy-boto3-sesv2 (==1.17.64.2)"] +shield = ["mypy-boto3-shield (==1.17.64.2)"] +signer = ["mypy-boto3-signer (==1.17.64.2)"] +sms = ["mypy-boto3-sms (==1.17.64.2)"] +sms-voice = ["mypy-boto3-sms-voice (==1.17.64.2)"] +snowball = ["mypy-boto3-snowball (==1.17.64.2)"] +sns = ["mypy-boto3-sns (==1.17.64.2)"] +sqs = ["mypy-boto3-sqs (==1.17.64.2)"] +ssm = ["mypy-boto3-ssm (==1.17.64.2)"] +sso = ["mypy-boto3-sso (==1.17.64.2)"] +sso-admin = ["mypy-boto3-sso-admin (==1.17.64.2)"] +sso-oidc = ["mypy-boto3-sso-oidc (==1.17.64.2)"] +stepfunctions = ["mypy-boto3-stepfunctions (==1.17.64.2)"] +storagegateway = ["mypy-boto3-storagegateway (==1.17.64.2)"] +sts = ["mypy-boto3-sts (==1.17.64.2)"] +support = ["mypy-boto3-support (==1.17.64.2)"] +swf = ["mypy-boto3-swf (==1.17.64.2)"] +synthetics = ["mypy-boto3-synthetics (==1.17.64.2)"] +textract = ["mypy-boto3-textract (==1.17.64.2)"] +timestream-query = ["mypy-boto3-timestream-query (==1.17.64.2)"] +timestream-write = ["mypy-boto3-timestream-write (==1.17.64.2)"] +transcribe = ["mypy-boto3-transcribe (==1.17.64.2)"] +transfer = ["mypy-boto3-transfer (==1.17.64.2)"] +translate = ["mypy-boto3-translate (==1.17.64.2)"] +waf = ["mypy-boto3-waf (==1.17.64.2)"] +waf-regional = ["mypy-boto3-waf-regional (==1.17.64.2)"] +wafv2 = ["mypy-boto3-wafv2 (==1.17.64.2)"] +wellarchitected = ["mypy-boto3-wellarchitected (==1.17.64.2)"] +workdocs = ["mypy-boto3-workdocs (==1.17.64.2)"] +worklink = ["mypy-boto3-worklink (==1.17.64.2)"] +workmail = ["mypy-boto3-workmail (==1.17.64.2)"] +workmailmessageflow = ["mypy-boto3-workmailmessageflow (==1.17.64.2)"] +workspaces = ["mypy-boto3-workspaces (==1.17.64.2)"] +xray = ["mypy-boto3-xray (==1.17.64.2)"] [[package]] name = "botocore" -version = "1.20.32" +version = "1.20.53" description = "Low-level, data-driven core of boto 3." category = "main" optional = false @@ -504,7 +511,7 @@ yaml = ["PyYAML (>=3.10)"] [[package]] name = "cryptography" -version = "3.4.6" +version = "3.4.7" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." category = "dev" optional = false @@ -542,7 +549,7 @@ python-versions = "*" [[package]] name = "dcicsnovault" -version = "4.6.0" +version = "4.7.6" description = "Storage support for 4DN Data Portals." category = "main" optional = false @@ -550,7 +557,6 @@ python-versions = ">=3.6,<3.7" [package.dependencies] aws_requests_auth = ">=0.4.1,<0.5.0" -awscli = ">=1.15.42,<2.0.0" "backports.statistics" = "0.1.0" boto3 = ">=1.7.42,<2.0.0" dcicutils = ">=1.8.3,<2.0.0" @@ -561,7 +567,6 @@ html5lib = "0.9999999" humanfriendly = ">=1.44.5,<2.0.0" jsonschema_serialize_fork = "2.1.1" keepalive = "0.5" -loremipsum = "1.0.5" MarkupSafe = ">=0.23,<1" netaddr = ">=0.7.18,<1" passlib = ">=1.6.5,<2.0.0" @@ -581,12 +586,10 @@ pytz = ">=2020.1" PyYAML = ">=5.1,<5.3" rdflib = ">=4.2.2,<5.0.0" rdflib-jsonld = ">=0.3.0,<1.0.0" -rfc3987 = ">=1.3.6,<2.0.0" rutter = ">=0.2,<1" simplejson = "3.17.0" SPARQLWrapper = ">=1.7.6,<2.0.0" SQLAlchemy = "1.3.16" -strict-rfc3339 = ">=0.7,<1" structlog = ">=18.1.0,<20" subprocess_middleware = ">=0.3,<1" transaction = ">=2.4.0,<3.0.0" @@ -622,21 +625,20 @@ webtest = ">=2.0.34,<3.0.0" [[package]] name = "docker" -version = "4.4.4" +version = "5.0.0" description = "A Python library for the Docker Engine API." category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.6" [package.dependencies] pywin32 = {version = "227", markers = "sys_platform == \"win32\""} requests = ">=2.14.2,<2.18.0 || >2.18.0" -six = ">=1.4.0" websocket-client = ">=0.32.0" [package.extras] ssh = ["paramiko (>=2.4.2)"] -tls = ["pyOpenSSL (>=17.5.0)", "cryptography (>=1.3.4)", "idna (>=2.0.0)"] +tls = ["pyOpenSSL (>=17.5.0)", "cryptography (>=3.4.7)", "idna (>=2.0.0)"] [[package]] name = "docopt" @@ -702,11 +704,11 @@ develop = ["mock", "pytest (>=3.0.0)", "pytest-cov", "pytz", "coverage (<5.0.0)" [[package]] name = "et-xmlfile" -version = "1.0.1" +version = "1.1.0" description = "An implementation of lxml.xmlfile for the standard library" category = "main" optional = false -python-versions = "*" +python-versions = ">=3.6" [[package]] name = "execnet" @@ -721,7 +723,7 @@ apipkg = ">=1.4" [[package]] name = "flake8" -version = "3.9.0" +version = "3.9.1" description = "the modular source code checker: pep8 pyflakes and co" category = "dev" optional = false @@ -778,25 +780,26 @@ python-versions = "*" [[package]] name = "gitdb" -version = "4.0.5" +version = "4.0.7" description = "Git Object Database" category = "main" optional = false python-versions = ">=3.4" [package.dependencies] -smmap = ">=3.0.1,<4" +smmap = ">=3.0.1,<5" [[package]] name = "gitpython" -version = "3.1.14" +version = "3.1.15" description = "Python Git Library" category = "main" optional = false -python-versions = ">=3.4" +python-versions = ">=3.5" [package.dependencies] gitdb = ">=4.0.1,<5" +typing-extensions = ">=3.7.4.0" [[package]] name = "granite-suite" @@ -867,7 +870,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "importlib-metadata" -version = "3.7.3" +version = "4.0.1" description = "Read metadata from Python packages" category = "dev" optional = false @@ -879,7 +882,7 @@ zipp = ">=0.5" [package.extras] docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "pytest-enabler", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] +testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] [[package]] name = "importlib-resources" @@ -1285,7 +1288,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "pyflakes" -version = "2.3.0" +version = "2.3.1" description = "passive checker of Python programs" category = "dev" optional = false @@ -1640,7 +1643,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] [[package]] name = "responses" -version = "0.13.1" +version = "0.13.3" description = "A utility library for mocking out the `requests` Python library." category = "dev" optional = false @@ -1689,7 +1692,7 @@ testing = ["nose", "coverage"] [[package]] name = "s3transfer" -version = "0.3.6" +version = "0.3.7" description = "An Amazon S3 Transfer Manager" category = "main" optional = false @@ -1743,15 +1746,15 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "smmap" -version = "3.0.5" +version = "4.0.0" description = "A pure Python implementation of a sliding window memory map manager" category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.5" [[package]] name = "soupsieve" -version = "2.2" +version = "2.2.1" description = "A modern CSS selector implementation for Beautiful Soup." category = "main" optional = false @@ -1791,14 +1794,6 @@ postgresql_psycopg2binary = ["psycopg2-binary"] postgresql_psycopg2cffi = ["psycopg2cffi"] pymysql = ["pymysql"] -[[package]] -name = "strict-rfc3339" -version = "0.7" -description = "Strict, simple, lightweight RFC3339 functions" -category = "main" -optional = false -python-versions = "*" - [[package]] name = "structlog" version = "19.2.0" @@ -1840,7 +1835,7 @@ python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "tqdm" -version = "4.59.0" +version = "4.60.0" description = "Fast, Extensible Progress Meter" category = "main" optional = false @@ -1877,9 +1872,9 @@ python-versions = "*" [[package]] name = "typing-extensions" -version = "3.7.4.3" +version = "3.10.0.0" description = "Backported and Experimental Type Hints for Python 3.5+" -category = "dev" +category = "main" optional = false python-versions = "*" @@ -2084,7 +2079,7 @@ test = ["zope.testing"] [metadata] lock-version = "1.1" python-versions = ">=3.6.1,<3.7" -content-hash = "f757ac1b0d257e70d7fc609424468b780f99229bba95d4625531c5863ed25b2d" +content-hash = "7e4cd37bd8cbbd1a6848c37089b6157d2abb937436a32ff6faaf4346c8fd3269" [metadata.files] apipkg = [ @@ -2108,8 +2103,8 @@ aws-xray-sdk = [ {file = "aws_xray_sdk-0.95-py2.py3-none-any.whl", hash = "sha256:72791618feb22eaff2e628462b0d58f398ce8c1bacfa989b7679817ab1fad60c"}, ] awscli = [ - {file = "awscli-1.19.32-py2.py3-none-any.whl", hash = "sha256:f22d8a856707a0d1cc4193cb26d98c5d171228525dd527dd959783c020ac13f6"}, - {file = "awscli-1.19.32.tar.gz", hash = "sha256:760796a154a8b1b3ae008890edf3b961610ef6f9ea19978316be86b575372591"}, + {file = "awscli-1.19.53-py2.py3-none-any.whl", hash = "sha256:cebe94ffdaa2dcc8c63eb31844df1f267a01e3c3376d47edbcf927e216438fca"}, + {file = "awscli-1.19.53.tar.gz", hash = "sha256:a58d23d834f2fceb4810190f0d69951fcdfbf27233b3def68e59e6a59dcbe547"}, ] "backports.statistics" = [ {file = "backports.statistics-0.1.0-py2.py3-none-any.whl", hash = "sha256:2732e003151620762ba3ea25b881b5ca0debe2fcbf41b32b6eaff5842a8b99d7"}, @@ -2121,23 +2116,23 @@ beautifulsoup4 = [ {file = "beautifulsoup4-4.9.3.tar.gz", hash = "sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25"}, ] bitarray = [ - {file = "bitarray-1.7.1.tar.gz", hash = "sha256:e4de977d708b7024760266d827b8285e4405dce4293f25508c4556970139018a"}, + {file = "bitarray-2.0.1.tar.gz", hash = "sha256:ec3a4f6d711a79ed23aea9541638d3353dc3f083f293a13180b14b482e3e19ef"}, ] boto = [ {file = "boto-2.49.0-py2.py3-none-any.whl", hash = "sha256:147758d41ae7240dc989f0039f27da8ca0d53734be0eb869ef16e3adcfa462e8"}, {file = "boto-2.49.0.tar.gz", hash = "sha256:ea0d3b40a2d852767be77ca343b58a9e3a4b00d9db440efb8da74b4e58025e5a"}, ] boto3 = [ - {file = "boto3-1.17.32-py2.py3-none-any.whl", hash = "sha256:85069e044c871571b7a72b600123761e1747ad18328fa5188f13b097a394251d"}, - {file = "boto3-1.17.32.tar.gz", hash = "sha256:2df49a4360ef0640902c26bf93ac0791b37755db3a2efd0d51a03eddeb9f807e"}, + {file = "boto3-1.17.53-py2.py3-none-any.whl", hash = "sha256:3bf3305571f3c8b738a53e9e7dcff59137dffe94670046c084a17f9fa4599ff3"}, + {file = "boto3-1.17.53.tar.gz", hash = "sha256:1d26f6e7ae3c940cb07119077ac42485dcf99164350da0ab50d0f5ad345800cd"}, ] boto3-stubs = [ - {file = "boto3-stubs-1.17.31.0.tar.gz", hash = "sha256:110b7343d44d02518cfe2e528e9a1bf20af7cb8c81e8d8914f70a9676eda4ef9"}, - {file = "boto3_stubs-1.17.31.0-py3-none-any.whl", hash = "sha256:3db39d41c69866c71bcf806fb5a9c734208ca5f5995a51744579749a1c531fa7"}, + {file = "boto3-stubs-1.17.64.2.tar.gz", hash = "sha256:c2835e609680c4327dd58d701b5ed6905f65afd95b094ed6514c1677bf90ee13"}, + {file = "boto3_stubs-1.17.64.2-py3-none-any.whl", hash = "sha256:6b43a0094cee77679b462c2caddab1aa62c13e46c1268e62dba9b0e0f466dabf"}, ] botocore = [ - {file = "botocore-1.20.32-py2.py3-none-any.whl", hash = "sha256:fc162b69adc54c9ef037fec31f93d46df0bf8a9c3d201ecb16d0a9ec5f2936ec"}, - {file = "botocore-1.20.32.tar.gz", hash = "sha256:1cef96e716f7481bfb3b8d3c3d45e640d588aa5af613cd6d7f33d784aa2e8d12"}, + {file = "botocore-1.20.53-py2.py3-none-any.whl", hash = "sha256:d5e70d17b91c9b5867be7d6de0caa7dde9ed789bed62f03ea9b60718dc9350bf"}, + {file = "botocore-1.20.53.tar.gz", hash = "sha256:e303500c4e80f6a706602da53daa6f751cfa8f491665c99a24ee732ab6321573"}, ] cached-property = [ {file = "cached-property-1.5.2.tar.gz", hash = "sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130"}, @@ -2260,18 +2255,18 @@ coveralls = [ {file = "coveralls-3.0.1.tar.gz", hash = "sha256:cbb942ae5ef3d2b55388cb5b43e93a269544911535f1e750e1c656aef019ce60"}, ] cryptography = [ - {file = "cryptography-3.4.6-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:57ad77d32917bc55299b16d3b996ffa42a1c73c6cfa829b14043c561288d2799"}, - {file = "cryptography-3.4.6-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:4169a27b818de4a1860720108b55a2801f32b6ae79e7f99c00d79f2a2822eeb7"}, - {file = "cryptography-3.4.6-cp36-abi3-manylinux2010_x86_64.whl", hash = "sha256:93cfe5b7ff006de13e1e89830810ecbd014791b042cbe5eec253be11ac2b28f3"}, - {file = "cryptography-3.4.6-cp36-abi3-manylinux2014_aarch64.whl", hash = "sha256:5ecf2bcb34d17415e89b546dbb44e73080f747e504273e4d4987630493cded1b"}, - {file = "cryptography-3.4.6-cp36-abi3-manylinux2014_x86_64.whl", hash = "sha256:fec7fb46b10da10d9e1d078d1ff8ed9e05ae14f431fdbd11145edd0550b9a964"}, - {file = "cryptography-3.4.6-cp36-abi3-win32.whl", hash = "sha256:df186fcbf86dc1ce56305becb8434e4b6b7504bc724b71ad7a3239e0c9d14ef2"}, - {file = "cryptography-3.4.6-cp36-abi3-win_amd64.whl", hash = "sha256:66b57a9ca4b3221d51b237094b0303843b914b7d5afd4349970bb26518e350b0"}, - {file = "cryptography-3.4.6-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:066bc53f052dfeda2f2d7c195cf16fb3e5ff13e1b6b7415b468514b40b381a5b"}, - {file = "cryptography-3.4.6-pp36-pypy36_pp73-manylinux2014_x86_64.whl", hash = "sha256:600cf9bfe75e96d965509a4c0b2b183f74a4fa6f5331dcb40fb7b77b7c2484df"}, - {file = "cryptography-3.4.6-pp37-pypy37_pp73-manylinux2010_x86_64.whl", hash = "sha256:0923ba600d00718d63a3976f23cab19aef10c1765038945628cd9be047ad0336"}, - {file = "cryptography-3.4.6-pp37-pypy37_pp73-manylinux2014_x86_64.whl", hash = "sha256:9e98b452132963678e3ac6c73f7010fe53adf72209a32854d55690acac3f6724"}, - {file = "cryptography-3.4.6.tar.gz", hash = "sha256:2d32223e5b0ee02943f32b19245b61a62db83a882f0e76cc564e1cec60d48f87"}, + {file = "cryptography-3.4.7-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:3d8427734c781ea5f1b41d6589c293089704d4759e34597dce91014ac125aad1"}, + {file = "cryptography-3.4.7-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:8e56e16617872b0957d1c9742a3f94b43533447fd78321514abbe7db216aa250"}, + {file = "cryptography-3.4.7-cp36-abi3-manylinux2010_x86_64.whl", hash = "sha256:37340614f8a5d2fb9aeea67fd159bfe4f5f4ed535b1090ce8ec428b2f15a11f2"}, + {file = "cryptography-3.4.7-cp36-abi3-manylinux2014_aarch64.whl", hash = "sha256:240f5c21aef0b73f40bb9f78d2caff73186700bf1bc6b94285699aff98cc16c6"}, + {file = "cryptography-3.4.7-cp36-abi3-manylinux2014_x86_64.whl", hash = "sha256:1e056c28420c072c5e3cb36e2b23ee55e260cb04eee08f702e0edfec3fb51959"}, + {file = "cryptography-3.4.7-cp36-abi3-win32.whl", hash = "sha256:0f1212a66329c80d68aeeb39b8a16d54ef57071bf22ff4e521657b27372e327d"}, + {file = "cryptography-3.4.7-cp36-abi3-win_amd64.whl", hash = "sha256:de4e5f7f68220d92b7637fc99847475b59154b7a1b3868fb7385337af54ac9ca"}, + {file = "cryptography-3.4.7-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:26965837447f9c82f1855e0bc8bc4fb910240b6e0d16a664bb722df3b5b06873"}, + {file = "cryptography-3.4.7-pp36-pypy36_pp73-manylinux2014_x86_64.whl", hash = "sha256:eb8cc2afe8b05acbd84a43905832ec78e7b3873fb124ca190f574dca7389a87d"}, + {file = "cryptography-3.4.7-pp37-pypy37_pp73-manylinux2010_x86_64.whl", hash = "sha256:7ec5d3b029f5fa2b179325908b9cd93db28ab7b85bb6c1db56b10e0b54235177"}, + {file = "cryptography-3.4.7-pp37-pypy37_pp73-manylinux2014_x86_64.whl", hash = "sha256:ee77aa129f481be46f8d92a1a7db57269a2f23052d5f2433b4621bb457081cc9"}, + {file = "cryptography-3.4.7.tar.gz", hash = "sha256:3d10de8116d25649631977cb37da6cbdd2d6fa0e0281d014a5b7d337255ca713"}, ] cycler = [ {file = "cycler-0.10.0-py2.py3-none-any.whl", hash = "sha256:1d8a5ae1ff6c5cf9b93e8811e581232ad8920aeec647c37316ceac982b08cb2d"}, @@ -2281,16 +2276,16 @@ dcicpyvcf = [ {file = "dcicpyvcf-1.0.0.tar.gz", hash = "sha256:c5bf8d585002ab3b95d13a47803376b456b931865e4189c38a18cca47b108449"}, ] dcicsnovault = [ - {file = "dcicsnovault-4.6.0-py3-none-any.whl", hash = "sha256:0e63c64064321a7dd3734cba2d4d7fcf2e3a0b0c7ac3ba661c8808ec6e7fe401"}, - {file = "dcicsnovault-4.6.0.tar.gz", hash = "sha256:78dc99c5f59da2a7bcb4a83de0c255cea467f216aaadc39ba2453c8e85491cc1"}, + {file = "dcicsnovault-4.7.6-py3-none-any.whl", hash = "sha256:0ff0c37ba9201dff7f3a778f5feb2acfd8eb22f68ebbbf1e579cf6e6b664dc8e"}, + {file = "dcicsnovault-4.7.6.tar.gz", hash = "sha256:4e2a3f043b4e4632053bc7b553a2ce67c5a501de3f6af096efda3506998642ad"}, ] dcicutils = [ {file = "dcicutils-1.12.0-py3-none-any.whl", hash = "sha256:c3f5e4601d306f8e3634d7359ff1735dad2b38bc4e8b91eeb370ab2b3eaebcbe"}, {file = "dcicutils-1.12.0.tar.gz", hash = "sha256:92747a45313966185052030f95476f91d15ee4821ca0cc9678e3ad63325325d0"}, ] docker = [ - {file = "docker-4.4.4-py2.py3-none-any.whl", hash = "sha256:f3607d5695be025fa405a12aca2e5df702a57db63790c73b927eb6a94aac60af"}, - {file = "docker-4.4.4.tar.gz", hash = "sha256:d3393c878f575d3a9ca3b94471a3c89a6d960b35feb92f033c0de36cc9d934db"}, + {file = "docker-5.0.0-py2.py3-none-any.whl", hash = "sha256:fc961d622160e8021c10d1bcabc388c57d55fb1f917175afbe24af442e6879bd"}, + {file = "docker-5.0.0.tar.gz", hash = "sha256:3e8bc47534e0ca9331d72c32f2881bb13b93ded0bcdeab3c833fb7cf61c0a9a5"}, ] docopt = [ {file = "docopt-0.6.2.tar.gz", hash = "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"}, @@ -2312,15 +2307,16 @@ elasticsearch-dsl = [ {file = "elasticsearch_dsl-6.4.0-py2.py3-none-any.whl", hash = "sha256:f60aea7fd756ac1fbe7ce114bbf4949aefbf495dfe8896640e787c67344f12f6"}, ] et-xmlfile = [ - {file = "et_xmlfile-1.0.1.tar.gz", hash = "sha256:614d9722d572f6246302c4491846d2c393c199cfa4edc9af593437691683335b"}, + {file = "et_xmlfile-1.1.0-py3-none-any.whl", hash = "sha256:a2ba85d1d6a74ef63837eed693bcb89c3f752169b0e3e7ae5b16ca5e1b3deada"}, + {file = "et_xmlfile-1.1.0.tar.gz", hash = "sha256:8eb9e2bc2f8c97e37a2dc85a09ecdcdec9d8a396530a6d5a33b30b9a92da0c5c"}, ] execnet = [ {file = "execnet-1.4.1-py2.py3-none-any.whl", hash = "sha256:d2b909c7945832e1c19cfacd96e78da68bdadc656440cfc7dfe59b766744eb8c"}, {file = "execnet-1.4.1.tar.gz", hash = "sha256:f66dd4a7519725a1b7e14ad9ae7d3df8e09b2da88062386e08e941cafc0ef3e6"}, ] flake8 = [ - {file = "flake8-3.9.0-py2.py3-none-any.whl", hash = "sha256:12d05ab02614b6aee8df7c36b97d1a3b2372761222b19b58621355e82acddcff"}, - {file = "flake8-3.9.0.tar.gz", hash = "sha256:78873e372b12b093da7b5e5ed302e8ad9e988b38b063b61ad937f26ca58fc5f0"}, + {file = "flake8-3.9.1-py2.py3-none-any.whl", hash = "sha256:3b9f848952dddccf635be78098ca75010f073bfe14d2c6bda867154bea728d2a"}, + {file = "flake8-3.9.1.tar.gz", hash = "sha256:1aa8990be1e689d96c745c5682b687ea49f2e05a443aff1f8251092b0014e378"}, ] flaky = [ {file = "flaky-3.7.0-py2.py3-none-any.whl", hash = "sha256:d6eda73cab5ae7364504b7c44670f70abed9e75f77dd116352f662817592ec9c"}, @@ -2339,12 +2335,12 @@ futures = [ {file = "futures-3.1.1.tar.gz", hash = "sha256:51ecb45f0add83c806c68e4b06106f90db260585b25ef2abfcda0bd95c0132fd"}, ] gitdb = [ - {file = "gitdb-4.0.5-py3-none-any.whl", hash = "sha256:91f36bfb1ab7949b3b40e23736db18231bf7593edada2ba5c3a174a7b23657ac"}, - {file = "gitdb-4.0.5.tar.gz", hash = "sha256:c9e1f2d0db7ddb9a704c2a0217be31214e91a4fe1dea1efad19ae42ba0c285c9"}, + {file = "gitdb-4.0.7-py3-none-any.whl", hash = "sha256:6c4cc71933456991da20917998acbe6cf4fb41eeaab7d6d67fbc05ecd4c865b0"}, + {file = "gitdb-4.0.7.tar.gz", hash = "sha256:96bf5c08b157a666fec41129e6d327235284cca4c81e92109260f353ba138005"}, ] gitpython = [ - {file = "GitPython-3.1.14-py3-none-any.whl", hash = "sha256:3283ae2fba31c913d857e12e5ba5f9a7772bbc064ae2bb09efafa71b0dd4939b"}, - {file = "GitPython-3.1.14.tar.gz", hash = "sha256:be27633e7509e58391f10207cd32b2a6cf5b908f92d9cd30da2e514e1137af61"}, + {file = "GitPython-3.1.15-py3-none-any.whl", hash = "sha256:a77824e516d3298b04fb36ec7845e92747df8fcfee9cacc32dd6239f9652f867"}, + {file = "GitPython-3.1.15.tar.gz", hash = "sha256:05af150f47a5cca3f4b0af289b73aef8cf3c4fe2385015b06220cbcdee48bb6e"}, ] granite-suite = [ {file = "granite-suite-0.1.11b0.tar.gz", hash = "sha256:8d5b893ff8658d42cd16e7b2013c2456dad6216aa836fea45767bee07a9aaf97"}, @@ -2382,8 +2378,8 @@ idna = [ {file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"}, ] importlib-metadata = [ - {file = "importlib_metadata-3.7.3-py3-none-any.whl", hash = "sha256:b74159469b464a99cb8cc3e21973e4d96e05d3024d337313fedb618a6e86e6f4"}, - {file = "importlib_metadata-3.7.3.tar.gz", hash = "sha256:742add720a20d0467df2f444ae41704000f50e1234f46174b51f9c6031a1bd71"}, + {file = "importlib_metadata-4.0.1-py3-none-any.whl", hash = "sha256:d7eb1dea6d6a6086f8be21784cc9e3bcfa55872b52309bc5fad53a8ea444465d"}, + {file = "importlib_metadata-4.0.1.tar.gz", hash = "sha256:8c501196e49fb9df5df43833bdb1e4328f64847763ec8a50703148b73784d581"}, ] importlib-resources = [ {file = "importlib_resources-5.1.2-py3-none-any.whl", hash = "sha256:ebab3efe74d83b04d6bf5cd9a17f0c5c93e60fb60f30c90f56265fce4682a469"}, @@ -2747,8 +2743,8 @@ pycryptodome = [ {file = "pycryptodome-3.10.1.tar.gz", hash = "sha256:3e2e3a06580c5f190df843cdb90ea28d61099cf4924334d5297a995de68e4673"}, ] pyflakes = [ - {file = "pyflakes-2.3.0-py2.py3-none-any.whl", hash = "sha256:910208209dcea632721cb58363d0f72913d9e8cf64dc6f8ae2e02a3609aba40d"}, - {file = "pyflakes-2.3.0.tar.gz", hash = "sha256:e59fd8e750e588358f1b8885e5a4751203a0516e0ee6d34811089ac294c8806f"}, + {file = "pyflakes-2.3.1-py2.py3-none-any.whl", hash = "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3"}, + {file = "pyflakes-2.3.1.tar.gz", hash = "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db"}, ] pyjwt = [ {file = "PyJWT-1.5.3-py2.py3-none-any.whl", hash = "sha256:a4e5f1441e3ca7b382fd0c0b416777ced1f97c64ef0c33bfa39daf38505cfd2f"}, @@ -2883,8 +2879,8 @@ requests = [ {file = "requests-2.25.1.tar.gz", hash = "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804"}, ] responses = [ - {file = "responses-0.13.1-py2.py3-none-any.whl", hash = "sha256:3b1ea9cf026edaaf25e853abc4d3b2687d25467e9d8d41e77ee525cad0673f3e"}, - {file = "responses-0.13.1.tar.gz", hash = "sha256:cf62ab0f4119b81d485521b2c950d8aa55a885c90126488450b7acb8ee3f77ac"}, + {file = "responses-0.13.3-py2.py3-none-any.whl", hash = "sha256:b54067596f331786f5ed094ff21e8d79e6a1c68ef625180a7d34808d6f36c11b"}, + {file = "responses-0.13.3.tar.gz", hash = "sha256:18a5b88eb24143adbf2b4100f328a2f5bfa72fbdacf12d97d41f07c26c45553d"}, ] rfc3987 = [ {file = "rfc3987-1.3.8-py2.py3-none-any.whl", hash = "sha256:10702b1e51e5658843460b189b185c0366d2cf4cff716f13111b0ea9fd2dce53"}, @@ -2899,8 +2895,8 @@ rutter = [ {file = "rutter-0.3.tar.gz", hash = "sha256:2607d3c3843f0e6094e62235b7a587b0603bf3985dd3522ee3f739648790bd5c"}, ] s3transfer = [ - {file = "s3transfer-0.3.6-py2.py3-none-any.whl", hash = "sha256:5d48b1fd2232141a9d5fb279709117aaba506cacea7f86f11bc392f06bfa8fc2"}, - {file = "s3transfer-0.3.6.tar.gz", hash = "sha256:c5dadf598762899d8cfaecf68eba649cd25b0ce93b6c954b156aaa3eed160547"}, + {file = "s3transfer-0.3.7-py2.py3-none-any.whl", hash = "sha256:efa5bd92a897b6a8d5c1383828dca3d52d0790e0756d49740563a3fb6ed03246"}, + {file = "s3transfer-0.3.7.tar.gz", hash = "sha256:35627b86af8ff97e7ac27975fe0a98a312814b46c6333d8a6b889627bcd80994"}, ] sentry-sdk = [ {file = "sentry-sdk-0.16.5.tar.gz", hash = "sha256:e12eb1c2c01cd9e9cfe70608dbda4ef451f37ef0b7cbb92e5d43f87c341d6334"}, @@ -2941,12 +2937,12 @@ six = [ {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, ] smmap = [ - {file = "smmap-3.0.5-py2.py3-none-any.whl", hash = "sha256:7bfcf367828031dc893530a29cb35eb8c8f2d7c8f2d0989354d75d24c8573714"}, - {file = "smmap-3.0.5.tar.gz", hash = "sha256:84c2751ef3072d4f6b2785ec7ee40244c6f45eb934d9e543e2c51f1bd3d54c50"}, + {file = "smmap-4.0.0-py2.py3-none-any.whl", hash = "sha256:a9a7479e4c572e2e775c404dcd3080c8dc49f39918c2cf74913d30c4c478e3c2"}, + {file = "smmap-4.0.0.tar.gz", hash = "sha256:7e65386bd122d45405ddf795637b7f7d2b532e7e401d46bbe3fb49b9986d5182"}, ] soupsieve = [ - {file = "soupsieve-2.2-py3-none-any.whl", hash = "sha256:d3a5ea5b350423f47d07639f74475afedad48cf41c0ad7a82ca13a3928af34f6"}, - {file = "soupsieve-2.2.tar.gz", hash = "sha256:407fa1e8eb3458d1b5614df51d9651a1180ea5fedf07feb46e45d7e25e6d6cdd"}, + {file = "soupsieve-2.2.1-py3-none-any.whl", hash = "sha256:c2c1c2d44f158cdbddab7824a9af8c4f83c76b1e23e049479aa432feb6c4c23b"}, + {file = "soupsieve-2.2.1.tar.gz", hash = "sha256:052774848f448cf19c7e959adf5566904d525f33a3f8b6ba6f6f8f26ec7de0cc"}, ] sparqlwrapper = [ {file = "SPARQLWrapper-1.8.5-py2-none-any.whl", hash = "sha256:357ee8a27bc910ea13d77836dbddd0b914991495b8cc1bf70676578155e962a8"}, @@ -2976,9 +2972,6 @@ sqlalchemy = [ {file = "SQLAlchemy-1.3.16-cp38-cp38-win_amd64.whl", hash = "sha256:7d98e0785c4cd7ae30b4a451416db71f5724a1839025544b4edbd92e00b91f0f"}, {file = "SQLAlchemy-1.3.16.tar.gz", hash = "sha256:7224e126c00b8178dfd227bc337ba5e754b197a3867d33b9f30dc0208f773d70"}, ] -strict-rfc3339 = [ - {file = "strict-rfc3339-0.7.tar.gz", hash = "sha256:5cad17bedfc3af57b399db0fed32771f18fc54bbd917e85546088607ac5e1277"}, -] structlog = [ {file = "structlog-19.2.0-py2.py3-none-any.whl", hash = "sha256:6640e6690fc31d5949bc614c1a630464d3aaa625284aeb7c6e486c3010d73e12"}, {file = "structlog-19.2.0.tar.gz", hash = "sha256:4287058cf4ce1a59bc5dea290d6386d37f29a37529c9a51cdf7387e51710152b"}, @@ -2991,8 +2984,8 @@ toml = [ {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] tqdm = [ - {file = "tqdm-4.59.0-py2.py3-none-any.whl", hash = "sha256:9fdf349068d047d4cfbe24862c425883af1db29bcddf4b0eeb2524f6fbdb23c7"}, - {file = "tqdm-4.59.0.tar.gz", hash = "sha256:d666ae29164da3e517fcf125e41d4fe96e5bb375cd87ff9763f6b38b5592fe33"}, + {file = "tqdm-4.60.0-py2.py3-none-any.whl", hash = "sha256:daec693491c52e9498632dfbe9ccfc4882a557f5fa08982db1b4d3adbe0887c3"}, + {file = "tqdm-4.60.0.tar.gz", hash = "sha256:ebdebdb95e3477ceea267decfc0784859aa3df3e27e22d23b83e9b272bf157ae"}, ] transaction = [ {file = "transaction-2.4.0-py2.py3-none-any.whl", hash = "sha256:b96a5e9aaa73f905759bc9ccf0021bf4864c01ac36666e0d28395e871f6d584a"}, @@ -3003,9 +2996,9 @@ translationstring = [ {file = "translationstring-1.3.tar.gz", hash = "sha256:4ee44cfa58c52ade8910ea0ebc3d2d84bdcad9fa0422405b1801ec9b9a65b72d"}, ] typing-extensions = [ - {file = "typing_extensions-3.7.4.3-py2-none-any.whl", hash = "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"}, - {file = "typing_extensions-3.7.4.3-py3-none-any.whl", hash = "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918"}, - {file = "typing_extensions-3.7.4.3.tar.gz", hash = "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c"}, + {file = "typing_extensions-3.10.0.0-py2-none-any.whl", hash = "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497"}, + {file = "typing_extensions-3.10.0.0-py3-none-any.whl", hash = "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84"}, + {file = "typing_extensions-3.10.0.0.tar.gz", hash = "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342"}, ] uptime = [ {file = "uptime-3.0.1.tar.gz", hash = "sha256:7c300254775b807ce46e3dcbcda30aa3b9a204b9c57a7ac1e79ee6dbe3942973"}, diff --git a/pyproject.toml b/pyproject.toml index 86a0525a32..1a511dce9d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,7 @@ certifi = ">=2020.11.8" chardet = "3.0.4" colorama = "0.3.3" dcicpyvcf = "1.0.0" -dcicsnovault = "4.6.0" +dcicsnovault = "4.7.6" dcicutils = "1.12.0" docutils = "0.12" elasticsearch = "6.8.1" From 403fed9cca8de871802acd813a2aa6486e4a9b68 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 4 May 2021 14:34:13 -0400 Subject: [PATCH 053/120] refactor dockerfile to take entrypoint as build arg, add entrypoints --- deploy/docker/production/Dockerfile | 4 +- deploy/docker/production/Dockerfile_indexer | 100 ------------------ deploy/docker/production/Makefile | 12 ++- deploy/docker/production/entrypoint.sh | 11 -- .../production/entrypoint_deployment.sh | 17 +++ .../docker/production/entrypoint_ingester.sh | 9 ++ 6 files changed, 40 insertions(+), 113 deletions(-) delete mode 100644 deploy/docker/production/Dockerfile_indexer create mode 100644 deploy/docker/production/entrypoint_deployment.sh create mode 100644 deploy/docker/production/entrypoint_ingester.sh diff --git a/deploy/docker/production/Dockerfile b/deploy/docker/production/Dockerfile index 0308a27124..21473e76e6 100644 --- a/deploy/docker/production/Dockerfile +++ b/deploy/docker/production/Dockerfile @@ -10,6 +10,8 @@ ARG CGAP_ENV_NAME ENV CGAP_ENV_NAME=${CGAP_ENV_NAME:-"cgap-mastertest"} ARG CGAP_BRANCH ENV CGAP_BRANCH=${CGAP_BRANCH:-"c4_519"} +ARG ENTRYPOINT +ENV ENTRYPOINT=${ENTRYPOINT:-"entrypoint.sh"} # Configure (global) Env ENV CGAP_REPO=https://github.com/dbmi-bgm/cgap-portal.git @@ -85,7 +87,7 @@ COPY mastertest.ini deploy/ini_files/. RUN touch production.ini RUN chown nginx:nginx production.ini -COPY entrypoint.sh . +COPY $ENTRYPOINT entrypoint.sh COPY assume_identity.py . RUN chmod +x entrypoint.sh RUN chmod +x assume_identity.py diff --git a/deploy/docker/production/Dockerfile_indexer b/deploy/docker/production/Dockerfile_indexer deleted file mode 100644 index ca6495033b..0000000000 --- a/deploy/docker/production/Dockerfile_indexer +++ /dev/null @@ -1,100 +0,0 @@ -# CGAP-Portal Indexer Dockerfile -# TODO: figure out how to manage this better (no duplication) - -# TODO: appropriately pin/verify this image -FROM python:3.6.12-buster - -MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" - -# Build Arguments -ARG CGAP_ENV_NAME -ENV CGAP_ENV_NAME=${CGAP_ENV_NAME:-"cgap-mastertest"} -ARG CGAP_BRANCH -ENV CGAP_BRANCH=${CGAP_BRANCH:-"c4_519"} - -# Configure (global) Env -ENV CGAP_REPO=https://github.com/dbmi-bgm/cgap-portal.git -ENV NGINX_USER=nginx -ENV DEBIAN_FRONTEND=noninteractive -ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1 -ENV PYTHONFAULTHANDLER=1 \ - PYTHONUNBUFFERED=1 \ - PYTHONHASHSEED=random \ - PIP_NO_CACHE_DIR=off \ - PIP_DISABLE_PIP_VERSION_CHECK=on \ - PIP_DEFAULT_TIMEOUT=100 \ - POETRY_VERSION=1.1.4 - -# Install nginx -COPY install_nginx.sh / -RUN bash /install_nginx.sh - -# Intall things needed for our system -RUN apt-get update -RUN apt-get install -y curl vim emacs postgresql-client net-tools -RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - -RUN apt-get install -y ca-certificates nodejs npm - -# Configure CGAP User -WORKDIR /home/nginx -RUN mkdir -p /home/nginx/cgap-portal - -# Configure venv -ENV VIRTUAL_ENV=/opt/venv -RUN python -m venv /opt/venv -ENV PATH="$VIRTUAL_ENV/bin:$PATH" -RUN chown -R nginx:nginx /opt/venv - -# Copy to /home/nginx/cgap-portal -RUN git clone $CGAP_REPO --branch $CGAP_BRANCH - -# Build the application -WORKDIR /home/nginx/cgap-portal -RUN pip install --upgrade pip -RUN pip install poetry==1.1.4 wheel==0.29.0 -RUN poetry install -RUN python setup_eb.py develop -RUN make fix-dist-info -RUN npm ci --no-fund --no-progress --no-optional --no-audit --python=/opt/venv/bin/python -RUN npm run build -RUN npm run build-scss -RUN make aws-ip-ranges -RUN cat /dev/urandom | head -c 256 | base64 > session-secret.b64 - -# Copy config files in (down here for quick debugging) -# Remove default configuration from Nginx -RUN rm /etc/nginx/nginx.conf -RUN rm /etc/nginx/conf.d/default.conf -COPY nginx.conf /etc/nginx/nginx.conf -# give nginx user permissions -RUN chown -R nginx:nginx /var/cache/nginx && \ - chown -R nginx:nginx /var/log/nginx && \ - chown -R nginx:nginx /etc/nginx/conf.d -RUN touch /var/run/nginx.pid && \ - chown -R nginx:nginx /var/run/nginx.pid -RUN rm -f /var/log/nginx/* -RUN touch /var/log/nginx/access.log && \ - chown -R nginx:nginx /var/log/nginx/access.log -RUN touch /var/log/nginx/error.log && \ - chown -R nginx:nginx /var/log/nginx/error.log - -# Provide base ini file -# will be picked up by IniFileManager -# *.ini must match the env name in secrets manager! -# For now, this is mastertest. - Will 04/29/21 -COPY mastertest.ini deploy/ini_files/. -RUN touch production.ini -RUN chown nginx:nginx production.ini - -COPY entrypoint.sh . -COPY entrypoint_indexer.sh entrypoint.sh -COPY assume_identity.py . -RUN chmod +x entrypoint.sh -RUN chmod +x assume_identity.py -EXPOSE 8000 - -# Container does not run as root -USER nginx -WORKDIR /home/nginx/cgap-portal - -ENTRYPOINT ["/home/nginx/cgap-portal/entrypoint.sh"] diff --git a/deploy/docker/production/Makefile b/deploy/docker/production/Makefile index d7b74c3d6a..57d28b5b00 100644 --- a/deploy/docker/production/Makefile +++ b/deploy/docker/production/Makefile @@ -7,6 +7,16 @@ wsgi: docker push 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest indexer: - docker build -t cgap-mastertest:latest-indexer -f Dockerfile_indexer . + docker build -t cgap-mastertest:latest-indexer -f Dockerfile . --build-arg ENTRYPOINT=entrypoint_indexer.sh docker tag cgap-mastertest:latest-indexer 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-indexer docker push 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-indexer + +ingester: + docker build -t cgap-mastertest:latest-ingester -f Dockerfile . --build-arg ENTRYPOINT=entrypoint_ingester.sh + docker tag cgap-mastertest:latest-ingester 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-ingester + docker push 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-ingester + +deployment: + docker build -t cgap-mastertest:latest-deployment -f Dockerfile . --build-arg ENTRYPOINT=entrypoint_deployment.sh + docker tag cgap-mastertest:latest-deployment 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-deployment + docker push 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-deployment diff --git a/deploy/docker/production/entrypoint.sh b/deploy/docker/production/entrypoint.sh index 1d0589c5e3..a974b8f407 100644 --- a/deploy/docker/production/entrypoint.sh +++ b/deploy/docker/production/entrypoint.sh @@ -6,19 +6,8 @@ echo "Starting up CGAP-Portal WSGI" # secrets manager - this builds production.ini poetry run python -m assume_identity -# XXX: this is deployment stuff, doesn't need to be run at this time -## Clear db/es since this is the local entry point -#poetry run clear-db-es-contents production.ini --app-name app --env $CGAP_ENV_NAME -# -## Create mapping -#poetry run create-mapping-on-deploy production.ini --app-name app -# -## Load Data (based on development.ini, for now just master-inserts) -#poetry run load-data production.ini --app-name app --prod - # Start nginx proxy service nginx start # Start application pserve production.ini - diff --git a/deploy/docker/production/entrypoint_deployment.sh b/deploy/docker/production/entrypoint_deployment.sh new file mode 100644 index 0000000000..343dbe338e --- /dev/null +++ b/deploy/docker/production/entrypoint_deployment.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +echo "Running a CGAP deployment on the given environment" + +# Run assume_identity.py to access the desired deployment configuration from +# secrets manager - this builds production.ini +poetry run python -m assume_identity + +# XXX: this is deployment stuff, doesn't need to be run at this time +## Clear db/es since this is the local entry point +poetry run clear-db-es-contents production.ini --app-name app --env $CGAP_ENV_NAME +# +## Create mapping +poetry run create-mapping-on-deploy production.ini --app-name app +# +## Load Data (based on development.ini, for now just master-inserts) +poetry run load-data production.ini --app-name app --prod diff --git a/deploy/docker/production/entrypoint_ingester.sh b/deploy/docker/production/entrypoint_ingester.sh new file mode 100644 index 0000000000..ee2e9855a5 --- /dev/null +++ b/deploy/docker/production/entrypoint_ingester.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +echo "Starting CGAP-Portal Ingester" + +# Run assume_identity.py to access the desired deployment configuration from +# secrets manager - this builds production.ini +poetry run python -m assume_identity + +poetry run ingestion-listener production.ini --app-name app From 4c8d779c9955d841d7d6d5cf92208dba0a98ec7e Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 4 May 2021 14:54:34 -0400 Subject: [PATCH 054/120] change indexer mode --- deploy/docker/production/entrypoint_indexer.sh | 7 +++++-- pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/deploy/docker/production/entrypoint_indexer.sh b/deploy/docker/production/entrypoint_indexer.sh index e221c8b8ae..a4c280bb90 100644 --- a/deploy/docker/production/entrypoint_indexer.sh +++ b/deploy/docker/production/entrypoint_indexer.sh @@ -6,5 +6,8 @@ echo "Starting up CGAP-Portal Indexer" # secrets manager - this builds production.ini poetry run python -m assume_identity -# Start indexer listener -es-index-listener production.ini --app-name app --verbose +# Start indexer, do 20 runs +for i in {1..20}; do + poetry run es-index-data production.ini --app-name app --verbose + sleep 1 +done diff --git a/pyproject.toml b/pyproject.toml index 1a511dce9d..fae082595d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -161,7 +161,7 @@ pytest-xdist = ">=1.14" # snovault commands batchupgrade = "snovault.batchupgrade:main" create-mapping = "snovault.elasticsearch.create_mapping:main" -es-index-listener = "snovault.elasticsearch.es_index_listener:main" +es-index-data = "snovault.commands.es_index_data:main" wipe-test-indices = "snovault.commands.wipe_test_indices:main" # encoded commands add-date-created = "encoded.commands.add_date_created:main" From 111987e765f385af28177119dfeb56a613b0a62a Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 4 May 2021 15:00:35 -0400 Subject: [PATCH 055/120] fix cmd declaration --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fae082595d..52d1f783a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -161,7 +161,6 @@ pytest-xdist = ">=1.14" # snovault commands batchupgrade = "snovault.batchupgrade:main" create-mapping = "snovault.elasticsearch.create_mapping:main" -es-index-data = "snovault.commands.es_index_data:main" wipe-test-indices = "snovault.commands.wipe_test_indices:main" # encoded commands add-date-created = "encoded.commands.add_date_created:main" From c01bd4140d622d610ba1cf8faf275aa2c626c043 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 4 May 2021 16:33:43 -0400 Subject: [PATCH 056/120] fix index path --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 52d1f783a9..9b6f854c03 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -170,7 +170,7 @@ configure-kibana-index = "encoded.commands.configure_kibana_index:main" create-mapping-on-deploy = "encoded.commands.create_mapping_on_deploy:main" dev-servers = "encoded.dev_servers:main" dis2pheno = "encoded.commands.parse_hpoa:main" -es-index-data = "encoded.commands.es_index_data:main" +es-index-data = "snovault.commands.es_index_data:main" export-data = "encoded.commands.export_data:main" extract-test-data = "encoded.commands.extract_test_data:main" import-data = "encoded.commands.import_data:main" From 6d9269e1137148b677ec77d5f1dc33572c3717a2 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Wed, 5 May 2021 09:06:59 -0400 Subject: [PATCH 057/120] remove verbose arg --- deploy/docker/production/entrypoint_indexer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/docker/production/entrypoint_indexer.sh b/deploy/docker/production/entrypoint_indexer.sh index a4c280bb90..38f044674e 100644 --- a/deploy/docker/production/entrypoint_indexer.sh +++ b/deploy/docker/production/entrypoint_indexer.sh @@ -8,6 +8,6 @@ poetry run python -m assume_identity # Start indexer, do 20 runs for i in {1..20}; do - poetry run es-index-data production.ini --app-name app --verbose + poetry run es-index-data production.ini --app-name app sleep 1 done From b3bcfb90e4f9041e473d93d6ab8946891b84cd7b Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Wed, 5 May 2021 10:52:15 -0400 Subject: [PATCH 058/120] small changes to deployment --- deploy/docker/production/entrypoint_deployment.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/deploy/docker/production/entrypoint_deployment.sh b/deploy/docker/production/entrypoint_deployment.sh index 343dbe338e..b4f3a8850e 100644 --- a/deploy/docker/production/entrypoint_deployment.sh +++ b/deploy/docker/production/entrypoint_deployment.sh @@ -6,12 +6,14 @@ echo "Running a CGAP deployment on the given environment" # secrets manager - this builds production.ini poetry run python -m assume_identity -# XXX: this is deployment stuff, doesn't need to be run at this time -## Clear db/es since this is the local entry point +# Clear db/es since this is the local entry point poetry run clear-db-es-contents production.ini --app-name app --env $CGAP_ENV_NAME -# + ## Create mapping poetry run create-mapping-on-deploy production.ini --app-name app -# -## Load Data (based on development.ini, for now just master-inserts) + +# Load Data (based on development.ini, for now just master-inserts) poetry run load-data production.ini --app-name app --prod + +# Load access keys +poetry run load-access-keys production.ini --app-name app From 4436192728bc590c5348ec3ba4b4b874c0667347 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 6 May 2021 09:59:55 -0400 Subject: [PATCH 059/120] make load_access_keys acquire s3 key from secret --- .../production/entrypoint_deployment.sh | 3 +- src/encoded/commands/load_access_keys.py | 53 +++++++++++++++++-- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/deploy/docker/production/entrypoint_deployment.sh b/deploy/docker/production/entrypoint_deployment.sh index b4f3a8850e..121ddcd256 100644 --- a/deploy/docker/production/entrypoint_deployment.sh +++ b/deploy/docker/production/entrypoint_deployment.sh @@ -16,4 +16,5 @@ poetry run create-mapping-on-deploy production.ini --app-name app poetry run load-data production.ini --app-name app --prod # Load access keys -poetry run load-access-keys production.ini --app-name app +# Note that the secret name must match that which was created for this environment +poetry run load-access-keys production.ini --app-name app --secret-name dev/beanstalk/cgap-dev diff --git a/src/encoded/commands/load_access_keys.py b/src/encoded/commands/load_access_keys.py index 3d365718ba..0f2cc46798 100644 --- a/src/encoded/commands/load_access_keys.py +++ b/src/encoded/commands/load_access_keys.py @@ -6,8 +6,9 @@ import boto3 from pyramid.paster import get_app from webtest import AppError +from botocore.exceptions import ClientError from dcicutils.misc_utils import TestApp -from dcicutils.beanstalk_utils import get_beanstalk_real_url +from dcicutils.beanstalk_utils import get_beanstalk_real_url, REGION log = structlog.getLogger(__name__) EPILOG = __doc__ @@ -84,6 +85,8 @@ def main(): ) parser.add_argument('config_uri', help='path to configfile') parser.add_argument('--app-name', help='Pyramid app name in configfile') + parser.add_argument('--secret-name', help='name of application identity stored in secrets manager within which' + 'to locate S3_ENCRYPT_KEY, for example: dev/beanstalk/cgap-dev') args = parser.parse_args() app = get_app(args.config_uri, args.app_name) @@ -97,9 +100,53 @@ def main(): if not env: raise RuntimeError('load_access_keys: cannot find env.name in settings') - encrypt_key = os.environ.get('S3_ENCRYPT_KEY') + # XXX: refactor into helper method in dcicutils, see assume_identity.py + if args.secret_name is not None: + secret_name = args.secret_name + region_name = REGION # us-east-1 + + # XXX: We should refactor a SecretsManager wrapper into dcicutils + session = boto3.session.Session(region_name=region_name) + # configure watchtower handler from session + client = session.client( + service_name='secretsmanager', + region_name=region_name + ) + + try: + get_secret_value_response = client.get_secret_value( + SecretId=secret_name + ) + except ClientError as e: # leaving some useful debug info to help narrow issues + if e.response['Error']['Code'] == 'DecryptionFailureException': + # Secrets Manager can't decrypt the protected secret text using the provided KMS key. + raise e + elif e.response['Error']['Code'] == 'InternalServiceErrorException': + # An error occurred on the server side. + raise e + elif e.response['Error']['Code'] == 'InvalidParameterException': + # You provided an invalid value for a parameter. + raise e + elif e.response['Error']['Code'] == 'InvalidRequestException': + # You provided a parameter value that is not valid for the current state of the resource. + raise e + elif e.response['Error']['Code'] == 'ResourceNotFoundException': + raise e + else: + raise e + else: + # Decrypts secret using the associated KMS CMK. + # Depending on whether the secret is a string or binary, one of these fields will be populated. + if 'SecretString' in get_secret_value_response: + identity = json.loads(get_secret_value_response['SecretString']) + encrypt_key = identity.get('S3_ENCRYPT_KEY', None) + else: + raise Exception('Got unexpected response structure from boto3') + else: + encrypt_key = os.environ.get('S3_ENCRYPT_KEY') + if not encrypt_key: - raise RuntimeError('load_access_keys: must define S3_ENCRYPT_KEY in env') + raise RuntimeError('load_access_keys: must define S3_ENCRYPT_KEY in env or in passed secret') # will need to use a dynamic region at some point (not just here) s3 = boto3.client('s3', region_name='us-east-1') From e81f41a2ecce075fc2be077eb2cc00e01d95ccdc Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 6 May 2021 14:44:04 -0400 Subject: [PATCH 060/120] write get_ecs_real_url --- deploy/docker/production/Dockerfile | 6 ++++-- src/encoded/commands/load_access_keys.py | 18 ++++++++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/deploy/docker/production/Dockerfile b/deploy/docker/production/Dockerfile index 21473e76e6..1a9fb3adff 100644 --- a/deploy/docker/production/Dockerfile +++ b/deploy/docker/production/Dockerfile @@ -1,7 +1,9 @@ # CGAP-Portal Dockerfile +# Note that images are pinned via sha256 as opposed to tag +# so that we don't pick up new images unintentionally -# TODO: appropriately pin/verify this image -FROM python:3.6.12-buster +# Debian Buster with Python 3.6.13 +FROM python@sha256:db248d2d0494973550d323dd6b82af7fc2f4c1e0365769a758abd7fac2aa70db MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" diff --git a/src/encoded/commands/load_access_keys.py b/src/encoded/commands/load_access_keys.py index 0f2cc46798..da980e6f01 100644 --- a/src/encoded/commands/load_access_keys.py +++ b/src/encoded/commands/load_access_keys.py @@ -43,9 +43,23 @@ def get_existing_key_ids(testapp, user_uuid, key_desc): return [res['@id'] for res in search_res['@graph']] +def get_ecs_real_url(): + """ Inspects Cloudformation stacks, looking for LB URL + TODO: pull into dcicutils + """ + cfn_client = boto3.client('cloudformation') + stacks = cfn_client.describe_stacks().get('Stacks', []) + for stack in stacks: + for output in stack['Outputs']: + if output.get('OutputKey', '') == 'ECSApplicationURL': # TODO remove hard-code + return output.get('OutputValue') + log.error('Did not locate the server from Cloudformation! Check ECS Stack metadata.') + return '' + + def generate_access_key(testapp, env, user_uuid, description): """ - Generate an access key for given user on given environment. + Generate an access for given user on given environment. Args: testapp (webtest.TestApp): current TestApp @@ -56,7 +70,7 @@ def generate_access_key(testapp, env, user_uuid, description): Returns: dict: access key contents with server """ - server = get_beanstalk_real_url(env) + server = get_ecs_real_url() # INCOMPATIBLE CHANGE; will break beanstalk -Will 5/6/21 access_key_req = {'user': user_uuid, 'description': description} res = testapp.post_json('/access_key', access_key_req).json return {'secret': res['secret_access_key'], From 85b79bcccd6b3e0dcd36a24f060a148a64734aea Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 6 May 2021 14:50:41 -0400 Subject: [PATCH 061/120] namespace ECS url --- src/encoded/commands/load_access_keys.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/encoded/commands/load_access_keys.py b/src/encoded/commands/load_access_keys.py index da980e6f01..2eb833df36 100644 --- a/src/encoded/commands/load_access_keys.py +++ b/src/encoded/commands/load_access_keys.py @@ -43,7 +43,7 @@ def get_existing_key_ids(testapp, user_uuid, key_desc): return [res['@id'] for res in search_res['@graph']] -def get_ecs_real_url(): +def get_ecs_real_url(env): """ Inspects Cloudformation stacks, looking for LB URL TODO: pull into dcicutils """ @@ -51,7 +51,7 @@ def get_ecs_real_url(): stacks = cfn_client.describe_stacks().get('Stacks', []) for stack in stacks: for output in stack['Outputs']: - if output.get('OutputKey', '') == 'ECSApplicationURL': # TODO remove hard-code + if output.get('OutputKey', '') == ('ECSApplicationURL%s' % env): return output.get('OutputValue') log.error('Did not locate the server from Cloudformation! Check ECS Stack metadata.') return '' @@ -70,7 +70,7 @@ def generate_access_key(testapp, env, user_uuid, description): Returns: dict: access key contents with server """ - server = get_ecs_real_url() # INCOMPATIBLE CHANGE; will break beanstalk -Will 5/6/21 + server = get_ecs_real_url(env) # INCOMPATIBLE CHANGE; will break beanstalk -Will 5/6/21 access_key_req = {'user': user_uuid, 'description': description} res = testapp.post_json('/access_key', access_key_req).json return {'secret': res['secret_access_key'], From 9de4497b895c20ff6d7816a262fe2fc57edc98fd Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 6 May 2021 14:52:28 -0400 Subject: [PATCH 062/120] remove - from env --- src/encoded/commands/load_access_keys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/encoded/commands/load_access_keys.py b/src/encoded/commands/load_access_keys.py index 2eb833df36..f7ca5cc2fd 100644 --- a/src/encoded/commands/load_access_keys.py +++ b/src/encoded/commands/load_access_keys.py @@ -51,7 +51,7 @@ def get_ecs_real_url(env): stacks = cfn_client.describe_stacks().get('Stacks', []) for stack in stacks: for output in stack['Outputs']: - if output.get('OutputKey', '') == ('ECSApplicationURL%s' % env): + if output.get('OutputKey', '') == ('ECSApplicationURL%s' % env.replace('-', '')): return output.get('OutputValue') log.error('Did not locate the server from Cloudformation! Check ECS Stack metadata.') return '' From 0d8210b37eaad50491a1cdc9a058bee7058b46a0 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Fri, 7 May 2021 10:36:34 -0400 Subject: [PATCH 063/120] skip access key integrated test for now --- src/encoded/tests/test_load_access_key.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/encoded/tests/test_load_access_key.py b/src/encoded/tests/test_load_access_key.py index 5d727b9a54..e935ca84b7 100644 --- a/src/encoded/tests/test_load_access_key.py +++ b/src/encoded/tests/test_load_access_key.py @@ -7,9 +7,8 @@ pytestmark = [pytest.mark.setone, pytest.mark.working] -# TODO: test load_access_keys.get_existing_key_ids, which would use ES - - +# TODO: Re-enable once running ECS in production +@pytest.mark.skip def test_gen_access_keys(testapp, admin): with mock.patch.object(load_access_keys, 'get_beanstalk_real_url') as mocked_url: mocked_url.return_value = 'http://fourfront-hotseat' From 147c6b2801cf360aeec163cff55cd6dfb1b6082d Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Mon, 10 May 2021 15:27:29 -0400 Subject: [PATCH 064/120] configure load_test_data for deployment, refactor dockerfile to organize layers better --- deploy/docker/production/Dockerfile | 78 ++++++++++++------------- deploy/docker/production/mastertest.ini | 2 +- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/deploy/docker/production/Dockerfile b/deploy/docker/production/Dockerfile index 1a9fb3adff..0a080cdd63 100644 --- a/deploy/docker/production/Dockerfile +++ b/deploy/docker/production/Dockerfile @@ -10,13 +10,14 @@ MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" # Build Arguments ARG CGAP_ENV_NAME ENV CGAP_ENV_NAME=${CGAP_ENV_NAME:-"cgap-mastertest"} +ARG CGAP_REPO +ENV CGAP_REPO=${CGAP_REPO:-"https://github.com/dbmi-bgm/cgap-portal.git"} ARG CGAP_BRANCH ENV CGAP_BRANCH=${CGAP_BRANCH:-"c4_519"} ARG ENTRYPOINT ENV ENTRYPOINT=${ENTRYPOINT:-"entrypoint.sh"} # Configure (global) Env -ENV CGAP_REPO=https://github.com/dbmi-bgm/cgap-portal.git ENV NGINX_USER=nginx ENV DEBIAN_FRONTEND=noninteractive ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1 @@ -28,58 +29,58 @@ ENV PYTHONFAULTHANDLER=1 \ PIP_DEFAULT_TIMEOUT=100 \ POETRY_VERSION=1.1.4 -# Install nginx +# Install nginx, base system COPY install_nginx.sh / -RUN bash /install_nginx.sh +RUN bash /install_nginx.sh && \ + apt-get update && \ + apt-get install -y curl vim emacs postgresql-client net-tools && \ + curl -sL https://deb.nodesource.com/setup_10.x | bash - && \ + apt-get install -y ca-certificates nodejs npm -# Intall things needed for our system -RUN apt-get update -RUN apt-get install -y curl vim emacs postgresql-client net-tools -RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - -RUN apt-get install -y ca-certificates nodejs npm - -# Configure CGAP User +# Configure CGAP User (nginx) WORKDIR /home/nginx -RUN mkdir -p /home/nginx/cgap-portal -# Configure venv +# Configure venv, repo, back-end build ENV VIRTUAL_ENV=/opt/venv RUN python -m venv /opt/venv ENV PATH="$VIRTUAL_ENV/bin:$PATH" -RUN chown -R nginx:nginx /opt/venv - -# Copy to /home/nginx/cgap-portal -RUN git clone $CGAP_REPO --branch $CGAP_BRANCH +RUN chown -R nginx:nginx /opt/venv && \ + mkdir -p /home/nginx/cgap-portal && \ + git clone $CGAP_REPO --branch $CGAP_BRANCH && \ + cd cgap-portal && \ + pip install --upgrade pip && \ + pip install poetry==1.1.4 wheel==0.29.0 && \ + poetry install && \ + python setup_eb.py develop && \ + make fix-dist-info -# Build the application +# Front-end WORKDIR /home/nginx/cgap-portal -RUN pip install --upgrade pip -RUN pip install poetry==1.1.4 wheel==0.29.0 -RUN poetry install -RUN python setup_eb.py develop -RUN make fix-dist-info -RUN npm ci --no-fund --no-progress --no-optional --no-audit --python=/opt/venv/bin/python -RUN npm run build -RUN npm run build-scss -RUN make aws-ip-ranges -RUN cat /dev/urandom | head -c 256 | base64 > session-secret.b64 +RUN npm ci --no-fund --no-progress --no-optional --no-audit --python=/opt/venv/bin/python && \ + npm run build && \ + npm run build-scss + +# Misc +RUN make aws-ip-ranges && \ + cat /dev/urandom | head -c 256 | base64 > session-secret.b64 # Copy config files in (down here for quick debugging) # Remove default configuration from Nginx -RUN rm /etc/nginx/nginx.conf -RUN rm /etc/nginx/conf.d/default.conf +RUN rm /etc/nginx/nginx.conf && \ + rm /etc/nginx/conf.d/default.conf COPY nginx.conf /etc/nginx/nginx.conf -# give nginx user permissions + +# nginx filesystem setup RUN chown -R nginx:nginx /var/cache/nginx && \ chown -R nginx:nginx /var/log/nginx && \ - chown -R nginx:nginx /etc/nginx/conf.d -RUN touch /var/run/nginx.pid && \ - chown -R nginx:nginx /var/run/nginx.pid -RUN rm -f /var/log/nginx/* -RUN touch /var/log/nginx/access.log && \ - chown -R nginx:nginx /var/log/nginx/access.log -RUN touch /var/log/nginx/error.log && \ - chown -R nginx:nginx /var/log/nginx/error.log + chown -R nginx:nginx /etc/nginx/conf.d && \ + touch /var/run/nginx.pid && \ + chown -R nginx:nginx /var/run/nginx.pid && \ + rm -f /var/log/nginx/* && \ + touch /var/log/nginx/access.log && \ + chown -R nginx:nginx /var/log/nginx/access.log && \ + touch /var/log/nginx/error.log && \ + chown -R nginx:nginx /var/log/nginx/error.log # Provide base ini file # will be picked up by IniFileManager @@ -97,6 +98,5 @@ EXPOSE 8000 # Container does not run as root USER nginx -WORKDIR /home/nginx/cgap-portal ENTRYPOINT ["/home/nginx/cgap-portal/entrypoint.sh"] diff --git a/deploy/docker/production/mastertest.ini b/deploy/docker/production/mastertest.ini index ab8db883ef..639e3de91e 100644 --- a/deploy/docker/production/mastertest.ini +++ b/deploy/docker/production/mastertest.ini @@ -21,7 +21,7 @@ indexer.namespace = cgap-mastertest index_server = ${INDEX_SERVER} elasticsearch.aws_auth = true production = true -load_test_data = encoded.loadxl:load_prod_data +load_test_data = encoded.loadxl:load_test_data sqlalchemy.url = postgresql://${RDS_USERNAME}:${RDS_PASSWORD}@${RDS_HOSTNAME}:${RDS_PORT}/${RDS_DB_NAME} [composite:indexer] From 0bbdc5f06fbf63d65c91dbd99cffeb38f8774b39 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 11 May 2021 12:26:16 -0400 Subject: [PATCH 065/120] small changes --- deploy/docker/production/entrypoint_indexer.sh | 7 +++++-- deploy/docker/production/mastertest.ini | 1 + src/encoded/ingestion_listener.py | 5 +++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/deploy/docker/production/entrypoint_indexer.sh b/deploy/docker/production/entrypoint_indexer.sh index 38f044674e..913f990a67 100644 --- a/deploy/docker/production/entrypoint_indexer.sh +++ b/deploy/docker/production/entrypoint_indexer.sh @@ -7,7 +7,10 @@ echo "Starting up CGAP-Portal Indexer" poetry run python -m assume_identity # Start indexer, do 20 runs -for i in {1..20}; do +i=0 +while [ $i -ne 20 ] +do + i=$(($i + 1)) poetry run es-index-data production.ini --app-name app - sleep 1 + sleep 3 done diff --git a/deploy/docker/production/mastertest.ini b/deploy/docker/production/mastertest.ini index 639e3de91e..2711e88202 100644 --- a/deploy/docker/production/mastertest.ini +++ b/deploy/docker/production/mastertest.ini @@ -1,4 +1,5 @@ [app:app] +beanstalk_app_version = c4_519 use = config:base.ini#app session.secret = %(here)s/session-secret.b64 file_upload_bucket = ${ENCODED_FILES_BUCKET} diff --git a/src/encoded/ingestion_listener.py b/src/encoded/ingestion_listener.py index 3a2c5a9312..2bb7c6c94b 100644 --- a/src/encoded/ingestion_listener.py +++ b/src/encoded/ingestion_listener.py @@ -31,7 +31,7 @@ from .ingestion.vcf_utils import VCFParser from .commands.reformat_vcf import runner as reformat_vcf from .commands.add_altcounts_by_gene import main as add_altcounts -from .ingestion.common import metadata_bundles_bucket, get_parameter # , IngestionReport +from .ingestion.common import metadata_bundles_bucket, get_parameter, IngestionReport from .ingestion.exceptions import UnspecifiedFormParameter, SubmissionFailure # , BadParameter from .ingestion.processors import get_ingestion_processor # from .types.base import get_item_or_none @@ -579,7 +579,8 @@ def discard(msg): # report results in error_log regardless of status msg = variant_builder.ingestion_report.brief_summary() log.error(msg) - self.update_status(msg=msg) + if self.update_status is not None and callable(self.update_status): + self.update_status(msg=msg) # if we had no errors, patch the file status to 'Ingested' if error > 0: From c0de63d18a137fa973744e58b36f562409eb82f2 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 13 May 2021 15:17:41 -0400 Subject: [PATCH 066/120] small changes --- deploy/docker/local/Dockerfile | 94 ++++++++++--------- .../production/entrypoint_deployment.sh | 2 + .../docker/production/entrypoint_indexer.sh | 8 +- .../docker/production/entrypoint_ingester.sh | 1 + .../commands/update_inserts_from_server.py | 1 - 5 files changed, 59 insertions(+), 47 deletions(-) diff --git a/deploy/docker/local/Dockerfile b/deploy/docker/local/Dockerfile index 992a1927df..0e510dbf3d 100644 --- a/deploy/docker/local/Dockerfile +++ b/deploy/docker/local/Dockerfile @@ -5,16 +5,26 @@ FROM python:3.6.12-buster MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" +# CGAP-Portal Dockerfile +# Note that images are pinned via sha256 as opposed to tag +# so that we don't pick up new images unintentionally + +# Debian Buster with Python 3.6.13 +FROM python@sha256:db248d2d0494973550d323dd6b82af7fc2f4c1e0365769a758abd7fac2aa70db + +MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" + # Build Arguments ARG CGAP_ENV_NAME -ENV CGAP_ENV_NAME=${CGAP_ENV_NAME:-"cgap-docker-will-test"} -# Repair the below after this is merged -#ARG CGAP_BRANCH -#ENV CGAP_BRANCH=${CGAP_BRANCH:-"master"} -ENV CGAP_BRANCH=c4_519 +ENV CGAP_ENV_NAME=${CGAP_ENV_NAME:-"cgap-mastertest"} +ARG CGAP_REPO +ENV CGAP_REPO=${CGAP_REPO:-"https://github.com/dbmi-bgm/cgap-portal.git"} +ARG CGAP_BRANCH +ENV CGAP_BRANCH=${CGAP_BRANCH:-"c4_519"} +ARG ENTRYPOINT +ENV ENTRYPOINT=${ENTRYPOINT:-"entrypoint.sh"} # Configure (global) Env -ENV CGAP_REPO=https://github.com/dbmi-bgm/cgap-portal.git ENV NGINX_USER=nginx ENV DEBIAN_FRONTEND=noninteractive ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1 @@ -26,57 +36,57 @@ ENV PYTHONFAULTHANDLER=1 \ PIP_DEFAULT_TIMEOUT=100 \ POETRY_VERSION=1.1.4 -# Install nginx +# Install nginx, base system COPY install_nginx.sh / -RUN bash /install_nginx.sh +RUN bash /install_nginx.sh && \ + apt-get update && \ + apt-get install -y curl vim emacs postgresql-client net-tools && \ + curl -sL https://deb.nodesource.com/setup_10.x | bash - && \ + apt-get install -y ca-certificates nodejs npm -# Install things needed for our system -RUN apt-get update -RUN apt-get install -y curl vim emacs postgresql-client net-tools -RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - -RUN apt-get install -y ca-certificates nodejs npm - -# Configure CGAP User +# Configure CGAP User (nginx) WORKDIR /home/nginx -RUN mkdir -p /home/nginx/cgap-portal -# Configure venv +# Configure venv, repo, back-end build ENV VIRTUAL_ENV=/opt/venv RUN python -m venv /opt/venv ENV PATH="$VIRTUAL_ENV/bin:$PATH" -RUN chown -R nginx:nginx /opt/venv - -# Copy to /home/nginx/cgap-portal -RUN git clone $CGAP_REPO --branch $CGAP_BRANCH +RUN chown -R nginx:nginx /opt/venv && \ + mkdir -p /home/nginx/cgap-portal && \ + git clone $CGAP_REPO --branch $CGAP_BRANCH && \ + cd cgap-portal && \ + pip install --upgrade pip && \ + pip install poetry==1.1.4 wheel==0.29.0 && \ + poetry install && \ + python setup_eb.py develop && \ + make fix-dist-info -# Build the application +# Front-end WORKDIR /home/nginx/cgap-portal -RUN pip install --upgrade pip -RUN pip install poetry==1.1.4 wheel==0.29.0 -RUN poetry install -RUN python setup_eb.py develop -RUN make fix-dist-info -RUN npm ci --no-fund --no-progress --python=/opt/venv/bin/python -RUN npm run build -RUN npm run build-scss -RUN make aws-ip-ranges -RUN cat /dev/urandom | head -c 256 | base64 > session-secret.b64 +RUN npm ci --no-fund --no-progress --no-optional --no-audit --python=/opt/venv/bin/python && \ + npm run build && \ + npm run build-scss + +# Misc +RUN make aws-ip-ranges && \ + cat /dev/urandom | head -c 256 | base64 > session-secret.b64 # Copy config files in (down here for quick debugging) # Remove default configuration from Nginx -RUN rm /etc/nginx/nginx.conf -RUN rm /etc/nginx/conf.d/default.conf +RUN rm /etc/nginx/nginx.conf && \ + rm /etc/nginx/conf.d/default.conf COPY nginx.conf /etc/nginx/nginx.conf -# give nginx user permissions + +# nginx filesystem setup RUN chown -R nginx:nginx /var/cache/nginx && \ chown -R nginx:nginx /var/log/nginx && \ - chown -R nginx:nginx /etc/nginx/conf.d -RUN touch /var/run/nginx.pid && \ - chown -R nginx:nginx /var/run/nginx.pid -RUN rm -f /var/log/nginx/* -RUN touch /var/log/nginx/access.log && \ - chown -R nginx:nginx /var/log/nginx/access.log -RUN touch /var/log/nginx/error.log && \ + chown -R nginx:nginx /etc/nginx/conf.d && \ + touch /var/run/nginx.pid && \ + chown -R nginx:nginx /var/run/nginx.pid && \ + rm -f /var/log/nginx/* && \ + touch /var/log/nginx/access.log && \ + chown -R nginx:nginx /var/log/nginx/access.log && \ + touch /var/log/nginx/error.log && \ chown -R nginx:nginx /var/log/nginx/error.log # Copy over ini file, entrypoint diff --git a/deploy/docker/production/entrypoint_deployment.sh b/deploy/docker/production/entrypoint_deployment.sh index 121ddcd256..089f9c8706 100644 --- a/deploy/docker/production/entrypoint_deployment.sh +++ b/deploy/docker/production/entrypoint_deployment.sh @@ -18,3 +18,5 @@ poetry run load-data production.ini --app-name app --prod # Load access keys # Note that the secret name must match that which was created for this environment poetry run load-access-keys production.ini --app-name app --secret-name dev/beanstalk/cgap-dev + +exit 0 diff --git a/deploy/docker/production/entrypoint_indexer.sh b/deploy/docker/production/entrypoint_indexer.sh index 913f990a67..5bd40c6345 100644 --- a/deploy/docker/production/entrypoint_indexer.sh +++ b/deploy/docker/production/entrypoint_indexer.sh @@ -6,11 +6,11 @@ echo "Starting up CGAP-Portal Indexer" # secrets manager - this builds production.ini poetry run python -m assume_identity -# Start indexer, do 20 runs -i=0 -while [ $i -ne 20 ] +# Start indexer, run forever +while true do - i=$(($i + 1)) poetry run es-index-data production.ini --app-name app sleep 3 done + +exit 0 diff --git a/deploy/docker/production/entrypoint_ingester.sh b/deploy/docker/production/entrypoint_ingester.sh index ee2e9855a5..4f506e290f 100644 --- a/deploy/docker/production/entrypoint_ingester.sh +++ b/deploy/docker/production/entrypoint_ingester.sh @@ -6,4 +6,5 @@ echo "Starting CGAP-Portal Ingester" # secrets manager - this builds production.ini poetry run python -m assume_identity +# will serve forever poetry run ingestion-listener production.ini --app-name app diff --git a/src/encoded/commands/update_inserts_from_server.py b/src/encoded/commands/update_inserts_from_server.py index e6c8622c8e..64a41bfa65 100644 --- a/src/encoded/commands/update_inserts_from_server.py +++ b/src/encoded/commands/update_inserts_from_server.py @@ -163,6 +163,5 @@ def main(): (len(svr_inserts[item_type]), item_type + '.json')) - if __name__ == "__main__": main() From 7702e05c5f4b5e0813b24a7bd8872787b6a543b5 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Fri, 14 May 2021 08:23:43 -0400 Subject: [PATCH 067/120] small changes --- deploy/docker/production/entrypoint_indexer.sh | 2 +- src/encoded/ingestion/vcf_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/docker/production/entrypoint_indexer.sh b/deploy/docker/production/entrypoint_indexer.sh index 5bd40c6345..08d11d1d0b 100644 --- a/deploy/docker/production/entrypoint_indexer.sh +++ b/deploy/docker/production/entrypoint_indexer.sh @@ -9,7 +9,7 @@ poetry run python -m assume_identity # Start indexer, run forever while true do - poetry run es-index-data production.ini --app-name app + poetry run es-index-data production.ini --app-name app || echo "Indexing Runtime Error thrown - check previous output" sleep 3 done diff --git a/src/encoded/ingestion/vcf_utils.py b/src/encoded/ingestion/vcf_utils.py index 67d7e2a9cf..96b89dd79b 100644 --- a/src/encoded/ingestion/vcf_utils.py +++ b/src/encoded/ingestion/vcf_utils.py @@ -342,7 +342,7 @@ def cast_field_value(self, t, value, sub_type=None): else: raise VCFParserException('Got array with no sub-type') else: - raise VCFParserException('Type was %s and not one of: string, integer, number, boolean, array' % type) + raise VCFParserException('Type was %s and not one of: string, integer, number, boolean, array' % t) def validate_variant_value(self, field, value, key='', exit_on_validation=False): """ Given a field, check the variant schema for the type of that field and cast From 76ccfbc53990c107f1846afb8b6a9301dc288904 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Fri, 14 May 2021 08:28:27 -0400 Subject: [PATCH 068/120] fix spacing --- deploy/docker/production/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/docker/production/Makefile b/deploy/docker/production/Makefile index 57d28b5b00..fd6054af40 100644 --- a/deploy/docker/production/Makefile +++ b/deploy/docker/production/Makefile @@ -9,14 +9,14 @@ wsgi: indexer: docker build -t cgap-mastertest:latest-indexer -f Dockerfile . --build-arg ENTRYPOINT=entrypoint_indexer.sh docker tag cgap-mastertest:latest-indexer 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-indexer - docker push 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-indexer + docker push 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-indexer ingester: docker build -t cgap-mastertest:latest-ingester -f Dockerfile . --build-arg ENTRYPOINT=entrypoint_ingester.sh docker tag cgap-mastertest:latest-ingester 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-ingester - docker push 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-ingester + docker push 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-ingester deployment: docker build -t cgap-mastertest:latest-deployment -f Dockerfile . --build-arg ENTRYPOINT=entrypoint_deployment.sh docker tag cgap-mastertest:latest-deployment 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-deployment - docker push 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-deployment + docker push 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-deployment From 7b6360289b8044d63fc5ddeb02ee143f0b949c67 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Mon, 17 May 2021 15:47:10 -0400 Subject: [PATCH 069/120] cleanup some things, start documentation --- deploy/docker/production/docker-compose.yml | 13 -------- deploy/docker/production/nginx.conf | 2 +- docs/source/docker-local.rst | 16 ++++++--- docs/source/docker-production.rst | 37 +++++++++++++++++++++ src/encoded/ingestion/queue_utils.py | 2 +- 5 files changed, 51 insertions(+), 19 deletions(-) delete mode 100644 deploy/docker/production/docker-compose.yml create mode 100644 docs/source/docker-production.rst diff --git a/deploy/docker/production/docker-compose.yml b/deploy/docker/production/docker-compose.yml deleted file mode 100644 index 09e0235faf..0000000000 --- a/deploy/docker/production/docker-compose.yml +++ /dev/null @@ -1,13 +0,0 @@ -version: "3.8" - -services: - - # Production component - cgap-production: - build: . - container_name: cgap - command: "/home/nginx/cgap-portal/entrypoint.sh" - environment: - IDENTITY: - ports: - - "8000:8000" # nginx proxy port (note application traffic is not forwarded) diff --git a/deploy/docker/production/nginx.conf b/deploy/docker/production/nginx.conf index 39d5b1e276..5362ef77c2 100644 --- a/deploy/docker/production/nginx.conf +++ b/deploy/docker/production/nginx.conf @@ -5,7 +5,7 @@ error_log /var/log/nginx/error.log warn; events { - worker_connections 2048; + worker_connections 1024; } http { resolver 8.8.8.8; diff --git a/docs/source/docker-local.rst b/docs/source/docker-local.rst index eda98dbf4e..85a4904bc2 100644 --- a/docs/source/docker-local.rst +++ b/docs/source/docker-local.rst @@ -1,5 +1,5 @@ -CGAP-Docker -=========== +CGAP-Docker (local) +=================== It is now possible to run a local deployment of CGAP without installing any system level dependencies other than Docker. A few important notes on this setup. @@ -28,8 +28,9 @@ AWS keys are sourced and run:: The first command will take awhile the first time you run it but should speed up after. Since it is doing a fresh rebuild every time it is a little slower than the old local deployment since it has to fully reinstall/rebuild both Python and the client. Because of this, it is recommended to continue active development using the existing installation setup. -Once the branch is ready for integrated testing, set the desired branch in ``docker-compose.yml`` and trigger a build. -When the app is brought online the behavior should be identical to that of the existing local deployment setup. +Once the branch is ready for "integrated" testing, set the desired branch in ``docker-compose.yml`` and trigger a build. +When the app is brought online the behavior should be identical to that of the existing local deployment setup. It will +also match the production behavior in ECS. To access the running container:: @@ -50,3 +51,10 @@ Below is a small list of useful Docker commands for advanced users:: $ docker-compose up -d # will start cluster in background using existing containers $ docker-compose up -d -V --build # trigger a rebuild/recreation of cluster containers $ docker system prune # will cleanup unused Docker components - BE CAREFUL WITH THIS + +Note that the Dockerfile's take arguments (set these in docker-compose.yml): + * CGAP_ENV_NAME="cgap-mastertest" + * CGAP_REPO="https://github.com/dbmi-bgm/cgap-portal.git" + * CGAP_BRANCH="master" + * ENTRYPOINT="entrypoint.sh" + diff --git a/docs/source/docker-production.rst b/docs/source/docker-production.rst new file mode 100644 index 0000000000..9e55389cae --- /dev/null +++ b/docs/source/docker-production.rst @@ -0,0 +1,37 @@ +CGAP-Docker (production) +======================== + +CGAP-Docker runs in production on Elastic Container Service, meant to be orchestrated from the 4dn-cloud-infra repository. End users will modify `deploy/docker/production/Makefile`` to suite their immediate build needs with respect to target AWS Account/ECR Repository/Tagging strategy. For more information on the specifics of the ECS setup, see 4dn-cloud-infra. + +The CGAP Application has been orchestrated into the ECS Service/Task paradigm. As of writing all core application services have their own image tag defined by the ``$ENTRYPOINT`` build argument. As such, they are all separate services with the following notable characteristics: + + * WSGI - services standard API requests - 8x parallelization on Fargate Spot + * Indexer - hits /index at 3 second intervals indefinitely - 4x parallelization on Fargate Spot + * Ingester - poll for ingestion tasks from SQS - 1x parallelization TODO add ability to add additional tasks through API + * Deployment - triggers the standard deployment actions - must be explicitly run either through ECS console or TODO through API. + +Building an Image +^^^^^^^^^^^^^^^^^ + +The production application configuration is in ``deploy/docker/production``. A description of all the relevent files is below. + + * assume_identity.py - script for pulling application configuration from Secrets Manager + * Dockerfile - production Dockerfile, essentially identical to local deployment, except the container does not run as root + * entrypoint.sh - WSGI entrypoint + * entrypoint_deploymentn.sh - deployment entrypoint + * entrypoint_indexer.sh - indexer entrypoint + * entrypoint_ingester.sh - ingester entrypoint + * install_nginx.sh - script for pulling in nginx + * Makefile - configures builds/pushes for relevant images + * mastertest.ini - base ini file used to build production.ini on the server + * nginx.conf - nginx configuration + + +The following instructions describe how to build/push images. Note though that we assume an existing ECS setup. For instructions on how to orchestrate ECS, see 4dn-cloud-infra, but that is not the focus of this documentation. + + * Ensure the orchestrator credentials are sourced, or that your IAM user has been granted sufficient perms to push to ECR. + * In the Makefile, replace "cgap-mastertest" with the env.name configured for the environment. This name should match the ECR repo name if you navigate to the ECR Console. + * Again in the Makefile, replace the ECR Repo URL (NOT the tags) with the one from the output of the ECR stack in the account. + * Run ``make login``, which should pull ECR credentials using the currently active AWS credentials. + * Run ``make info`` for information on tags. + * Run the appropriate make target to build/deploy the desired version by pushing a new image to ECR. Note that the upload process may take a long time if you made application code (non-configuration) changes. diff --git a/src/encoded/ingestion/queue_utils.py b/src/encoded/ingestion/queue_utils.py index c48bd9e3f8..fed14fac73 100644 --- a/src/encoded/ingestion/queue_utils.py +++ b/src/encoded/ingestion/queue_utils.py @@ -22,7 +22,7 @@ class and QueueManager should be refactored into a "helper" class, but for now t def __init__(self, registry, override_name=None): """ Does initial setup for interacting with SQS """ - self.batch_size = 10 + self.batch_size = 1 # NOTE: this value is important because we don't want to block other jobs self.env_name = registry.settings.get('env.name', None) if not self.env_name: # replace with something usable backup = socket.gethostname()[:80].replace('.', '-') From 266bcbef9de034b34148f9392d0d5c1e8ec2c7fa Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 18 May 2021 12:14:40 -0400 Subject: [PATCH 070/120] small changes --- docs/source/docker-production.rst | 8 ++++---- src/encoded/ingestion/queue_utils.py | 6 ++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/source/docker-production.rst b/docs/source/docker-production.rst index 9e55389cae..c008ee182c 100644 --- a/docs/source/docker-production.rst +++ b/docs/source/docker-production.rst @@ -1,9 +1,9 @@ CGAP-Docker (production) ======================== -CGAP-Docker runs in production on Elastic Container Service, meant to be orchestrated from the 4dn-cloud-infra repository. End users will modify `deploy/docker/production/Makefile`` to suite their immediate build needs with respect to target AWS Account/ECR Repository/Tagging strategy. For more information on the specifics of the ECS setup, see 4dn-cloud-infra. +CGAP-Docker runs in production on AWS Elastic Container Service, meant to be orchestrated from the 4dn-cloud-infra repository. End users will modify ``deploy/docker/production/Makefile`` to suite their immediate build needs with respect to target AWS Account/ECR Repository/Tagging strategy. For more information on the specifics of the ECS setup, see 4dn-cloud-infra. -The CGAP Application has been orchestrated into the ECS Service/Task paradigm. As of writing all core application services have their own image tag defined by the ``$ENTRYPOINT`` build argument. As such, they are all separate services with the following notable characteristics: +The CGAP Application has been orchestrated into the ECS Service/Task paradigm. As of writing all core application services have their own image tag varied by passing the ``$ENTRYPOINT`` build argument. As such, they are all separate services with the following notable characteristics: * WSGI - services standard API requests - 8x parallelization on Fargate Spot * Indexer - hits /index at 3 second intervals indefinitely - 4x parallelization on Fargate Spot @@ -13,10 +13,10 @@ The CGAP Application has been orchestrated into the ECS Service/Task paradigm. A Building an Image ^^^^^^^^^^^^^^^^^ -The production application configuration is in ``deploy/docker/production``. A description of all the relevent files is below. +The production application configuration is in ``deploy/docker/production``. A description of all the relevant files follows. * assume_identity.py - script for pulling application configuration from Secrets Manager - * Dockerfile - production Dockerfile, essentially identical to local deployment, except the container does not run as root + * Dockerfile - production Dockerfile, essentially identical to local deployment except the container does not run as root * entrypoint.sh - WSGI entrypoint * entrypoint_deploymentn.sh - deployment entrypoint * entrypoint_indexer.sh - indexer entrypoint diff --git a/src/encoded/ingestion/queue_utils.py b/src/encoded/ingestion/queue_utils.py index fed14fac73..4a22c64bf1 100644 --- a/src/encoded/ingestion/queue_utils.py +++ b/src/encoded/ingestion/queue_utils.py @@ -16,9 +16,11 @@ class IngestionQueueManager: class and QueueManager should be refactored into a "helper" class, but for now this is sufficient and is tested independently here. - We will use a single queue to keep track of VCF File uuids to be indexed. + We will use a single queue to keep track of File uuids to be indexed. This used to manage only VCFs + but now the Ingestion functionality is generic and can be extended to arbitrary processing on + any type. """ - BUCKET_EXTENSION = '-vcfs' + BUCKET_EXTENSION = '-ingestion-queue' # XXX: breaking change, matches 4dn-cloud-infra resources def __init__(self, registry, override_name=None): """ Does initial setup for interacting with SQS """ From 8892aa36562a60820ccb8cb959d5007aeb570d5a Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 20 May 2021 17:45:50 -0400 Subject: [PATCH 071/120] bring Dockerfile to top level --- .dockerignore | 1 + .../production/Dockerfile => Dockerfile | 54 ++++++++++++------- 2 files changed, 35 insertions(+), 20 deletions(-) create mode 100644 .dockerignore rename deploy/docker/production/Dockerfile => Dockerfile (73%) diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..3c3629e647 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +node_modules diff --git a/deploy/docker/production/Dockerfile b/Dockerfile similarity index 73% rename from deploy/docker/production/Dockerfile rename to Dockerfile index 0a080cdd63..fd9040d827 100644 --- a/deploy/docker/production/Dockerfile +++ b/Dockerfile @@ -3,6 +3,7 @@ # so that we don't pick up new images unintentionally # Debian Buster with Python 3.6.13 +# TODO: maybe swap in ubuntu 20.04 and install Python manually? FROM python@sha256:db248d2d0494973550d323dd6b82af7fc2f4c1e0365769a758abd7fac2aa70db MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" @@ -10,12 +11,8 @@ MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" # Build Arguments ARG CGAP_ENV_NAME ENV CGAP_ENV_NAME=${CGAP_ENV_NAME:-"cgap-mastertest"} -ARG CGAP_REPO -ENV CGAP_REPO=${CGAP_REPO:-"https://github.com/dbmi-bgm/cgap-portal.git"} -ARG CGAP_BRANCH -ENV CGAP_BRANCH=${CGAP_BRANCH:-"c4_519"} ARG ENTRYPOINT -ENV ENTRYPOINT=${ENTRYPOINT:-"entrypoint.sh"} +ENV ENTRYPOINT=${ENTRYPOINT:-"deploy/docker/production/entrypoint.sh"} # Configure (global) Env ENV NGINX_USER=nginx @@ -30,7 +27,7 @@ ENV PYTHONFAULTHANDLER=1 \ POETRY_VERSION=1.1.4 # Install nginx, base system -COPY install_nginx.sh / +COPY deploy/docker/production/install_nginx.sh / RUN bash /install_nginx.sh && \ apt-get update && \ apt-get install -y curl vim emacs postgresql-client net-tools && \ @@ -40,24 +37,41 @@ RUN bash /install_nginx.sh && \ # Configure CGAP User (nginx) WORKDIR /home/nginx -# Configure venv, repo, back-end build +# Configure venv ENV VIRTUAL_ENV=/opt/venv RUN python -m venv /opt/venv ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +# Upgrade pip, install in layer +RUN pip install --upgrade pip && \ + pip install poetry==1.1.4 wheel==0.29.0 + +# Adjust permissions RUN chown -R nginx:nginx /opt/venv && \ - mkdir -p /home/nginx/cgap-portal && \ - git clone $CGAP_REPO --branch $CGAP_BRANCH && \ - cd cgap-portal && \ - pip install --upgrade pip && \ - pip install poetry==1.1.4 wheel==0.29.0 && \ - poetry install && \ + mkdir -p /home/nginx/cgap-portal + +WORKDIR /home/nginx/cgap-portal + +# Do the back-end dependency install +COPY pyproject.toml . +COPY poetry.lock . +RUN poetry install --no-root + +# Do the front-end dependency install +COPY package.json . +COPY package-lock.json . +RUN npm ci --no-fund --no-progress --no-optional --no-audit --python=/opt/venv/bin/python + +# Copy over the rest of the code +COPY . . + +# Build remaining back-end +RUN poetry install && \ python setup_eb.py develop && \ make fix-dist-info -# Front-end -WORKDIR /home/nginx/cgap-portal -RUN npm ci --no-fund --no-progress --no-optional --no-audit --python=/opt/venv/bin/python && \ - npm run build && \ +# Build front-end +RUN npm run build && \ npm run build-scss # Misc @@ -68,7 +82,7 @@ RUN make aws-ip-ranges && \ # Remove default configuration from Nginx RUN rm /etc/nginx/nginx.conf && \ rm /etc/nginx/conf.d/default.conf -COPY nginx.conf /etc/nginx/nginx.conf +COPY deploy/docker/production/nginx.conf /etc/nginx/nginx.conf # nginx filesystem setup RUN chown -R nginx:nginx /var/cache/nginx && \ @@ -86,12 +100,12 @@ RUN chown -R nginx:nginx /var/cache/nginx && \ # will be picked up by IniFileManager # *.ini must match the env name in secrets manager! # For now, this is mastertest. - Will 04/29/21 -COPY mastertest.ini deploy/ini_files/. +COPY deploy/docker/production/mastertest.ini deploy/ini_files/. RUN touch production.ini RUN chown nginx:nginx production.ini COPY $ENTRYPOINT entrypoint.sh -COPY assume_identity.py . +COPY deploy/docker/production/assume_identity.py . RUN chmod +x entrypoint.sh RUN chmod +x assume_identity.py EXPOSE 8000 From a10890adfbda8cae069a4c74546665c9a632f970 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 20 May 2021 17:47:18 -0400 Subject: [PATCH 072/120] remove beanstalk deploy code --- .ebextensions/01_apache.config | 27 ---- .ebextensions/04_upgrade_mod_wsgi.config | 31 ---- .ebextensions/05_set_wsgi.config | 179 ----------------------- .ebextensions/09_cloudwatch.config | 26 ---- .ebextensions/10_set_timeout.config | 4 - .ebextensions/11_logs.config | 47 ------ .ebextensions/13_run_npm.config | 37 ----- .ebextensions/20_packages.config | 77 ---------- .elasticbeanstalk/.gitignore | 5 - .elasticbeanstalk/config.yml | 20 --- deploy/deploy_beanstalk.py | 171 ---------------------- 11 files changed, 624 deletions(-) delete mode 100644 .ebextensions/01_apache.config delete mode 100644 .ebextensions/04_upgrade_mod_wsgi.config delete mode 100644 .ebextensions/05_set_wsgi.config delete mode 100644 .ebextensions/09_cloudwatch.config delete mode 100644 .ebextensions/10_set_timeout.config delete mode 100644 .ebextensions/11_logs.config delete mode 100644 .ebextensions/13_run_npm.config delete mode 100644 .ebextensions/20_packages.config delete mode 100644 .elasticbeanstalk/.gitignore delete mode 100644 .elasticbeanstalk/config.yml delete mode 100644 deploy/deploy_beanstalk.py diff --git a/.ebextensions/01_apache.config b/.ebextensions/01_apache.config deleted file mode 100644 index 394737b62c..0000000000 --- a/.ebextensions/01_apache.config +++ /dev/null @@ -1,27 +0,0 @@ -files: - "/etc/httpd/conf.modules.d/00-mpm.conf": - mode: "000644" - owner: root - group: root - content: | - # use prefork mpm. - # event mpm may be best for mod_wsgi, but doesn't seem to work with EB - LoadModule mpm_prefork_module modules/mod_mpm_prefork.so - - # For convenience, also set the following Apache directives here - # Find some Apache config tips for Elasticbeanstalk below: - # https://aws.amazon.com/premiumsupport/knowledge-center/apache-backend-elb/ - StartServers 20 - MinSpareServers 20 - MaxSpareServers 20 - MaxRequestWorkers 20 - ServerLimit 20 - MaxConnectionsPerChild 1000 - # set Timeout higher than idle timeout on load balancer - # should match RequestReadTimeout set in ../conf.d/mod_reqtimeout.conf - Timeout 62 - KeepAlive On - # set KeepAliveTimeout higher than Timeout - KeepAliveTimeout 63 - MaxKeepAliveRequests 100 - LogFormat "%{X-Forwarded-For}i %h %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"" combined diff --git a/.ebextensions/04_upgrade_mod_wsgi.config b/.ebextensions/04_upgrade_mod_wsgi.config deleted file mode 100644 index 9db1c847bf..0000000000 --- a/.ebextensions/04_upgrade_mod_wsgi.config +++ /dev/null @@ -1,31 +0,0 @@ -packages: - yum: - git: [] - gcc-c++: [] - -files: - "/tmp/update-wsgi.sh" : - mode: "000755" - owner: root - group: root - content: | - # https://stackoverflow.com/questions/33818007/error-building-installing-mod-wsgi-on-aws-elasticbeanstalk-for-django-deployment - # httpd24 comes with apxs, which is needed to compile mod_wsgi -kmp 9-Mar-2020 - sudo yum install -y httpd24-devel - # update mod_wsgi version - # https://serverfault.com/a/885445 - # https://modwsgi.readthedocs.io/en/develop/user-guides/quick-installation-guide.html - cd /tmp - wget -q "https://github.com/GrahamDumpleton/mod_wsgi/archive/4.6.5.tar.gz" && \ - tar -xzf '4.6.5.tar.gz' && \ - cd mod_wsgi-4.6.5/ && \ - sudo ./configure --with-python=/opt/python/run/venv/bin/python && \ - sudo make && \ - sudo make install && \ - sudo service httpd restart - sudo make clean - -commands: - 01_mod_wsgi_update: - command: /tmp/update-wsgi.sh &> /var/log/mod_wsgi_update.log - cwd: /tmp diff --git a/.ebextensions/05_set_wsgi.config b/.ebextensions/05_set_wsgi.config deleted file mode 100644 index 9bfebbede8..0000000000 --- a/.ebextensions/05_set_wsgi.config +++ /dev/null @@ -1,179 +0,0 @@ -files: - "/opt/elasticbeanstalk/local/override_wsgi_conf.py": - mode: "000755" - owner: root - group: root - content: | - #!/usr/bin/env python - # This file creates a Python script that runs on appdeploy and configdeploy - # pre hooks to override the WSGI config file - # See https://forums.aws.amazon.com/thread.jspa?threadID=163369 - - import os - import sys - sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) - import config - - # remove old wsgi hooks and config files - old_wsgi_hooks_conf = [ - '/opt/elasticbeanstalk/hooks/configdeploy/pre/99patchwsgi.py', - '/opt/elasticbeanstalk/hooks/appdeploy/pre/99patchwsgi.py', - '/etc/httpd/conf.d/encoded-apache.conf', - '/etc/httpd/wsgi.conf.d/extra_config.conf' - ] - for old_wsgi_file in old_wsgi_hooks_conf: - if os.path.exists(old_wsgi_file): - os.remove(old_wsgi_file) - else: - print("File %s does not exist, so needn't be deleted." % old_wsgi_file) - - MY_APACHE_TEMPLATE = ''' - # Customized wsgi.conf. If you're seeing this, good! - - # No need to load modules/mod_wsgi.so, since it's already loaded here: - # /etc/httpd/conf.modules.d/10-wsgi.conf - - WSGIPythonHome /opt/python/run/baselinenv - WSGISocketPrefix run/wsgi - WSGIRestrictEmbedded On - WSGIPassAuthorization On - - ### START originally in encoded-apache.conf - LogLevel info - - - Order deny,allow - Allow from all - - Require all granted - - - - - Order deny,allow - Allow from all - - Require all granted - - # Limit upload size to 500 MB (375MB before base64 encoding) - LimitRequestBody 524288000 - # Apache adds -gzip to outgoing ETag in mod_deflate, remove inbound. - # https://issues.apache.org/bugzilla/show_bug.cgi?id=39727 - RequestHeader edit If-Match -gzip\"$ \" - RequestHeader edit If-None-Match -gzip\"$ \" - - - # Serve static resources directly from Apache - Alias /favicon.ico /opt/python/current/app/src/encoded/static/img/favicon.ico - - # Compress JSON responses. - AddOutputFilterByType DEFLATE application/javascript application/json text/css text/html text/javascript - - # Source map type (to enable compression) - - ForceType application/json - - - RewriteEngine On - - # Proxy internal redirects for file downloads - SSLProxyEngine On - RewriteCond %{ENV:REDIRECT_STATUS} . - RewriteRule ^/_proxy/(.+)$ $1 [proxy] - - # Redirect http to https from the load balancer - # https://stackoverflow.com/a/38751749 - - # Note in 4DN the rewrite condition is: - # RewriteCond %{HTTP_HOST} ^(data\.4dnucleome\.org|testportal\.4dnucleome\.org)$ - # but if CGAP has the equivalent of a testportal, I don't know about it. -kmp 2-Sep-2020 - RewriteCond %{HTTP_HOST} ^(cgap\.hms\.harvard\.edu)$ - RewriteRule (.*) https://%{HTTP_HOST}%{REQUEST_URI} [redirect=permanent,last,qsappend] - - ### END originally in encoded-apache.conf - - - - # force use of main Python interpreter - WSGIApplicationGroup %{GLOBAL} - - ### START originally in /etc/httpd/wsgi.conf.d/extra_config.conf - Header always set Access-Control-Allow-Origin "*" - Header always set Access-Control-Allow-Methods "GET, HEAD, OPTIONS" - Header always set Access-Control-Allow-Headers "Accept, Origin, Range, X-Requested-With" - Header always set Access-Control-Expose-Headers: "Content-Length, Content-Range, Content-Type" - RewriteCond %{REQUEST_METHOD} OPTIONS - RewriteRule ^ - [redirect=200,last] - - # Indexer. Configure first to avoid catchall '/' - # Use default number of processes=1, so this is not a multiprocess daemon - WSGIDaemonProcess encoded-indexer user=wsgi group=wsgi threads=1 python-path=/opt/python/current/app:/opt/python/run/venv/lib64/python3.6/site-packages:/opt/python/run/venv/lib/python3.6/site-packages display-name=encoded-indexer - WSGIScriptAlias /_indexer /opt/python/current/app/parts/production-indexer/wsgi process-group=encoded-indexer application-group=%{GLOBAL} - - # Ingestion listener (very similar config to the indexer, but not multi-processed the same way) - WSGIDaemonProcess encoded-ingestion-listener user=wsgi group=wsgi threads=1 python-path=/opt/python/current/app:/opt/python/run/venv/lib64/python3.6/site-packages:/opt/python/run/venv/lib/python3.6/site-packages display-name=encoded-ingestion-listener - WSGIScriptAlias /_ingestion_listener /opt/python/current/app/parts/production-ingestion-listener/wsgi process-group=encoded-ingestion-listener application-group=%{GLOBAL} - - # https://github.com/GrahamDumpleton/mod_wsgi/issues/2 - # Try Graham Dumpleton's fix since we have upgraded WSGI - # Original fix (next 3 lines) - # SetEnvIf Request_Method HEAD X_REQUEST_METHOD=HEAD - # LogFormat "%v:%p %h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\" %{X-Stats}o&server_time=%D" vhost_combined_stats - # ErrorLogFormat "%M" - - ### END originally in /etc/httpd/wsgi.conf.d/extra_config.conf - - Alias /static/ /opt/python/current/app/src/encoded/static/ - - Order allow,deny - Allow from all - - - WSGIScriptAlias / /opt/python/current/app/parts/production/wsgi - - - Require all granted - - - # Customized WSGIDaemonProcess settings - # https://modwsgi.readthedocs.io/en/latest/configuration-directives/WSGIDaemonProcess.html - # Notes: - # - might want to add "request-timeout", though it would interfere with long running requests - # - "queue-timeout" should not occur since Apache servers match total WSGI threads. Set to value of Apache Timeout - # - maybe use "restart-interval" than "maximum-requests", but handle long requests - - WSGIDaemonProcess wsgi processes=5 threads=4 display-name='%{GROUP}' python-path=/opt/python/current/app:/opt/python/run/venv/lib64/python3.6/site-packages:/opt/python/run/venv/lib/python3.6/site-packages user=wsgi group=wsgi home=/opt/python/current/app graceful-timeout=30 deadlock-timeout=60 queue-timeout=62 maximum-requests=1000 - WSGIProcessGroup wsgi - - ''' - - - def main(): - if 'EB_SYSTEM_STARTUP' in os.environ: - return - try: - WSGI_STAGING_CONFIG = config.get_container_config('wsgi_staging_config') - print("WSGI_STAGING_CONFIG=", WSGI_STAGING_CONFIG) - open(WSGI_STAGING_CONFIG, 'w').write(MY_APACHE_TEMPLATE) - except Exception, e: - config.emit_error_event(config.USER_ERROR_MESSAGES['badappconfig']) - config.diagnostic("Error generating config during configdeploy/pre: %s" % str(e)) - sys.exit(1) - - - if __name__ == '__main__': - config.configure_stdout_logger() - main() - -commands: - 01_app_deploy_dir: - command: "mkdir -p /opt/elasticbeanstalk/hooks/appdeploy/pre" - - 02_config_deploy_dir: - command: "mkdir -p /opt/elasticbeanstalk/hooks/configdeploy/pre" - - 03_app_deploy_file: - command: "cp -p /opt/elasticbeanstalk/local/override_wsgi_conf.py /opt/elasticbeanstalk/hooks/appdeploy/pre/90_override_wsgi_conf.py" - - 04_config_deploy_file: - command: "cp -p /opt/elasticbeanstalk/local/override_wsgi_conf.py /opt/elasticbeanstalk/hooks/configdeploy/pre/90_override_wsgi_conf.py" diff --git a/.ebextensions/09_cloudwatch.config b/.ebextensions/09_cloudwatch.config deleted file mode 100644 index 89a6f587c0..0000000000 --- a/.ebextensions/09_cloudwatch.config +++ /dev/null @@ -1,26 +0,0 @@ -packages: - yum: - perl-DateTime: [] - perl-Sys-Syslog: [] - perl-LWP-Protocol-https: [] - perl-Switch: [] - perl-URI: [] - perl-Bundle-LWP: [] - -sources: - /opt/cloudwatch: https://aws-cloudwatch.s3.amazonaws.com/downloads/CloudWatchMonitoringScripts-1.2.1.zip - -container_commands: - 01-setupcron: - command: | - echo '*/5 * * * * root perl /opt/cloudwatch/aws-scripts-mon/mon-put-instance-data.pl `{"Fn::GetOptionSetting" : { "OptionName" : "CloudWatchMetrics", "DefaultValue" : "--mem-util --disk-space-util --disk-path=/" }}` > /dev/null' > /etc/cron.d/cwpump - 02-changeperm: - command: chmod 644 /etc/cron.d/cwpump - 03-changeperm: - command: chmod u+x /opt/cloudwatch/aws-scripts-mon/mon-put-instance-data.pl - -option_settings: - "aws:autoscaling:launchconfiguration" : - IamInstanceProfile : "aws-elasticbeanstalk-ec2-role" - "aws:elasticbeanstalk:customoption" : - CloudWatchMetrics : "--mem-util --mem-used --mem-avail --disk-space-util --disk-space-used --disk-space-avail --disk-path=/ --auto-scaling" diff --git a/.ebextensions/10_set_timeout.config b/.ebextensions/10_set_timeout.config deleted file mode 100644 index 099702df61..0000000000 --- a/.ebextensions/10_set_timeout.config +++ /dev/null @@ -1,4 +0,0 @@ -option_settings: - - namespace: aws:elasticbeanstalk:command - option_name: Timeout - value: 3300 diff --git a/.ebextensions/11_logs.config b/.ebextensions/11_logs.config deleted file mode 100644 index 1953f91e11..0000000000 --- a/.ebextensions/11_logs.config +++ /dev/null @@ -1,47 +0,0 @@ -files: - "/opt/elasticbeanstalk/tasks/bundlelogs.d/deploy.conf" : - mode: "000644" - owner: root - group: root - content: | - /var/log/deploy.log - "/opt/elasticbeanstalk/tasks/bundlelogs.d/create_mapping.conf" : - mode: "000644" - owner: root - group: root - content: | - /var/log/create_mapping.log - "/etc/logrotate.elasticbeanstalk.hourly/logrotate.elasticbeanstalk.httpd.conf": - mode: "000644" - owner: root - group: root - content: | - /var/log/httpd/* { - copytruncate - size 10M - missingok - notifempty - rotate 24 - dateext - dateformat -%s - olddir /var/log/httpd/rotated - } - - # Previous configuration caused problems with graceful restart - # dropping connections. Initially switched to graceful b/c reload - # caused connections to hang. Finally switch to using copytruncate - # /var/log/httpd/* { - # size 10M - # missingok - # notifempty - # rotate 5 - # sharedscripts - # compress - # dateext - # dateformat -%s - # create - # postrotate - # /sbin/service httpd graceful > /dev/null 2>/dev/null || true - # endscript - # olddir /var/log/httpd/rotated - #} diff --git a/.ebextensions/13_run_npm.config b/.ebextensions/13_run_npm.config deleted file mode 100644 index 041c390660..0000000000 --- a/.ebextensions/13_run_npm.config +++ /dev/null @@ -1,37 +0,0 @@ -files: - "/opt/elasticbeanstalk/hooks/appdeploy/post/99_run_npm.sh": - mode: "000755" - owner: root - group: root - content: | - #!/bin/bash - - # run npm install and npm build needed for application - # these were previously run in bin/buildout as part of - # container_commands in 20_packages.config, but that - # stopped working. - # commands taken from buildout.cfg - - cd /opt/python/current/app - - echo "Running NPM install in post-deploy hook 99_run_npm..." >> /var/log/deploy.log - echo "Running NPM build in 99_run_npm..." - - su -c "npm install --no-fund --no-progress --python=/opt/python/run/venv/bin/python" ec2-user >> /var/log/deploy.log - - echo "Running NPM build in 99_run_npm..." >> /var/log/deploy.log - echo "Running NPM build in 99_run_npm..." - - su -c "npm run build" ec2-user >> /var/log/deploy.log - - echo "Running NPM build-scss in post-deploy hook 99_run_npm..." >> /var/log/deploy.log - echo "Running NPM build in 99_run_npm..." - - su -c "npm run build-scss" ec2-user >> /var/log/deploy.log - - echo "Finished NPM build in 99_run_npm." >> /var/log/deploy.log - echo "Finished NPM build in 99_run_npm." - - echo "Restarting Apache" - - sudo service httpd restart diff --git a/.ebextensions/20_packages.config b/.ebextensions/20_packages.config deleted file mode 100644 index d9a5c858c3..0000000000 --- a/.ebextensions/20_packages.config +++ /dev/null @@ -1,77 +0,0 @@ -packages: - yum: - git: [] - postgresql93-devel: [] - libffi-devel: [] - libjpeg-turbo-devel: [] - libtiff: [] - bsdtar: [] - graphviz: [] - mod24_ssl: [] - gcc-c++: [] - gcc: [] - -container_commands: - 0000_nodejs_install: - command: "curl --silent --location https://rpm.nodesource.com/setup_12.x | sudo bash - && yum install nodejs -y && node --version >> /var/log/deploy.log" - 0100_setup_wsgi_home: - command: "mkdir -p /home/wsgi && chown wsgi:wsgi /home/wsgi" - 0190_mostly_ugprade_pip: # pin a version believed to work - command: "source /opt/python/run/venv/bin/activate && pip install --upgrade pip==21.0.1 >> /var/log/deploy.log" - 0191_check_pip_version_anew: - command: "source /opt/python/run/venv/bin/activate && pip --version" - 0200_install_poetry: # pin a version believed to work - command: "source /opt/python/run/venv/bin/activate && pip install poetry==1.1.4 >> /var/log/deploy.log" - 0201_install_poetry: - command: "source /opt/python/run/venv/bin/activate && poetry --version" - 0210_superstitiously_make_clean: - command: "make clean" - 0217_check_pip_state: - command: "source /opt/python/run/venv/bin/activate && pip freeze" - 0218_uninstall_setuptools: - command: "source /opt/python/run/venv/bin/activate && pip uninstall -y setuptools" - 0219_install_setuptools: - command: "source /opt/python/run/venv/bin/activate && pip install setuptools==44.1.1" # known to work - 0220_populate_venv: - command: "source /opt/python/run/venv/bin/activate && poetry install" - 0221_get_aws_ips: - command: "make aws-ip-ranges" - 0400_superstitiously_activate_venv: - command: "source /opt/python/run/venv/bin/activate" - 0420_still_more_debugging_info: - command: "source /opt/python/run/venv/bin/activate && echo $PATH" - 0480_npm_tmp_perms: - command: "chown -R ec2-user /tmp" - 0490_app_bundle_perms: - command: "chown -R ec2-user /opt/python/bundle/" - 0500_secret_key: - command: "cat /dev/urandom | head -c 256 | base64 > session-secret.b64" - 0600_generate_production_ini: - command: "source /opt/python/run/venv/bin/activate && python deploy/generate_production_ini.py" - 0696_pip_install_encoded: - command: "source /opt/python/run/venv/bin/activate && python setup_eb.py develop --verbose && make fix-dist-info" - 0700_clear_db_es_contents: - command: "source /opt/python/run/venv/bin/activate && clear-db-es-contents production.ini --app-name app --skip-es --env fourfront-cgapdev >> /var/log/deploy.log" - leader_only: true - 0810_elastic_search_mapping: - command: "source /opt/python/run/venv/bin/activate && create-mapping-on-deploy production.ini --app-name app --clear-queue &> /var/log/create_mapping.log" - leader_only: true - 0820_load_dummy_data: - command: "source /opt/python/run/venv/bin/activate && load-data production.ini --app-name app >> /var/log/deploy.log" - leader_only: true - 0830_load_access_keys: - command: "source /opt/python/run/venv/bin/activate && load-access-keys production.ini --app-name app >> /var/log/deploy.log" - leader_only: true - 0900_restart_apache: - command: "sudo service httpd restart" - -option_settings: - "aws:elasticbeanstalk:application:environment": - "LC_ALL" : "en_US.UTF-8" - "LANG" : "en_US.UTF-8" - "aws:elasticbeanstalk:container:python:staticfiles": - "/static/": "src/encoded/static/" - "aws:elasticbeanstalk:container:python": - WSGIPath: parts/production/wsgi - NumProcesses: 5 - NumThreads: 4 diff --git a/.elasticbeanstalk/.gitignore b/.elasticbeanstalk/.gitignore deleted file mode 100644 index bca646a711..0000000000 --- a/.elasticbeanstalk/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ - -# Elastic Beanstalk Files -.elasticbeanstalk/* -!.elasticbeanstalk/*.cfg.yml -!.elasticbeanstalk/*.global.yml diff --git a/.elasticbeanstalk/config.yml b/.elasticbeanstalk/config.yml deleted file mode 100644 index 39ce025a1c..0000000000 --- a/.elasticbeanstalk/config.yml +++ /dev/null @@ -1,20 +0,0 @@ -branch-defaults: - FF-687: - ec2_keyname: alexkb - environment: 4dn-web-alex-2 - aws_beanstalk: - environment: 4dn-web-prod - group_suffix: null - fix_404_no_auth: - environment: 4dn-web-alex - master: - environment: fourfront-cgap - production: - environment: fourfront-cgap -global: - application_name: 4dn-web - default_ec2_keyname: 4dn-encode - default_platform: Python 3.4 - default_region: us-east-1 - profile: null - sc: git diff --git a/deploy/deploy_beanstalk.py b/deploy/deploy_beanstalk.py deleted file mode 100644 index 949b75b038..0000000000 --- a/deploy/deploy_beanstalk.py +++ /dev/null @@ -1,171 +0,0 @@ -import os -from time import sleep -import sys -import subprocess -import hashlib -import argparse -from datetime import datetime - - -def tag(name): - subprocess.check_output(['git', 'tag', name, '-m', 'version created for staging deploy']) - subprocess.check_output(['git', 'push', 'origin-travis', name]) - - -def merge(source, merge_to): - res1 = subprocess.check_output(['git', 'status']).decode('utf-8').strip() - - print("status on master is: " + res1) - subprocess.check_output(['git', 'stash']) - - subprocess.check_output( - ['git', 'checkout', merge_to]) - - res = subprocess.check_output(['git', 'status']).decode('utf-8').strip() - print("status on prod is: " + res) - - res2 = subprocess.check_output( - ['git', 'merge', source, '-m', 'merged']).decode('utf-8').strip() - print(res2) - subprocess.check_output( - ['git', 'push', 'origin-travis', merge_to]).decode('utf-8').strip() - subprocess.check_output(['git', 'stash', 'pop']) - - -def get_git_version(): - version = os.environ.get("TRAVIS_COMMIT", "")[:7] - if not version: - version = subprocess.check_output( - ['git', '-C', os.path.dirname(__file__), 'describe']).decode('utf-8').strip() - version = version[:7] - diff = subprocess.check_output( - ['git', '-C', os.path.dirname(__file__), 'diff', '--no-ext-diff']) - if diff: - version += '-patch' + hashlib.sha1(diff).hexdigest()[:7] - return "v-" + version - - -def update_version(version, branch): - filename = 'buildout.cfg' - regex = 's/encoded_version.*/encoded_version = %s/' % (version) - - print("updated buildout.cfg with version", version) - subprocess.check_output( - ['sed', '-i', regex, filename]) - commit_with_previous_msg(filename, branch) - - -def commit_with_previous_msg(filename, branch): - print("adding file to git") - subprocess.check_output( - ['git', 'add', filename]) - - msg = parse(previous_git_commit()) - - print("git commit -m " + msg) - subprocess.check_output( - ['git', 'commit', '-m', 'version bump + ' + msg]) - - subprocess.check_output( - ['git', 'push', 'origin-travis', branch]) - - -def previous_git_commit(): - return subprocess.check_output( - ['git', 'log', '-1'] - ).decode('utf-8').strip() - - -def parse(commit): - author, msg = "", "" - # parse up some commit lines - commit_lines = commit.split('\n') - author = commit_lines[1].split(":")[1].strip() - msg = " ".join(l.strip() for l in commit_lines[3:] if l) - - return "%s - %s" % (author, msg) - - -def deploy(deploy_to=None): - ''' - run eb deploy and show the output - ''' - print("start deployment to elastic beanstalk deploy to is %s" % str(deploy_to)) - - wait = [20, 40, 60, 120, 120, 120, 120] - for time in wait: - try: - if not deploy_to: - p = subprocess.Popen(['eb', 'deploy'], stderr=subprocess.STDOUT, stdout=subprocess.PIPE) - else: - p = subprocess.Popen(['eb', 'deploy', deploy_to], stderr=subprocess.STDOUT, stdout=subprocess.PIPE) - except Exception: - # we often get errors due to timeouts - sleep(time) - else: - break - - time_started = datetime.now() - print('Started deployment at {}. Waiting 2 minutes & exiting.'.format(time_started.strftime('%H:%M:%S:%f'))) - sleep(120) - - # MAYBE TODO: Setup new thread and listen re: "Deploying new version to instance(s).". Exit if this occurs before 2min. - # - #while True: - # out = p.stdout.readline() - # out = out.decode('utf-8') - # curr_time = datetime.now() - # if out != '': - # sys.stdout.write('[' + curr_time.strftime('%H:%M:%S:%f') + '] ' + out) - # sys.stdout.flush() - # if ("Deploying new version to instance(s)." in out) or (time_started + timedelta(minutes=2) <= curr_time): # 2 min time limit - # print('Killing sub-process & exiting.') - # sleep(5) - # p.kill() - # break - # if out == '' and p.poll() is not None: - # print('Deploy sub-process complete. Exiting.') - # break - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="update version if relevant and deploy" - ) - args = parser.parse_args() - branch = os.environ.get("TRAVIS_BRANCH") - merge_to = os.environ.get("tibanna_merge", "").strip() - deploy_to = os.environ.get("tibanna_deploy", "").strip() - - # Ref: https://hms-dbmi.atlassian.net/browse/C4-114 - # Will and I believe this code to be stale and no longer needed. - # It's retained temporarily because it illustrates some interesting tool use - # I might want to borrow later for other purposes. -kmp 8-Apr-2020 - # - # try: - # if deploy_to in ['fourfront-staging', 'fourfront-webprod', 'fourfront-webprod2']: - # print("deploy to staging") - # ver = get_git_version() - # # checkout correct branch - # print("checkout master") - # subprocess.check_output( - # ['git', 'checkout', branch]) - # - # print("update version") - # update_version(ver, branch) - # if merge_to: - # print("merge from %s to %s" % (branch, merge_to)) - # merge(branch, merge_to) - # print("tag it") - # tag(ver) - # except Exception as e: - # # this can all go wrong if somebody pushes during the build - # # or what not, in which case we just won't update the tag / merge - # print("got the following expection but we will ignore it") - # print(e) - # print("switching back to source branch") - # subprocess.check_output( - # ['git', 'checkout', branch]) - - print("now let's deploy") - deploy(deploy_to) From 450ff4c8dfce183a64d23b96689256d1c9b32b84 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Fri, 21 May 2021 09:59:18 -0400 Subject: [PATCH 073/120] refactor to unify local, production Dockerfile at top level --- Dockerfile | 24 ++++++-- Makefile | 4 +- deploy/docker/local/Dockerfile | 100 --------------------------------- docker-compose.yml | 5 +- 4 files changed, 23 insertions(+), 110 deletions(-) delete mode 100644 deploy/docker/local/Dockerfile diff --git a/Dockerfile b/Dockerfile index fd9040d827..fa2733d4ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# CGAP-Portal Dockerfile +# CGAP-Portal (Production) Dockerfile # Note that images are pinned via sha256 as opposed to tag # so that we don't pick up new images unintentionally @@ -11,8 +11,12 @@ MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" # Build Arguments ARG CGAP_ENV_NAME ENV CGAP_ENV_NAME=${CGAP_ENV_NAME:-"cgap-mastertest"} +ARG BUILD_PATH +ENV BUILD_PATH=${BUILD_PATH:-"deploy/docker/production/"} +ARG INI_BASE +ENV INI_BASE=${INI_BASE:-"mastertest.ini"} ARG ENTRYPOINT -ENV ENTRYPOINT=${ENTRYPOINT:-"deploy/docker/production/entrypoint.sh"} +ENV ENTRYPOINT=${ENTRYPOINT:-"entrypoint.sh"} # Configure (global) Env ENV NGINX_USER=nginx @@ -27,7 +31,7 @@ ENV PYTHONFAULTHANDLER=1 \ POETRY_VERSION=1.1.4 # Install nginx, base system -COPY deploy/docker/production/install_nginx.sh / +COPY $BUILD_PATH/install_nginx.sh / RUN bash /install_nginx.sh && \ apt-get update && \ apt-get install -y curl vim emacs postgresql-client net-tools && \ @@ -82,7 +86,7 @@ RUN make aws-ip-ranges && \ # Remove default configuration from Nginx RUN rm /etc/nginx/nginx.conf && \ rm /etc/nginx/conf.d/default.conf -COPY deploy/docker/production/nginx.conf /etc/nginx/nginx.conf +COPY $BUILD_PATH/nginx.conf /etc/nginx/nginx.conf # nginx filesystem setup RUN chown -R nginx:nginx /var/cache/nginx && \ @@ -100,11 +104,19 @@ RUN chown -R nginx:nginx /var/cache/nginx && \ # will be picked up by IniFileManager # *.ini must match the env name in secrets manager! # For now, this is mastertest. - Will 04/29/21 -COPY deploy/docker/production/mastertest.ini deploy/ini_files/. +RUN if [[ $BUILD_PATH == *"production"* ]]; then \ + echo "Detected production build" && \ + cp $BUILD_PATH/$INI_BASE deploy/ini_files/. ; \ + else \ + echo "Detected local build" && \ + cp $BUILD_PATH/docker_development.ini development.ini ; \ + fi + + RUN touch production.ini RUN chown nginx:nginx production.ini -COPY $ENTRYPOINT entrypoint.sh +COPY $BUILD_PATH/$ENTRYPOINT entrypoint.sh COPY deploy/docker/production/assume_identity.py . RUN chmod +x entrypoint.sh RUN chmod +x assume_identity.py diff --git a/Makefile b/Makefile index 8d29700736..4a9871d1a5 100644 --- a/Makefile +++ b/Makefile @@ -173,10 +173,10 @@ update: # updates dependencies poetry update build-docker: - docker-compose build + docker-compose build --build-arg BUILD_PATH=deploy/docker/local/ build-docker-clean: - docker-compose build --no-cache + docker-compose build --no-cache BUILD_PATH=deploy/docker/local/ deploy-docker: docker-compose up -V diff --git a/deploy/docker/local/Dockerfile b/deploy/docker/local/Dockerfile deleted file mode 100644 index 0e510dbf3d..0000000000 --- a/deploy/docker/local/Dockerfile +++ /dev/null @@ -1,100 +0,0 @@ -# CGAP-Portal Dockerfile - -# TODO: appropriately pin/verify this image -FROM python:3.6.12-buster - -MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" - -# CGAP-Portal Dockerfile -# Note that images are pinned via sha256 as opposed to tag -# so that we don't pick up new images unintentionally - -# Debian Buster with Python 3.6.13 -FROM python@sha256:db248d2d0494973550d323dd6b82af7fc2f4c1e0365769a758abd7fac2aa70db - -MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" - -# Build Arguments -ARG CGAP_ENV_NAME -ENV CGAP_ENV_NAME=${CGAP_ENV_NAME:-"cgap-mastertest"} -ARG CGAP_REPO -ENV CGAP_REPO=${CGAP_REPO:-"https://github.com/dbmi-bgm/cgap-portal.git"} -ARG CGAP_BRANCH -ENV CGAP_BRANCH=${CGAP_BRANCH:-"c4_519"} -ARG ENTRYPOINT -ENV ENTRYPOINT=${ENTRYPOINT:-"entrypoint.sh"} - -# Configure (global) Env -ENV NGINX_USER=nginx -ENV DEBIAN_FRONTEND=noninteractive -ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1 -ENV PYTHONFAULTHANDLER=1 \ - PYTHONUNBUFFERED=1 \ - PYTHONHASHSEED=random \ - PIP_NO_CACHE_DIR=off \ - PIP_DISABLE_PIP_VERSION_CHECK=on \ - PIP_DEFAULT_TIMEOUT=100 \ - POETRY_VERSION=1.1.4 - -# Install nginx, base system -COPY install_nginx.sh / -RUN bash /install_nginx.sh && \ - apt-get update && \ - apt-get install -y curl vim emacs postgresql-client net-tools && \ - curl -sL https://deb.nodesource.com/setup_10.x | bash - && \ - apt-get install -y ca-certificates nodejs npm - -# Configure CGAP User (nginx) -WORKDIR /home/nginx - -# Configure venv, repo, back-end build -ENV VIRTUAL_ENV=/opt/venv -RUN python -m venv /opt/venv -ENV PATH="$VIRTUAL_ENV/bin:$PATH" -RUN chown -R nginx:nginx /opt/venv && \ - mkdir -p /home/nginx/cgap-portal && \ - git clone $CGAP_REPO --branch $CGAP_BRANCH && \ - cd cgap-portal && \ - pip install --upgrade pip && \ - pip install poetry==1.1.4 wheel==0.29.0 && \ - poetry install && \ - python setup_eb.py develop && \ - make fix-dist-info - -# Front-end -WORKDIR /home/nginx/cgap-portal -RUN npm ci --no-fund --no-progress --no-optional --no-audit --python=/opt/venv/bin/python && \ - npm run build && \ - npm run build-scss - -# Misc -RUN make aws-ip-ranges && \ - cat /dev/urandom | head -c 256 | base64 > session-secret.b64 - -# Copy config files in (down here for quick debugging) -# Remove default configuration from Nginx -RUN rm /etc/nginx/nginx.conf && \ - rm /etc/nginx/conf.d/default.conf -COPY nginx.conf /etc/nginx/nginx.conf - -# nginx filesystem setup -RUN chown -R nginx:nginx /var/cache/nginx && \ - chown -R nginx:nginx /var/log/nginx && \ - chown -R nginx:nginx /etc/nginx/conf.d && \ - touch /var/run/nginx.pid && \ - chown -R nginx:nginx /var/run/nginx.pid && \ - rm -f /var/log/nginx/* && \ - touch /var/log/nginx/access.log && \ - chown -R nginx:nginx /var/log/nginx/access.log && \ - touch /var/log/nginx/error.log && \ - chown -R nginx:nginx /var/log/nginx/error.log - -# Copy over ini file, entrypoint -COPY docker_development.ini development.ini -COPY entrypoint.sh . -RUN chmod +x entrypoint.sh -EXPOSE 8000 -# Uncomment this to run as the unprivileged user -# USER nginx - -ENTRYPOINT ["/home/nginx/cgap-portal/entrypoint.sh"] diff --git a/docker-compose.yml b/docker-compose.yml index 940ef20e66..f26bf32df7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,7 +17,8 @@ services: #command: ["postgres", "-c", "log_statement=all"] # ElasticSearch Component - # Disabled for now as too compute intensive + # Disabled for now as too compute intensive, but feel free to try it if you dont need + # to test any search APIs # To connect use: # elasticsearch.server = es:9200 # es: @@ -42,7 +43,7 @@ services: # After running once, comment this out so the container will serve without reloading inserts. # Has no effect if TEST is specified. app: - build: ./deploy/docker/local + build: . container_name: cgap command: "/home/nginx/cgap-portal/entrypoint.sh" environment: From ff6846d3ff92e869e116d0d9d0b04f8a5d6a10a0 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Fri, 21 May 2021 14:47:35 -0400 Subject: [PATCH 074/120] finish refactor --- .dockerignore | 2 ++ .github/workflows/main.yml | 7 ++++++- Dockerfile | 6 +++--- Makefile | 4 ++-- deploy/docker/production/Makefile | 8 ++++---- 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/.dockerignore b/.dockerignore index 3c3629e647..4b548fd91a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,3 @@ +.docker-compose.yml +Dockerfile node_modules diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 188a21662a..3422097809 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -25,7 +25,7 @@ jobs: # Build matrix strategy: matrix: - test_type: ['UNIT', 'NPM'] + test_type: ['UNIT', 'NPM', 'Docker'] # Steps represent a sequence of tasks that will be executed as part of the job steps: @@ -42,6 +42,7 @@ jobs: check-latest: false - name: Install/Link Postgres + ${{ matrix.test_type == 'NPM' || matrix.test_type == 'UNIT' }} run: | sudo apt-get install curl ca-certificates gnupg curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - @@ -117,3 +118,7 @@ jobs: # Until the next version of snovault, the following two are prudent. We can remove them soon. -kmp 9-Mar-2021 poetry run wipe-test-indices $TRAVIS_JOB_ID search-cgap-testing-6-8-vo4mdkmkshvmyddc65ux7dtaou.us-east-1.es.amazonaws.com:443 poetry run wipe-test-indices cgap-test-$TRAVIS_JOB_ID search-cgap-testing-6-8-vo4mdkmkshvmyddc65ux7dtaou.us-east-1.es.amazonaws.com:443 + + - name: Docker Build + if: ${{ matrix.test_type == 'Docker' }} + run: make build-docker \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index fa2733d4ed..f41dafaacd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" ARG CGAP_ENV_NAME ENV CGAP_ENV_NAME=${CGAP_ENV_NAME:-"cgap-mastertest"} ARG BUILD_PATH -ENV BUILD_PATH=${BUILD_PATH:-"deploy/docker/production/"} +ENV BUILD_PATH=${BUILD_PATH:-"deploy/docker/production"} ARG INI_BASE ENV INI_BASE=${INI_BASE:-"mastertest.ini"} ARG ENTRYPOINT @@ -104,7 +104,8 @@ RUN chown -R nginx:nginx /var/cache/nginx && \ # will be picked up by IniFileManager # *.ini must match the env name in secrets manager! # For now, this is mastertest. - Will 04/29/21 -RUN if [[ $BUILD_PATH == *"production"* ]]; then \ +SHELL ["/bin/bash", "-c"] +RUN if [[ $BUILD_PATH =~ .*production.* ]]; then \ echo "Detected production build" && \ cp $BUILD_PATH/$INI_BASE deploy/ini_files/. ; \ else \ @@ -112,7 +113,6 @@ RUN if [[ $BUILD_PATH == *"production"* ]]; then \ cp $BUILD_PATH/docker_development.ini development.ini ; \ fi - RUN touch production.ini RUN chown nginx:nginx production.ini diff --git a/Makefile b/Makefile index 4a9871d1a5..acefe7c9fd 100644 --- a/Makefile +++ b/Makefile @@ -173,10 +173,10 @@ update: # updates dependencies poetry update build-docker: - docker-compose build --build-arg BUILD_PATH=deploy/docker/local/ + docker-compose build --build-arg BUILD_PATH=deploy/docker/local build-docker-clean: - docker-compose build --no-cache BUILD_PATH=deploy/docker/local/ + docker-compose build --no-cache BUILD_PATH=deploy/docker/local deploy-docker: docker-compose up -V diff --git a/deploy/docker/production/Makefile b/deploy/docker/production/Makefile index fd6054af40..99aa4f6f9e 100644 --- a/deploy/docker/production/Makefile +++ b/deploy/docker/production/Makefile @@ -2,21 +2,21 @@ login: aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 645819926742.dkr.ecr.us-east-1.amazonaws.com wsgi: - docker build -t cgap-mastertest:latest . + docker build -t cgap-mastertest:latest . --build-arg BUILD_PATH=deploy/docker/production docker tag cgap-mastertest:latest 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest docker push 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest indexer: - docker build -t cgap-mastertest:latest-indexer -f Dockerfile . --build-arg ENTRYPOINT=entrypoint_indexer.sh + docker build -t cgap-mastertest:latest-indexer -f Dockerfile . --build-arg ENTRYPOINT=entrypoint_indexer --build-arg BUILD_PATH=deploy/docker/production docker tag cgap-mastertest:latest-indexer 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-indexer docker push 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-indexer ingester: - docker build -t cgap-mastertest:latest-ingester -f Dockerfile . --build-arg ENTRYPOINT=entrypoint_ingester.sh + docker build -t cgap-mastertest:latest-ingester -f Dockerfile . --build-arg ENTRYPOINT=entrypoint_ingester.sh --build-arg BUILD_PATH=deploy/docker/production docker tag cgap-mastertest:latest-ingester 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-ingester docker push 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-ingester deployment: - docker build -t cgap-mastertest:latest-deployment -f Dockerfile . --build-arg ENTRYPOINT=entrypoint_deployment.sh + docker build -t cgap-mastertest:latest-deployment -f Dockerfile . --build-arg ENTRYPOINT=entrypoint_deployment.sh --build-arg BUILD_PATH=deploy/docker/production docker tag cgap-mastertest:latest-deployment 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-deployment docker push 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-deployment From 28c8cdad1788e32d92325ce28188a27545381184 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Fri, 21 May 2021 14:49:28 -0400 Subject: [PATCH 075/120] repair workflow --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3422097809..e275eeca69 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -42,7 +42,7 @@ jobs: check-latest: false - name: Install/Link Postgres - ${{ matrix.test_type == 'NPM' || matrix.test_type == 'UNIT' }} + if: ${{ matrix.test_type == 'NPM' || matrix.test_type == 'UNIT' }} run: | sudo apt-get install curl ca-certificates gnupg curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - From fb210e839af4273e10ba8e7f1d59833786ebf594 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Fri, 21 May 2021 14:50:14 -0400 Subject: [PATCH 076/120] remove stale test file --- deploy/tests/test_deploy_beanstalk.py | 33 --------------------------- 1 file changed, 33 deletions(-) delete mode 100644 deploy/tests/test_deploy_beanstalk.py diff --git a/deploy/tests/test_deploy_beanstalk.py b/deploy/tests/test_deploy_beanstalk.py deleted file mode 100644 index a2ced93c7b..0000000000 --- a/deploy/tests/test_deploy_beanstalk.py +++ /dev/null @@ -1,33 +0,0 @@ -import pytest - -from unittest import mock -from ..deploy_beanstalk import previous_git_commit, parse - - -GIT_LOG_OUTPUT = '''commit bc061fc755015162741eec71f1a71ea6c3fdb786 -Author: j1z0 -Date: Thu Sep 22 22:23:54 2016 -0400 - - we need .aws for both master and production - -''' - - -pytestmark = [pytest.mark.working, pytest.mark.unit] - - -def test_parse_git_commit(): - author = "j1z0 " - msg = "we need .aws for both master and production" - expected = "%s - %s" % (author, msg) - actual = parse(GIT_LOG_OUTPUT) - - print("expected result: ", expected) - print("actual result: ", actual) - assert expected == actual - - -if __name__ == "__main__": - print(".") - test_parse_git_commit() - print("all good!") From e1b25240d92aae6577b5cc56cac80b86bc7bb165 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Fri, 21 May 2021 14:51:34 -0400 Subject: [PATCH 077/120] do not install if doing docker build --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e275eeca69..324ec34750 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -52,6 +52,7 @@ jobs: echo "/usr/lib/postgresql/11/bin" >> $GITHUB_PATH sudo ln -s /usr/lib/postgresql/11/bin/initdb /usr/local/bin/initdb - name: Install Deps + if: ${{ matrix.test_type == 'NPM' || matrix.test_type == 'UNIT' }} run: | node --version make build From 816cb60ab58629a459722e70a2a764f7840dd323 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Mon, 24 May 2021 12:32:37 -0400 Subject: [PATCH 078/120] remove outdated documentation, add some more --- README.rst | 5 +- docs/source/account_creation.rst | 26 --- docs/source/beanstalk-deployment.rst | 117 ------------- docs/source/biosample_metadata.rst | 239 --------------------------- docs/source/docker-local.rst | 11 +- docs/source/index.rst | 1 + docs/source/introduction.rst | 50 ------ docs/source/local_installation.rst | 12 +- 8 files changed, 21 insertions(+), 440 deletions(-) delete mode 100644 docs/source/account_creation.rst delete mode 100644 docs/source/beanstalk-deployment.rst delete mode 100644 docs/source/biosample_metadata.rst delete mode 100644 docs/source/introduction.rst diff --git a/README.rst b/README.rst index 6f211802d0..b96438d6e6 100644 --- a/README.rst +++ b/README.rst @@ -28,11 +28,12 @@ Welcome to CGAP! We are a team of scientists, clinicians, and developers who aim Be warned that features are under active development and may not be stable! Visit the production deployment for the best experience. For installation and more information on getting started, see our `documentation page `_. -For information on how to run CGAP with Docker, see `here. <./docs/source/docker-setup.rst>`_ +For information on how to run CGAP with Docker, see `here. <./docs/source/docker-local.rst>`_ + +For information on CGAP-Docker in production, see `here. <./docs/source/docker-production.rst>`_ Navigating this Repository ^^^^^^^^^^^^^^^^^^^^^^^^^^ - * .ebextensions/ contains Beanstalk deployment scripts * .github/ contains Github Action Workflows * bin/ contains the few remaining executables * deploy/docker contains Docker setups (see docker-compose.yml) diff --git a/docs/source/account_creation.rst b/docs/source/account_creation.rst deleted file mode 100644 index c12e37ca2f..0000000000 --- a/docs/source/account_creation.rst +++ /dev/null @@ -1,26 +0,0 @@ -Account Creation -~~~~~~~~~~~~~~~~ - - -* Please email data wranglers at `support@4dnucleome.org `_ to get set up with an account with the access credentials for your role. -* Please provide an email address which you wish to use for your account and CC your PI for validation purposes. **The email associated with the account you use for login must be the same as the one registered with the 4DN-DCIC.** - - * This can be any email address (\ *e.g. an institutional email account*\ ) but must be connected to either a Google or Github account. - * For more information on linking your institutional email account to a Google account, see below. - -Signing in with your institutional email address -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - -* The DCIC uses the `OAuth `_ authentication system which will allow you to login with a Google or `GitHub `_ account. -* If you prefer to use your institutional email address to log in to the portal (recommended), you need to have a Google or GitHub account registered with that email address. -* If you do not already have a Google or GitHub account with your email address, you can set up one up by visiting the `Google account creation page with the non-gmail option `_. - -NOTE that it is important not to register this account to have gmail as your institutional email address must be the primary email associated with the google account for authentication to work properly! - -Once your account request is processed, you will then be able to log in with the 'LOG IN WITH GOOGLE' option using your institutional email address and Google account password. - - -.. image:: /static/img/docs/submitting-metadata/new-google-acct.png - :target: /static/img/docs/submitting-metadata/new-google-acct.png - :alt: Embedded fields diff --git a/docs/source/beanstalk-deployment.rst b/docs/source/beanstalk-deployment.rst deleted file mode 100644 index a0409c3e4b..0000000000 --- a/docs/source/beanstalk-deployment.rst +++ /dev/null @@ -1,117 +0,0 @@ - -Beanstalk Deployment -==================== - -**NOTE** Much of this document is outdated. As of September, 2019, deployments are managed through torb and dcicutils/beanstalk_utils. The Travis deployment section is still applicable. - -Beanstalk deployment through travis ------------------------------------ - -Currently Travis is set to deploy to beansalk on succesful build. - - -* Branch 'master' will deploy to the 4dn-web-dev environment (if all test pass) -* Branch 'prodution' will deploy to the 4dn-prod environment (if all tests pass) - -So to push something to production it should go through the following steps. - - -#. Pull request is created for feature branch. -#. Pull request accepted and merged to master. -#. Travis will pick this up run tests and deploy to 4dn-web-dev -#. If that is all succcesful to deploy to production do. -#. git checkout production -#. git merge master -#. edit deploy_beanstalk.py and change version number on line 10 to be next version. -#. Check in your changes. -#. git push origin production -#. Travis will then run tests and if pass will deploy to production - -Dropping database ------------------ - -For test environment the database is not dropped for each deploy. This means that new upserts, -which change existing data will in most cases not execute succesfully on the test environment (Unit upgrades are put back in place). - -When that happens we need to drop the database and recreate it, so the inserts can be run. - -The Old hard way to do it.. boooo :( -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Easiest way to do that is to ssh into the beanstalk instance and do the follow: - -** Note ** to ssh in first ``pip install awsebcli`` then follow the setup instructions. With that installed you can simply type eb ssh (ensuring that the master branch is checked out). (If this doesn't work, try ``eb init`` before ``eb ssh``\ ) - -Once connected do the following: - -.. code-block:: bash - - source /opt/python/current/env - sudo service httpd stop - echo $RDS_PASSWORD - - dropdb -p $RDS_PORT -h $RDS_HOSTNAME -U $RDS_USERNAME -e $RDS_DB_NAME - - createdb -p $RDS_PORT -h $RDS_HOSTNAME -U $RDS_USERNAME -e $RDS_DB_NAME - - - # drop indexes in elastic search - curl -XDELETE 'http://172.31.49.128:9872/annotations' - - # for 4dn-web-dev (Development Environment) - curl -XDELETE 'http://172.31.49.128:9872/snovault' - - # for production (PLEASE DONT SCREW THIS UP :) ) - curl -XDELETE 'http://172.31.49.128:9872/ffprod' - - sudo shutdown -r now - - # this will drop you back to your local machine, if you want to trigger latest build from master (and you know it's a clean build) - - git checkout master - eb deploy - -The New awesome way to do it: -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: bash - - sudo /opt/python/current/app/bin/dropdb - - # to bring things up again from back home - git checkout production - eb deploy - -Bye bye data. Use at your own risk, all warranties void. - -** Note ** this will temporarily bring the site down, for a couple of minutes - -Database backup / restore -------------------------- - -Database snapshots are automatically taken every day. To restore a backup on production (4dnweb-prod) - - -#. Go to the RDS tab and then look at the snapshots page. -#. Select the backup you want to restore. -#. Click Restore Snapshot -#. You will be prompted for a DB Instance Name, name it what you like. -#. Go to 4dnweb-prod environment and select configuration -> software configuration -#. Change the enviornment variable bnSTaLk_HOSTNAME to the name you just used for the new database. -#. Redeploy the applicaition production. - -Rebuilding beanstalk environemtns ---------------------------------- - -Any attempt to delete one of the beanstalk environment will most likely fail due to an inability to delete a secuirty group. This is because our RDS security group sets inbound rules for the beanstalk enviroments. So before you rebuild a beanstalk environment do the following: - - -#. Go to EC2's (aws console) -#. Select Security Groups -#. Search for sg-ab1d63d1 (this is our RDS security group) -#. Select inboud rules. -#. Find the inboud rule associated with the beanstalk environment security group (probably sg-something) -#. Remove that inboud rule. -#. Rebuild the envrionemnt. -#. You will need to add a new inbound rule to the RDS security group with the security group of the rebuilt Abeanstalk environment before deployment will be successful. -#. Security group id for beanstalk environment can be found under configuration -> Instances -> EC2 security Groups. diff --git a/docs/source/biosample_metadata.rst b/docs/source/biosample_metadata.rst deleted file mode 100644 index 3675b9d812..0000000000 --- a/docs/source/biosample_metadata.rst +++ /dev/null @@ -1,239 +0,0 @@ -Sample Metadata ---------------- - - -* The 4DN consortium will collect metadata on the preparation of a biological sample (biosample) in order to make the data FAIR, Findable, Accessible, Interoperable and Reusable, to the extent that such a service benefits the network and scientific community at large. -* Many 4DN experiments are performed using cell lines. Other experiments may be performed on isolated primary cells or tissues. -* Experimenters may also perform assays where cells are transiently treated, for example by addition of a drug or introduction of a silencing construct, or stably genomically modified through Crispr technologies. - -This page outlines and describes the types of metadata that is requested for biological samples. - - -* The first part of the document outlines the few fields shared by all biosamples. -* The Cell Lines and Samples Working Group has focused on developing requirements for cell line metadata and this is the primary focus of the `remainder of this document <#basic-biosample-metadata>`_. - -*Note that the working group is still discussing some of the metadata and requirements are evolving. If you have any questions or concerns please feel free to `contact us `_.* - -Basic Biosample Metadata -^^^^^^^^^^^^^^^^^^^^^^^^ - -Biosample Fields -^^^^^^^^^^^^^^^^ - -``description`` - **Required** {:.text-400} -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -* A brief specific description of the biosample -* example "K562 cells prepared for in situ Hi-C experiments" -* example "GM12878 cells modified using Crispr to delete CTCF sites in the PARK2 region prepared for chromatin capture experiments" - -``biosource`` - **Required** {:.text-400} -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -* The value of this field is a reference to usually one **Biosource** object whose metadata is submitted separately. -* This **Biosource** object refers to a cell line, tissue or primary cell and has its own associated metadata. - - * **NOTE**\ : The tiered cell lines all have an existing biosource in the database that can be re-used and referenced by it's accession, alias or uuid - while other biosources may require you to submit metadata for them. - -* It is possible, though rare, for a single biosample to consist of more than one biosource - eg. pooling of two different cell lines - in these cases you can reference multiple biosources in this field. - -``cell_culture_details`` - **Required only for cultured cell lines** {:.text-400} -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -* The value of this field is a reference to a *BiosampleCellCulture* object whose metadata is submitted separately and is detailed in the *Cell Culture Metadata section below*. - -``modifications`` - **Required** if cells have been genomically modified {:.text-400} -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -* **Genetic modifications** - this field is **required** when a Biosample has been genomically modified eg. Crispr modification of a cell line. -* The value of this field is a list of one or more references to a **Modification** object whose metadata is submitted separately. -* Modifications include information on expression or targeting vectors stably transfected to generate Crispr'ed or other genomically modified samples. - -``treatments`` - **Required** if cells have been treated 'transiently' with drugs or by transfection. {:.text-400} -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -* This field is used when a Biosample has been treated with a chemical/drug or transiently altered using RNA interference techniques. -* The value of this field is a reference to a **Treatment** object whose metadata is submitted separately. -* There are currently two general types of treatments - more will be added as needed. - - #. Addition of a drug or chemical - #. Transient or inducible RNA interference - -``biosample_protocols`` - Optional {:.text-400} -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -* Protocols used in Biosample Preparation - this is distinct from SOPs and protocol for cell cultures. -* example protocol description "Preparation of isolated brain tissue from BALB/c adult mice for chromatin capture experiments" -* The value of this field is a list of references to a **Protocol** object - an alias or uuid. -* The **Protocol** object can include an attachment to a pdf document describing the steps of the preparation. -* The **Protocol** object is of type 'Biosample preparation protocol' and can be further classified as 'Tissue Preparation Methods' if applicable. - -Cell Culture Metadata -^^^^^^^^^^^^^^^^^^^^^ - - -* The consortium has designated 4 cell lines as `Tier 1 `_\ , which will be a primary focus of 4DN research and integrated analysis. -* A number of other lines that are expected to be used by multiple labs and have approved SOPs for maintaining them have been designated `Tier 2 `_. -* In addition, some labs may wish to submit datasets produced using other cell lines. - -To maintain consistent data standards and in order to facilitate integrated analysis the Cell Lines and Samples Working Group has adopted the following policy. - -Certain types of metadata, if not submitted will prevent your data from being flagged “gold standard”. For your data to be considered “gold standard”, you will need to obtain your cells from the approved source and grow them precisely according to the approved SOP and include the following required information: - - -#. A light microscopy image (DIC or phase contrast) of the cells at the time of harvesting (omics) or under each experimental condition (imaging); -#. culture start date, culture harvest date, culture duration, passage number and doubling number - -Other metadata is strongly encouraged and the exact requirements may vary somewhat depending on the cell type and when the data was produced (i.e. some older experiments can be 'grandfathered' in even if they do not 'pass' all the requirements). - -The biosample cell culture metadata fields that can be submitted are described below. - -BiosampleCellCulture fields -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -``description`` - Strongly Encouraged {:.text-400} -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -* A short description of the cell culture procedure -* example "Details on culturing a preparation of K562 cells" - -``morphology_image`` - **Required** {:.text-400} -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -* Phase Contrast or DIC Image of at least 50 cells showing morphology at the time of collection -* This is an authentication standard particularly relevant to Tiered cell lines. -* The value of this field is a reference to an **Image** object that needs to be submitted separately. - -``culture_start_date`` - **Required** {:.text-400} -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -* The date the the cells were most recently thawed and cultured for the submitted experiment -* Date can be submitted in as YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSTZD ((TZD is the time zone designator; use Z to express time in UTC or for time expressed in local time add a time zone offset from UTC +HH:MM or -HH:MM). -* example Date only (most common use case) - "2017-01-01" -* example Date and Time (uncommonly used) -"2017-01-01T17:00:00+00:00" - note for time; hours, minutes, seconds and offset are required but may be 00 filled. - -``culture_harvest_date`` - **Required** {:.text-400} -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -* The date the culture was harvested for biosample preparation. -* Date format as above. - -``culture_duration`` - **Required** {:.text-400} -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -* Total Days in Culture. -* Total number of culturing days since receiving original vial, including pyramid stocking and expansion since thawing the working stock, through to harvest date. -* The field value is a number - can be floating point -* example "5" -* example "3.5" - -``passage_number`` - **Required** {:.text-400} -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -* Number of passages since receiving original vial, including pyramid stocking and expansion since thawing the working stock, through to harvest date. -* Only integer values are allowed in this field eg. 3, 5, 11 - -``doubling_number`` - **Required** {:.text-400} -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -* The number of times the population has doubled since the time of thaw (culture start date) until harvest. -* This may be determined and reported in different ways - - #. passage ratio and number of passages - #. direct cell counts. - -* Therefore, this field takes a string value -* example "7.88" -* example "5 passages split 1:4" - -``follows_sop`` - **Required** {:.text-400} -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -* Flag to indicate if the 4DN SOP for the specified cell line was followed - options '\ **Yes**\ ' or '\ **No**\ ' -* If a cell line is not one of the 'Tiered' 4DN lines this field should be set to 'No' - -``protocols_additional`` - **Required** if 'follows_sop' is 'No' {:.text-400} -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -* Protocols used in Cell Culture when there is deviation from a 4DN approved SOP. -* Protocols describing non-4DN protocols or deviations from 4DN SOPs, including additional culture manipulations eg. stem cell differentiation or cell cycle synchronization if they do not follow recommended 4DN SOPs -* The value of this field is a list of references to a **Protocol** object - an alias or uuid. -* The **Protocol** object can include an attachment to the pdf document. - -``doubling_time`` - Optional {:.text-400} -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -* Population Doubling Time -* The average time from thaw (culture start date) until harvest it takes for the population to double. -* Researchers can record the number of times they split the cells and by what ratio as a simple approximation of doubling time. This is especially important for some cell lines eg. IMR90 (a mortal line) and HI and H9 human stem cells. -* eg. '2 days' - -``authentication_protocols`` - Optional {:.text-400} -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -* References to one or more **Protocol** objects can be submitted in this field. -* The **Protocol** objects should be of the type 'Authentication document' -* The **Protocol** object can be further classified by indicating a specific classification eg. 'Karyotyping authentication' or 'Differentiation authentication'. -* The **Protocol** description should include specific information on the kind of authentication - - * example "g-banding karyotype report" - * example "images of FoxA2 and Sox17 expression in differentiated endoderm cells" - -* The **Protocol** object can include an attachment to the pdf or image document. - -``karyotype`` - Optional description of cell ploidy and karyotype {:.text-400} -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -* Description of cell Ploidy - a textual description of the population ploidy and/or karyotype. -* Important for potentially genomically unstable lines and strongly encouraged if the passage number of an unstable line is greater than 10. -* A textual description of chromosome count and any noted rearrangements or copy number variations. -* examples include - - * chromosome counts or structural variation using sequencing data - * chromosome counts using droplet PCR - * cytological G-banding - -* Using this field allows this information to be queried in searches. -* **NOTE** An image or authentication document (see above) may be submitted in place or in addition to this. - -``differentiation_state`` - Optional {:.text-400} -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -* For cells that have undergone differentiation a description of the differention state and markers used to determine the state. -* Using this field allows this information to be queried in searches. -* example 'Definitive endoderm as determined by the expression of Sox17 and FoxA2' -* **NOTE** An authentication document (see above) can be submitted in place or in addition to this. - -``synchronization_stage`` - Optional {:.text-400} -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -* If a culture is synchronized then the cell cycle stage or description of the point from which the biosample used in an experiment is prepared. -* Using this field allows this information to be queried in searches. -* example 'M-phase metaphase arrested cells' -* **NOTE** An authentication document (see above) can be submitted in place or in addition to this. - -``cell_line_lot_number`` - Strongly Suggested for non-Tier 1 cells {:.text-400} -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -* For 4DN Tier2 or unclassified cell lines - a lot number or other information to uniquely identify the source/lot of the cells diff --git a/docs/source/docker-local.rst b/docs/source/docker-local.rst index 85a4904bc2..bd84a70015 100644 --- a/docs/source/docker-local.rst +++ b/docs/source/docker-local.rst @@ -1,12 +1,13 @@ CGAP-Docker (local) =================== -It is now possible to run a local deployment of CGAP without installing any system level +With Docker, it is possible to run a local deployment of CGAP without installing any system level dependencies other than Docker. A few important notes on this setup. -* This is not ideal for active development as you cannot run unit tests or edit source files in the container on the host machine (in your local editor). +* Although the build dependency layer is cached, it still takes around 4 minutes to rebuild the front-end for each image. This limitation is tolerable considering the local deployment now identically matches the execution runtime of production. * ElasticSearch is too compute intensive to virtualize on most machines. For this reason we use the CGAP test ES cluster for this deployment instead of spinning up an ES cluster in Docker. If you want to attempt to run containerized ES, see ``docker-compose.yml``. -* VERY IMPORTANT: Do not upload the local deployment container image to any registry. This utility is in beta - . +* This setup only works when users have sourced AWS Keys in the main account (to connect to the shared ES cluster). +* IMPORTANT: Do not upload the local deployment container image to any registry. Start by installing Docker:: @@ -14,9 +15,9 @@ Start by installing Docker:: $ brew install docker -Prior to building the image, navigate to deploy/docker/local and open docker_development.ini +Prior to building the image, navigate to deploy/docker/local and open docker_development.ini. -* Modify env.name and indexer.namespace - these values must be globally unique (feel free to just replace the name) +* Modify env.name and indexer.namespace - these values must be globally unique with respect to our infrastructure (feel free to just replace the name) * Consider changing load_prod_data to load_local_data if you need to load more inserts There are two new Make targets that should be sufficient for normal use. To build the image locally, ensure your diff --git a/docs/source/index.rst b/docs/source/index.rst index a8c1a01c15..afcac1415e 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -42,6 +42,7 @@ Be warned that features are under active development and may not be stable! Visi dataflow_overview ingestion docker-local + docker-production *Data Model* diff --git a/docs/source/introduction.rst b/docs/source/introduction.rst deleted file mode 100644 index 7cb9dd8ebf..0000000000 --- a/docs/source/introduction.rst +++ /dev/null @@ -1,50 +0,0 @@ -Introduction ------------- - -* The 4DN Data Portal will be the primary access point to the omics and imaging data, analysis tools, and integrative models - generated and utilized by the 4DN Network. -* The primary high level organizing principle for the data is sets of replicate experiments. -* A good entry point for exploring available data is the `Browse Page `_. -* See `below <#metadata-structure>`_ for an overview of our metadata model. -* As of September 2017, the portal is currently open to the network - for data submission for standard functional genomics experiments (Hi-C and - variants, ChIA-PET and variants, RNA-seq, ChIP-seq, - ATAC-seq). -* Continuing developments in the metadata model and data portal are ongoing. - -Notes for prospective submitters -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If you would like submit data to the portal: - - -* You will need to `create a user account `_. -* Please skim through the `metadata structure <#metadata-structure>`_. -* Check out the other pages in the **Help** menu for detailed information on the submission process. -* Of note are the required metadata for the biological samples used in experiments, which is specified `on this page `_. -* We can set up a webex call to discuss the details and the most convenient approach for your existing system. - -Metadata Structure ------------------- - -The DCIC, with input from different 4DN Network Working groups, has defined a metadata structure to describe: - - -* biological samples -* experimental methods -* data files -* analysis steps -* and other pertinent data. - -The framework for the the metadata structure is based on the work of -`ENCODE DCC `_. - -The metadata is organized as objects that are related with each other. - -In our database: - - -* The objects are stored in the `JavaScript Object Notation format `_. -* The schemas for the different object types are described in `JSON-LD format `_. -* The json schemas can be found `here `_. -* A documentation of the metadata schema is also available as a google doc `here `_. diff --git a/docs/source/local_installation.rst b/docs/source/local_installation.rst index e0e75fc0a1..55335197ae 100644 --- a/docs/source/local_installation.rst +++ b/docs/source/local_installation.rst @@ -1,10 +1,20 @@ Local Installation ================== +Docker Instructions +^^^^^^^^^^^^^^^^^^^ + +See `docker local docs. <./docker-local.rst>`_ + +Legacy Instructions +^^^^^^^^^^^^^^^^^^^ + The following instructions are for running a CGAP deployment with macOS and homebrew. +Note that as of summer 2021, these instructions are out of date. Please refer to the Docker setup. There are no guarantees the legacy instructions will work from this point forward. + CGAP is known to work with Python 3.6.x and will not work with Python 3.7 or greater. If part of the HMS team, it is -recommended to use Python 3.6.13, since that's what is running on our servers. It is best practice to create a fresh Python +recommended to use Python 3.6.13, since that's what is running on our servers. It is best practice to create a fresh Python virtualenv using one of these versions before proceeding to the following steps. * Step 0: Obtain AWS keys. These will need to added to your environment variables or through the AWS CLI (installed later in this process). From a9a93dbb15b57d6a15ccf92b637efefddf779db8 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Mon, 24 May 2021 13:01:16 -0400 Subject: [PATCH 079/120] add more documentation --- docs/source/docker-local.rst | 35 +++++++++++++++---------------- docs/source/docker-production.rst | 28 ++++++++++++++----------- 2 files changed, 33 insertions(+), 30 deletions(-) diff --git a/docs/source/docker-local.rst b/docs/source/docker-local.rst index bd84a70015..fb968ebc7c 100644 --- a/docs/source/docker-local.rst +++ b/docs/source/docker-local.rst @@ -1,4 +1,4 @@ -CGAP-Docker (local) +CGAP-Docker (Local) =================== With Docker, it is possible to run a local deployment of CGAP without installing any system level @@ -15,23 +15,20 @@ Start by installing Docker:: $ brew install docker -Prior to building the image, navigate to deploy/docker/local and open docker_development.ini. +Prior to building the image, navigate to ``deploy/docker/local``, open ``docker_development.ini`` and make the following modifications (at a minimum). * Modify env.name and indexer.namespace - these values must be globally unique with respect to our infrastructure (feel free to just replace the name) * Consider changing load_prod_data to load_local_data if you need to load more inserts +* Once you have loaded inserts once, comment out L54 in ``docker-compose.yml`` to disable automatic insert reloading -There are two new Make targets that should be sufficient for normal use. To build the image locally, ensure your -AWS keys are sourced and run:: +There are two new Make targets that should be sufficient for normal use. To build the image locally, ensure your AWS keys are sourced and run:: $ make build-docker # runs docker-compose build + $ make build-docker-clean # runs a no-cache build, regenerating all layers $ make deploy-docker # runs docker-compose up -The first command will take awhile the first time you run it but should speed up after. Since it is doing a fresh -rebuild every time it is a little slower than the old local deployment since it has to fully reinstall/rebuild both Python -and the client. Because of this, it is recommended to continue active development using the existing installation setup. -Once the branch is ready for "integrated" testing, set the desired branch in ``docker-compose.yml`` and trigger a build. -When the app is brought online the behavior should be identical to that of the existing local deployment setup. It will -also match the production behavior in ECS. +The build will take around 10 minutes the first time but will speed up dramatically after due to layer caching. In general, the rate limiting step for rebuilding is the front-end build (unless you are also updating dependencies, which will slow down the build further). Although this may seem like a drawback, the key benefit is that what you are running in Docker is essentially identical to that which is orchestrated on ECS in production. This should reduce our reliance/need for test environments. + To access the running container:: @@ -39,6 +36,15 @@ To access the running container:: $ docker exec -it bash +Common Issues +^^^^^^^^^^^^^ + +Some notable issues that you may encounter include: + + * The NPM build may fail/hang - this can happen when Docker does not have enough resources. Try upping the amount CPU/RAM you are allocating to Docker. + * Nginx install fails to locate GPG key - this happens when the Docker internal cache has run out of space and needs to be cleaned - see documentation on `docker prune `_. + + Docker Command Cheatsheet ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -51,11 +57,4 @@ Below is a small list of useful Docker commands for advanced users:: $ docker-compose up # will start cluster and log all output to console $ docker-compose up -d # will start cluster in background using existing containers $ docker-compose up -d -V --build # trigger a rebuild/recreation of cluster containers - $ docker system prune # will cleanup unused Docker components - BE CAREFUL WITH THIS - -Note that the Dockerfile's take arguments (set these in docker-compose.yml): - * CGAP_ENV_NAME="cgap-mastertest" - * CGAP_REPO="https://github.com/dbmi-bgm/cgap-portal.git" - * CGAP_BRANCH="master" - * ENTRYPOINT="entrypoint.sh" - + $ docker system prune # will cleanup ALL unused Docker components - BE CAREFUL WITH THIS diff --git a/docs/source/docker-production.rst b/docs/source/docker-production.rst index c008ee182c..1031ba3f0f 100644 --- a/docs/source/docker-production.rst +++ b/docs/source/docker-production.rst @@ -1,4 +1,4 @@ -CGAP-Docker (production) +CGAP-Docker (Production) ======================== CGAP-Docker runs in production on AWS Elastic Container Service, meant to be orchestrated from the 4dn-cloud-infra repository. End users will modify ``deploy/docker/production/Makefile`` to suite their immediate build needs with respect to target AWS Account/ECR Repository/Tagging strategy. For more information on the specifics of the ECS setup, see 4dn-cloud-infra. @@ -15,23 +15,27 @@ Building an Image The production application configuration is in ``deploy/docker/production``. A description of all the relevant files follows. - * assume_identity.py - script for pulling application configuration from Secrets Manager - * Dockerfile - production Dockerfile, essentially identical to local deployment except the container does not run as root + * Dockerfile - at repo top level - configurable file that builds both all local and production images. + * docker-compose.yml - at repo top level - configures the local deployment - unused in production. + * assume_identity.py - script for pulling application configuration from Secrets Manager. Note that this secret is meant to be generated by the Datastore stack in 4dn-cloud-infra and manually filled out. * entrypoint.sh - WSGI entrypoint - * entrypoint_deploymentn.sh - deployment entrypoint + * entrypoint_deployment.sh - deployment entrypoint * entrypoint_indexer.sh - indexer entrypoint * entrypoint_ingester.sh - ingester entrypoint * install_nginx.sh - script for pulling in nginx - * Makefile - configures builds/pushes for relevant images + * Makefile - configures builds/pushes for relevant images/tags * mastertest.ini - base ini file used to build production.ini on the server * nginx.conf - nginx configuration -The following instructions describe how to build/push images. Note though that we assume an existing ECS setup. For instructions on how to orchestrate ECS, see 4dn-cloud-infra, but that is not the focus of this documentation. +The following instructions describe how to build and push images. Note though that we assume an existing ECS setup. For instructions on how to orchestrate ECS, see 4dn-cloud-infra, but that is not the focus of this documentation. - * Ensure the orchestrator credentials are sourced, or that your IAM user has been granted sufficient perms to push to ECR. - * In the Makefile, replace "cgap-mastertest" with the env.name configured for the environment. This name should match the ECR repo name if you navigate to the ECR Console. - * Again in the Makefile, replace the ECR Repo URL (NOT the tags) with the one from the output of the ECR stack in the account. - * Run ``make login``, which should pull ECR credentials using the currently active AWS credentials. - * Run ``make info`` for information on tags. - * Run the appropriate make target to build/deploy the desired version by pushing a new image to ECR. Note that the upload process may take a long time if you made application code (non-configuration) changes. + 1. Ensure the orchestrator credentials are sourced, or that your IAM user has been granted sufficient perms to push to ECR. + 2. In the Makefile, replace "cgap-mastertest" with the env.name configured for the environment. This name should match the ECR repo name if you navigate to the ECR Console. + 3. Again in the Makefile, replace the ECR Repo URL (NOT the tags) with the one from the output of the ECR stack in the account. + 4. Run ``make login``, which should pull ECR credentials using the currently active AWS credentials. + 5. Run ``make info`` for information on tags. + 6. Run the appropriate make target to build/deploy the desired version by pushing a new image to ECR. Note that the upload process may take a long time if you made application code (non-configuration) changes. + + +Note that steps 1, 4 and 6 are all that are needed to be repeated once initial setup is done, assuming you are continuously pushing to the same location. To change which ECS orchestration you are effectively deploying to, all steps must be repeated in the relevant account. From b09a25eaddfe13d4ac3623708483f9150cba5b62 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Mon, 24 May 2021 13:31:21 -0400 Subject: [PATCH 080/120] add more info --- docs/source/docker-production.rst | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/source/docker-production.rst b/docs/source/docker-production.rst index 1031ba3f0f..653636cc64 100644 --- a/docs/source/docker-production.rst +++ b/docs/source/docker-production.rst @@ -39,3 +39,29 @@ The following instructions describe how to build and push images. Note though th Note that steps 1, 4 and 6 are all that are needed to be repeated once initial setup is done, assuming you are continuously pushing to the same location. To change which ECS orchestration you are effectively deploying to, all steps must be repeated in the relevant account. + + +Tagging Strategy +^^^^^^^^^^^^^^^^ + +As stated previously, the cgap-portal consists of 4 core components that have been translated into ECS Services/Tasks. The task definitions specify image tags that differentiate each of the 4 components. Those tags are: + + * ``latest`` - push to this tag to modify the WSGI image + * ``latest-indexer`` - push to this tag to modify the indexer image + * ``latest-ingester`` - push to this tag to modify the ingester image + * ``latest-deployment`` - push to this tag to modify the deployment image + +After all new image versions have been pushed, issue a forced deployment update to the ECS cluster. This action will spawn a new set of tasks for all services using the newer image tags. For WSGI, once the new tasks are deemed healthy by ECS and the Application Load Balancer, they will be added to the WSGI Target Group and immediately begin serving requests. At that time the old tasks will begin the de-registration process from the target group, after which they will be spun down. The remaining new tasks will come online more quickly since they do not need to pass load balancer health checks. Once the old tasks have been cleaned up, it is safe to trigger a deployment task through the Deployment Service. + +Deciding what Image(s) to Update +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Generally speaking, all 4 tags must be pushed in order for the deployment to be considered "valid". With that said, there are a few scenarios where it may make sense to do a partial image bundle upload. Note that when uploading to production, none of the following apply. + + * If you are only making front-end changes, feel free to push only the ``latest`` tag. + * If you are not ingesting VCF files or modifying the ingestion pipeline, you do not need to update the ``latest-ingester`` image. + * If you want to phase back-end changes in prior to releasing them on the front-end, push ``latest-indexer`` and ``latest-deployment`` images. + * If you are only modifying the deployment commands ie: ``entrypoint_deployment.sh``, then you only need to update the ``latest-deployment`` tag. + + +Note that these images have implied dependencies on each other that are not obvious. We consider the deployment and indexer images to be bundled together in the sense that updating one without the other will trigger undefined behavior if the deployment is triggered. It is thus imperative that, at a bare minimum before triggering an ECS Cluster update, that both the ``latest-indexer`` and ``latest-deployment`` tags have been updated in the case that any back-end changes have been made, whether it be to application, it's dependencies or the Dockerfile itself. From 3b00329820e1acbb40849acb0cd60ab2fdc44bc9 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Mon, 24 May 2021 14:50:15 -0400 Subject: [PATCH 081/120] cleanup some more unused stuff --- examples/s3cp.py | 51 ------ examples/submit_file.py | 188 ----------------------- jest/environment.js | 20 --- parts/production-indexer/wsgi | 24 --- parts/production-ingestion-listener/wsgi | 24 --- parts/production/wsgi | 24 --- 6 files changed, 331 deletions(-) delete mode 100644 examples/s3cp.py delete mode 100644 examples/submit_file.py delete mode 100644 jest/environment.js delete mode 100644 parts/production-indexer/wsgi delete mode 100644 parts/production-ingestion-listener/wsgi delete mode 100755 parts/production/wsgi diff --git a/examples/s3cp.py b/examples/s3cp.py deleted file mode 100644 index 4c3cc90b0f..0000000000 --- a/examples/s3cp.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python -# -*- coding: latin-1 -*- -import requests, subprocess, shlex, urlparse, os, sys - -AUTHID='user'; AUTHPW='secret'; HEADERS = {'content-type': 'application/json'}; SERVER = 'https://www.encodeproject.org/' -S3_SERVER='s3://encode-files/' - -#get all the file objects -files = requests.get( - 'https://www.encodeproject.org/search/?type=file&frame=embedded&limit=all', - auth=(AUTHID,AUTHPW), headers=HEADERS).json()['@graph'] - -#select your file -f_obj = files[123] - -#make the URL that will get redirected - get it from the file object's href property -encode_url = urlparse.urljoin(SERVER,f_obj.get('href')) - -#stream=True avoids actually downloading the file, but it evaluates the redirection -r = requests.get(encode_url, auth=(AUTHID,AUTHPW), headers=HEADERS, allow_redirects=True, stream=True) -try: - r.raise_for_status -except Exception: - print '%s href does not resolve' %(f_obj.get('accession')) - sys.exit() - -#this is the actual S3 https URL after redirection -s3_url = r.url - -#release the connection -r.close() - -#split up the url into components -o = urlparse.urlparse(s3_url) - -#pull out the filename -filename = os.path.basename(o.path) - -#hack together the s3 cp url (with the s3 method instead of https) -bucket_url = S3_SERVER.rstrip('/') + o.path -#print bucket_url - -#ls the file from the bucket -s3ls_string = subprocess.check_output(shlex.split('aws s3 ls %s' %(bucket_url))) -if s3ls_string.rstrip() == "": - print >> sys.stderr, "%s not in bucket" %(bucket_url) -else: - print "%s %s" %(f_obj.get('accession'), s3ls_string.rstrip()) - -#do the actual s3 cp -#return_value = subprocess.check_call(shlex.split('aws s3 cp %s %s' %(bucket_url, filename))) diff --git a/examples/submit_file.py b/examples/submit_file.py deleted file mode 100644 index 4ca2a96d5e..0000000000 --- a/examples/submit_file.py +++ /dev/null @@ -1,188 +0,0 @@ -""" Example file submission script - -Requires the `aws` command line utility: http://aws.amazon.com/cli/ -""" -import hashlib -import json -import os -import requests -import subprocess -import sys -import time - -host = 'http://localhost:6543' -encoded_access_key = '...' -encoded_secret_access_key = '...' - -path = 'example.fastq.gz' -my_lab = '/labs/your-lab-here' -my_award = '/awards/your-award-here' - -# From http://hgwdev.cse.ucsc.edu/~galt/encode3/validatePackage/validateEncode3-latest.tgz -encValData = 'encValData' -assembly = 'hg19' - -# ~2s/GB -print("Calculating md5sum.") -md5sum = hashlib.md5() -with open(path, 'rb') as f: - for chunk in iter(lambda: f.read(1024*1024), b''): - md5sum.update(chunk) - -data = { - "dataset": "ENCSR000ACY", - "replicate": "/replicates/6e85c807-684a-46e3-b4b9-1f7990e85720/", - "file_format": "fastq", - "file_size": os.path.getsize(path), - "md5sum": md5sum.hexdigest(), - "output_type": "reads", - "read_length": 101, - "run_type": "single-ended", - "platform": "ENCODE:HiSeq2000", - "submitted_file_name": path, - "lab": my_lab, - "award": my_award -} - - -#################### -# Local validation - -gzip_types = [ - "CEL", - "bam", - "bed", - "csfasta", - "csqual", - "fasta", - "fastq", - "gff", - "gtf", - "tar", - "sam", - "wig" -] - -magic_number = open(path, 'rb').read(2) -is_gzipped = magic_number == b'\x1f\x8b' -if data['file_format'] in gzip_types: - assert is_gzipped, 'Expected gzipped file' -else: - assert not is_gzipped, 'Expected un-gzipped file' - -chromInfo = '-chromInfo=%s/%s/chrom.sizes' % (encValData, assembly) -validate_map = { - ('fasta', None): ['-type=fasta'], - ('fastq', None): ['-type=fastq'], - ('bam', None): ['-type=bam', chromInfo], - ('bigWig', None): ['-type=bigWig', chromInfo], - ('bed', 'bed3'): ['-type=bed3', chromInfo], - ('bigBed', 'bed3'): ['-type=bed3', chromInfo], - ('bed', 'bed6'): ['-type=bed6+', chromInfo], # if this fails we will drop to bed3+ - ('bigBed', 'bed6'): ['-type=bigBed6+', chromInfo], # if this fails we will drop to bigBed3+ - ('bed', 'bedLogR'): ['-type=bed9+1', chromInfo, '-as=%s/as/bedLogR.as' % encValData], - ('bigBed', 'bedLogR'): ['-type=bigBed9+1', chromInfo, '-as=%s/as/bedLogR.as' % encValData], - ('bed', 'bedMethyl'): ['-type=bed9+2', chromInfo, '-as=%s/as/bedMethyl.as' % encValData], - ('bigBed', 'bedMethyl'): ['-type=bigBed9+2', chromInfo, '-as=%s/as/bedMethyl.as' % encValData], - ('bed', 'broadPeak'): ['-type=bed6+3', chromInfo, '-as=%s/as/broadPeak.as' % encValData], - ('bigBed', 'broadPeak'): ['-type=bigBed6+3', chromInfo, '-as=%s/as/broadPeak.as' % encValData], - ('bed', 'gappedPeak'): ['-type=bed12+3', chromInfo, '-as=%s/as/gappedPeak.as' % encValData], - ('bigBed', 'gappedPeak'): ['-type=bigBed12+3', chromInfo, '-as=%s/as/gappedPeak.as' % encValData], - ('bed', 'narrowPeak'): ['-type=bed6+4', chromInfo, '-as=%s/as/narrowPeak.as' % encValData], - ('bigBed', 'narrowPeak'): ['-type=bigBed6+4', chromInfo, '-as=%s/as/narrowPeak.as' % encValData], - ('bed', 'bedRnaElements'): ['-type=bed6+3', chromInfo, '-as=%s/as/bedRnaElements.as' % encValData], - ('bigBed', 'bedRnaElements'): ['-type=bed6+3', chromInfo, '-as=%s/as/bedRnaElements.as' % encValData], - ('bed', 'bedExonScore'): ['-type=bed6+3', chromInfo, '-as=%s/as/bedExonScore.as' % encValData], - ('bigBed', 'bedExonScore'): ['-type=bigBed6+3', chromInfo, '-as=%s/as/bedExonScore.as' % encValData], - ('bed', 'bedRrbs'): ['-type=bed9+2', chromInfo, '-as=%s/as/bedRrbs.as' % encValData], - ('bigBed', 'bedRrbs'): ['-type=bigBed9+2', chromInfo, '-as=%s/as/bedRrbs.as' % encValData], - ('bed', 'enhancerAssay'): ['-type=bed9+1', chromInfo, '-as=%s/as/enhancerAssay.as' % encValData], - ('bigBed', 'enhancerAssay'): ['-type=bigBed9+1', chromInfo, '-as=%s/as/enhancerAssay.as' % encValData], - ('bed', 'modPepMap'): ['-type=bed9+7', chromInfo, '-as=%s/as/modPepMap.as' % encValData], - ('bigBed', 'modPepMap'): ['-type=bigBed9+7', chromInfo, '-as=%s/as/modPepMap.as' % encValData], - ('bed', 'pepMap'): ['-type=bed9+7', chromInfo, '-as=%s/as/pepMap.as' % encValData], - ('bigBed', 'pepMap'): ['-type=bigBed9+7', chromInfo, '-as=%s/as/pepMap.as' % encValData], - ('bed', 'openChromCombinedPeaks'): ['-type=bed9+12', chromInfo, '-as=%s/as/openChromCombinedPeaks.as' % encValData], - ('bigBed', 'openChromCombinedPeaks'): ['-type=bigBed9+12', chromInfo, '-as=%s/as/openChromCombinedPeaks.as' % encValData], - ('bed', 'peptideMapping'): ['-type=bed6+4', chromInfo, '-as=%s/as/peptideMapping.as' % encValData], - ('bigBed', 'peptideMapping'): ['-type=bigBed6+4', chromInfo, '-as=%s/as/peptideMapping.as' % encValData], - ('bed', 'shortFrags'): ['-type=bed6+21', chromInfo, '-as=%s/as/shortFrags.as' % encValData], - ('bigBed', 'shortFrags'): ['-type=bigBed6+21', chromInfo, '-as=%s/as/shortFrags.as' % encValData], - ('rcc', None): ['-type=rcc'], - ('idat', None): ['-type=idat'], - ('bedpe', None): ['-type=bed3+', chromInfo], - ('bedpe', 'mango'): ['-type=bed3+', chromInfo], - ('gtf', None): None, - ('tar', None): None, - ('tsv', None): None, - ('csv', None): None, - ('2bit', None): None, - ('csfasta', None): ['-type=csfasta'], - ('csqual', None): ['-type=csqual'], - ('CEL', None): None, - ('sam', None): None, - ('wig', None): None, - ('hdf5', None): None, - ('gff', None): None -} - -validate_args = validate_map.get((data['file_format'], data.get('file_format_type'))) -if validate_args is not None: - print("Validating file.") - try: - subprocess.check_output(['validateFiles'] + validate_args + [path]) - except subprocess.CalledProcessError as e: - print(e.output) - raise - - -#################### -# POST metadata - -headers = { - 'Content-type': 'application/json', - 'Accept': 'application/json', -} - -print("Submitting metadata.") -r = requests.post( - host + '/file', - auth=(encoded_access_key, encoded_secret_access_key), - data=json.dumps(data), - headers=headers, -) -try: - r.raise_for_status() -except Exception: - print('Submission failed: %s %s' % (r.status_code, r.reason)) - print(r.text) - raise -item = r.json()['@graph'][0] -print(json.dumps(item, indent=4, sort_keys=True)) - - -#################### -# POST file to S3 - -creds = item['upload_credentials'] -env = os.environ.copy() -env.update({ - 'AWS_ACCESS_KEY_ID': creds['access_key'], - 'AWS_SECRET_ACCESS_KEY': creds['secret_key'], - 'AWS_SECURITY_TOKEN': creds['session_token'], -}) - -# ~10s/GB from Stanford - AWS Oregon -# ~12-15s/GB from AWS Ireland - AWS Oregon -print("Uploading file.") -start = time.time() -try: - subprocess.check_call(['aws', 's3', 'cp', path, creds['upload_url']], env=env) -except subprocess.CalledProcessError as e: - # The aws command returns a non-zero exit code on error. - print("Upload failed with exit code %d" % e.returncode) - sys.exit(e.returncode) -else: - end = time.time() - duration = end - start - print("Uploaded in %.2f seconds" % duration) diff --git a/jest/environment.js b/jest/environment.js deleted file mode 100644 index f8a4420f0a..0000000000 --- a/jest/environment.js +++ /dev/null @@ -1,20 +0,0 @@ -'use strict'; - -//jest.mock('scriptjs'); -var jsdom = require('jsdom').jsdom; - -if (window.DOMParser === undefined) { - // jsdom - window.DOMParser = function DOMParser() {}; - window.DOMParser.prototype.parseFromString = function parseFromString(markup, type) { - var parsingMode = 'auto'; - type = type || ''; - if (type.indexOf('xml') >= 0) { - parsingMode = 'xml'; - } else if (type.indexOf('html') >= 0) { - parsingMode = 'html'; - } - var doc = jsdom(markup, {parsingMode: parsingMode}); - return doc; - }; -} diff --git a/parts/production-indexer/wsgi b/parts/production-indexer/wsgi deleted file mode 100644 index 85af513622..0000000000 --- a/parts/production-indexer/wsgi +++ /dev/null @@ -1,24 +0,0 @@ -import os - -from logging.config import fileConfig -from paste.deploy import loadapp - - -# This is sort of like as if we did: -# CONFIG_FILE = "/opt/python/current/app/production.ini" -# except we want to bind to the bundle name so that when links change -# between /opt/python/ondeck/app and /opt/python/current/app -# we continue to point to the right directory. -# NOTE: If this doesn't work, we might try using "../../production.ini" - -FILE_DIR = os.path.dirname(os.path.abspath(__file__)) - -PARTS_DIR = os.path.dirname(FILE_DIR) - -HOME_DIR = os.path.dirname(PARTS_DIR) - -CONFIG_FILE = os.path.join(HOME_DIR, "production.ini") - -fileConfig(CONFIG_FILE) - -application = loadapp("config:" + CONFIG_FILE, name="indexer") diff --git a/parts/production-ingestion-listener/wsgi b/parts/production-ingestion-listener/wsgi deleted file mode 100644 index 6bef8f1f44..0000000000 --- a/parts/production-ingestion-listener/wsgi +++ /dev/null @@ -1,24 +0,0 @@ -import os - -from logging.config import fileConfig -from paste.deploy import loadapp - - -# This is sort of like as if we did: -# CONFIG_FILE = "/opt/python/current/app/production.ini" -# except we want to bind to the bundle name so that when links change -# between /opt/python/ondeck/app and /opt/python/current/app -# we continue to point to the right directory. -# NOTE: If this doesn't work, we might try using "../../production.ini" - -FILE_DIR = os.path.dirname(os.path.abspath(__file__)) - -PARTS_DIR = os.path.dirname(FILE_DIR) - -HOME_DIR = os.path.dirname(PARTS_DIR) - -CONFIG_FILE = os.path.join(HOME_DIR, "production.ini") - -fileConfig(CONFIG_FILE) - -application = loadapp("config:" + CONFIG_FILE, name="ingester") diff --git a/parts/production/wsgi b/parts/production/wsgi deleted file mode 100755 index 742fdec457..0000000000 --- a/parts/production/wsgi +++ /dev/null @@ -1,24 +0,0 @@ -import os - -from logging.config import fileConfig -from paste.deploy import loadapp - - -# This is sort of like as if we did: -# CONFIG_FILE = "/opt/python/current/app/production.ini" -# except we want to bind to the bundle name so that when links change -# between /opt/python/ondeck/app and /opt/python/current/app -# we continue to point to the right directory. -# NOTE: If this doesn't work, we might try using "../../production.ini" - -FILE_DIR = os.path.dirname(os.path.abspath(__file__)) - -PARTS_DIR = os.path.dirname(FILE_DIR) - -HOME_DIR = os.path.dirname(PARTS_DIR) - -CONFIG_FILE = os.path.join(HOME_DIR, "production.ini") - -fileConfig(CONFIG_FILE) - -application = loadapp("config:" + CONFIG_FILE, name=None) From fcb47d077154f5b59e6ddfa7eac509c1c13551d1 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Mon, 24 May 2021 14:52:45 -0400 Subject: [PATCH 082/120] slight updates to readme --- README.rst | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index b96438d6e6..04157ab0cd 100644 --- a/README.rst +++ b/README.rst @@ -34,14 +34,9 @@ For information on CGAP-Docker in production, see `here. <./docs/source/docker-p Navigating this Repository ^^^^^^^^^^^^^^^^^^^^^^^^^^ - * .github/ contains Github Action Workflows + * .github/workflows/ contains Github Action Workflows * bin/ contains the few remaining executables * deploy/docker contains Docker setups (see docker-compose.yml) * docs/ contains documentation - * etc/ is deprecated/unused - * examples/ is deprecated/unused - * jest/ is deprecated/unused - * parts/ contains production WSGI entry points * scripts/ contains misc scripts - * src/ where the code is - * src-docker/ mount point where container code is \ No newline at end of file + * src/ where the code is \ No newline at end of file From 197ebe2709d56a1aabbb2be38ad2fa8e934f652b Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 25 May 2021 13:57:42 -0400 Subject: [PATCH 083/120] work in some feedback from Kent --- docs/source/docker-local.rst | 26 +++++++++++++++++++++++--- docs/source/docker-production.rst | 23 ++++++++++++++++------- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/docs/source/docker-local.rst b/docs/source/docker-local.rst index fb968ebc7c..89a3a60f87 100644 --- a/docs/source/docker-local.rst +++ b/docs/source/docker-local.rst @@ -5,30 +5,44 @@ With Docker, it is possible to run a local deployment of CGAP without installing dependencies other than Docker. A few important notes on this setup. * Although the build dependency layer is cached, it still takes around 4 minutes to rebuild the front-end for each image. This limitation is tolerable considering the local deployment now identically matches the execution runtime of production. -* ElasticSearch is too compute intensive to virtualize on most machines. For this reason we use the CGAP test ES cluster for this deployment instead of spinning up an ES cluster in Docker. If you want to attempt to run containerized ES, see ``docker-compose.yml``. * This setup only works when users have sourced AWS Keys in the main account (to connect to the shared ES cluster). * IMPORTANT: Do not upload the local deployment container image to any registry. -Start by installing Docker:: +Installing Docker +^^^^^^^^^^^^^^^^^ + +Install Docker with (OSX assumed):: $ brew install docker +Configuring CGAP Docker +^^^^^^^^^^^^^^^^^^^^^^^ + + Prior to building the image, navigate to ``deploy/docker/local``, open ``docker_development.ini`` and make the following modifications (at a minimum). -* Modify env.name and indexer.namespace - these values must be globally unique with respect to our infrastructure (feel free to just replace the name) +* Modify ``env.name`` and ``indexer.namespace`` - these values must be globally unique with respect to our infrastructure (feel free to just replace the name) * Consider changing load_prod_data to load_local_data if you need to load more inserts * Once you have loaded inserts once, comment out L54 in ``docker-compose.yml`` to disable automatic insert reloading +Building CGAP Docker +^^^^^^^^^^^^^^^^^^^^ + + There are two new Make targets that should be sufficient for normal use. To build the image locally, ensure your AWS keys are sourced and run:: $ make build-docker # runs docker-compose build $ make build-docker-clean # runs a no-cache build, regenerating all layers $ make deploy-docker # runs docker-compose up + $ make deploy-docker-daemon # runs services in background The build will take around 10 minutes the first time but will speed up dramatically after due to layer caching. In general, the rate limiting step for rebuilding is the front-end build (unless you are also updating dependencies, which will slow down the build further). Although this may seem like a drawback, the key benefit is that what you are running in Docker is essentially identical to that which is orchestrated on ECS in production. This should reduce our reliance/need for test environments. +Accessing CGAP Docker at Runtime +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + To access the running container:: @@ -36,6 +50,12 @@ To access the running container:: $ docker exec -it bash +Alternative Configuration with Local ElasticSearch +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +ElasticSearch is too compute intensive to virtualize on most machines. For this reason we use the CGAP test ES cluster for this deployment instead of spinning up an ES cluster in Docker. It is possible however to modify ``docker-compose.yml`` to spinup a local Elasticsearch + + Common Issues ^^^^^^^^^^^^^ diff --git a/docs/source/docker-production.rst b/docs/source/docker-production.rst index 653636cc64..233ea478d4 100644 --- a/docs/source/docker-production.rst +++ b/docs/source/docker-production.rst @@ -3,12 +3,21 @@ CGAP-Docker (Production) CGAP-Docker runs in production on AWS Elastic Container Service, meant to be orchestrated from the 4dn-cloud-infra repository. End users will modify ``deploy/docker/production/Makefile`` to suite their immediate build needs with respect to target AWS Account/ECR Repository/Tagging strategy. For more information on the specifics of the ECS setup, see 4dn-cloud-infra. -The CGAP Application has been orchestrated into the ECS Service/Task paradigm. As of writing all core application services have their own image tag varied by passing the ``$ENTRYPOINT`` build argument. As such, they are all separate services with the following notable characteristics: - - * WSGI - services standard API requests - 8x parallelization on Fargate Spot - * Indexer - hits /index at 3 second intervals indefinitely - 4x parallelization on Fargate Spot - * Ingester - poll for ingestion tasks from SQS - 1x parallelization TODO add ability to add additional tasks through API - * Deployment - triggers the standard deployment actions - must be explicitly run either through ECS console or TODO through API. +The CGAP Application has been orchestrated into the ECS Service/Task paradigm. As of writing all core application services have their own image tag varied by passing the ``$ENTRYPOINT`` build argument. As such, they are all separate services described by the following table: + ++------------+--------------------------------+-----+------+------+-----+--------------------------+ +| Kind | Use | Num | Spot | vCPU | Mem | Notes | ++============+================================+=====+======+======+=====+==========================+ +| WSGI | Services standard API requests | 8 | Yes | .25 | 512 | Needs autoscaling. | ++------------+--------------------------------+-----+------+------+-----+--------------------------+ +| Indexer | Hits /index at 3sec | 4 | Yes | .25 | 512 | Needs autoscaling. | +| | intervals indefinitely. | | | | | Could use CPU% | ++------------+--------------------------------+-----+------+------+-----+--------------------------+ +| Ingester | Polls SQS for ingestion tasks | 1 | No | .25 | 512 | Need API to add tasks. | ++------------+--------------------------------+-----+------+------+-----+--------------------------+ +| Deployment | Triggers the standard | 0 | Yes | .5 | 1024| Run explicitly or (TODO) | +| | deployment actions. | | | | | by API. | ++------------+--------------------------------+-----+------+------+-----+--------------------------+ Building an Image ^^^^^^^^^^^^^^^^^ @@ -53,7 +62,7 @@ As stated previously, the cgap-portal consists of 4 core components that have be After all new image versions have been pushed, issue a forced deployment update to the ECS cluster. This action will spawn a new set of tasks for all services using the newer image tags. For WSGI, once the new tasks are deemed healthy by ECS and the Application Load Balancer, they will be added to the WSGI Target Group and immediately begin serving requests. At that time the old tasks will begin the de-registration process from the target group, after which they will be spun down. The remaining new tasks will come online more quickly since they do not need to pass load balancer health checks. Once the old tasks have been cleaned up, it is safe to trigger a deployment task through the Deployment Service. -Deciding what Image(s) to Update +Deciding Which Image(s) to Update ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Generally speaking, all 4 tags must be pushed in order for the deployment to be considered "valid". With that said, there are a few scenarios where it may make sense to do a partial image bundle upload. Note that when uploading to production, none of the following apply. From e77a7f90f1ad2aedeb83d7d591439a9662ee7d7b Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Wed, 2 Jun 2021 10:48:15 -0400 Subject: [PATCH 084/120] small changes to deployment --- .dockerignore | 3 +++ deploy/docker/production/entrypoint_deployment.sh | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index 4b548fd91a..d4ba7704a2 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,6 @@ .docker-compose.yml Dockerfile node_modules +.cache +.pytest_cache +aws-ip-ranges.json diff --git a/deploy/docker/production/entrypoint_deployment.sh b/deploy/docker/production/entrypoint_deployment.sh index 089f9c8706..c43a5221c0 100644 --- a/deploy/docker/production/entrypoint_deployment.sh +++ b/deploy/docker/production/entrypoint_deployment.sh @@ -7,7 +7,8 @@ echo "Running a CGAP deployment on the given environment" poetry run python -m assume_identity # Clear db/es since this is the local entry point -poetry run clear-db-es-contents production.ini --app-name app --env $CGAP_ENV_NAME +# 'skip' is provided as an argument so that this step doesn't run +poetry run clear-db-es-contents production.ini --app-name app --env skip ## Create mapping poetry run create-mapping-on-deploy production.ini --app-name app From 63590d73799488775254dba95876ac7988c5e14e Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Wed, 2 Jun 2021 11:59:01 -0400 Subject: [PATCH 085/120] disable load data --- deploy/docker/production/entrypoint_deployment.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/docker/production/entrypoint_deployment.sh b/deploy/docker/production/entrypoint_deployment.sh index c43a5221c0..7b0c5b2019 100644 --- a/deploy/docker/production/entrypoint_deployment.sh +++ b/deploy/docker/production/entrypoint_deployment.sh @@ -14,7 +14,8 @@ poetry run clear-db-es-contents production.ini --app-name app --env skip poetry run create-mapping-on-deploy production.ini --app-name app # Load Data (based on development.ini, for now just master-inserts) -poetry run load-data production.ini --app-name app --prod +# Not necessary after first deploy +# poetry run load-data production.ini --app-name app --prod # Load access keys # Note that the secret name must match that which was created for this environment From 4a0609882f61377a1148841305ffbc2fc2e35d75 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Wed, 2 Jun 2021 13:25:06 -0400 Subject: [PATCH 086/120] bring in revision history --- poetry.lock | 20 ++++++++++++++++---- pyproject.toml | 2 +- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index ea76dcb8f3..52b8170d78 100644 --- a/poetry.lock +++ b/poetry.lock @@ -533,7 +533,7 @@ python-versions = "*" [[package]] name = "dcicsnovault" -version = "4.7.6" +version = "4.8.0.0b2" description = "Storage support for 4DN Data Portals." category = "main" optional = false @@ -2094,7 +2094,7 @@ test = ["zope.testing"] [metadata] lock-version = "1.1" python-versions = ">=3.6.1,<3.7" -content-hash = "1a7ffdfee7346742528b4ce9387fcc3b37f0fd851c6d401ef65b2ccf0fc716e8" +content-hash = "d24c39534ce21435226126346309035bc7fee7a10f92e549d42305d743aa181f" [metadata.files] apipkg = [ @@ -2170,24 +2170,36 @@ cffi = [ {file = "cffi-1.14.5-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:48e1c69bbacfc3d932221851b39d49e81567a4d4aac3b21258d9c24578280058"}, {file = "cffi-1.14.5-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:69e395c24fc60aad6bb4fa7e583698ea6cc684648e1ffb7fe85e3c1ca131a7d5"}, {file = "cffi-1.14.5-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:9e93e79c2551ff263400e1e4be085a1210e12073a31c2011dbbda14bda0c6132"}, + {file = "cffi-1.14.5-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:24ec4ff2c5c0c8f9c6b87d5bb53555bf267e1e6f70e52e5a9740d32861d36b6f"}, + {file = "cffi-1.14.5-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3c3f39fa737542161d8b0d680df2ec249334cd70a8f420f71c9304bd83c3cbed"}, + {file = "cffi-1.14.5-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:681d07b0d1e3c462dd15585ef5e33cb021321588bebd910124ef4f4fb71aef55"}, {file = "cffi-1.14.5-cp36-cp36m-win32.whl", hash = "sha256:58e3f59d583d413809d60779492342801d6e82fefb89c86a38e040c16883be53"}, {file = "cffi-1.14.5-cp36-cp36m-win_amd64.whl", hash = "sha256:005a36f41773e148deac64b08f233873a4d0c18b053d37da83f6af4d9087b813"}, {file = "cffi-1.14.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2894f2df484ff56d717bead0a5c2abb6b9d2bf26d6960c4604d5c48bbc30ee73"}, {file = "cffi-1.14.5-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:0857f0ae312d855239a55c81ef453ee8fd24136eaba8e87a2eceba644c0d4c06"}, {file = "cffi-1.14.5-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:cd2868886d547469123fadc46eac7ea5253ea7fcb139f12e1dfc2bbd406427d1"}, {file = "cffi-1.14.5-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:35f27e6eb43380fa080dccf676dece30bef72e4a67617ffda586641cd4508d49"}, + {file = "cffi-1.14.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:06d7cd1abac2ffd92e65c0609661866709b4b2d82dd15f611e602b9b188b0b69"}, + {file = "cffi-1.14.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0f861a89e0043afec2a51fd177a567005847973be86f709bbb044d7f42fc4e05"}, + {file = "cffi-1.14.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cc5a8e069b9ebfa22e26d0e6b97d6f9781302fe7f4f2b8776c3e1daea35f1adc"}, {file = "cffi-1.14.5-cp37-cp37m-win32.whl", hash = "sha256:9ff227395193126d82e60319a673a037d5de84633f11279e336f9c0f189ecc62"}, {file = "cffi-1.14.5-cp37-cp37m-win_amd64.whl", hash = "sha256:9cf8022fb8d07a97c178b02327b284521c7708d7c71a9c9c355c178ac4bbd3d4"}, {file = "cffi-1.14.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8b198cec6c72df5289c05b05b8b0969819783f9418e0409865dac47288d2a053"}, {file = "cffi-1.14.5-cp38-cp38-manylinux1_i686.whl", hash = "sha256:ad17025d226ee5beec591b52800c11680fca3df50b8b29fe51d882576e039ee0"}, {file = "cffi-1.14.5-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:6c97d7350133666fbb5cf4abdc1178c812cb205dc6f41d174a7b0f18fb93337e"}, {file = "cffi-1.14.5-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8ae6299f6c68de06f136f1f9e69458eae58f1dacf10af5c17353eae03aa0d827"}, + {file = "cffi-1.14.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:04c468b622ed31d408fea2346bec5bbffba2cc44226302a0de1ade9f5ea3d373"}, + {file = "cffi-1.14.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:06db6321b7a68b2bd6df96d08a5adadc1fa0e8f419226e25b2a5fbf6ccc7350f"}, + {file = "cffi-1.14.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:293e7ea41280cb28c6fcaaa0b1aa1f533b8ce060b9e701d78511e1e6c4a1de76"}, {file = "cffi-1.14.5-cp38-cp38-win32.whl", hash = "sha256:b85eb46a81787c50650f2392b9b4ef23e1f126313b9e0e9013b35c15e4288e2e"}, {file = "cffi-1.14.5-cp38-cp38-win_amd64.whl", hash = "sha256:1f436816fc868b098b0d63b8920de7d208c90a67212546d02f84fe78a9c26396"}, {file = "cffi-1.14.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1071534bbbf8cbb31b498d5d9db0f274f2f7a865adca4ae429e147ba40f73dea"}, {file = "cffi-1.14.5-cp39-cp39-manylinux1_i686.whl", hash = "sha256:9de2e279153a443c656f2defd67769e6d1e4163952b3c622dcea5b08a6405322"}, {file = "cffi-1.14.5-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:6e4714cc64f474e4d6e37cfff31a814b509a35cb17de4fb1999907575684479c"}, {file = "cffi-1.14.5-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:158d0d15119b4b7ff6b926536763dc0714313aa59e320ddf787502c70c4d4bee"}, + {file = "cffi-1.14.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bf1ac1984eaa7675ca8d5745a8cb87ef7abecb5592178406e55858d411eadc0"}, + {file = "cffi-1.14.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:df5052c5d867c1ea0b311fb7c3cd28b19df469c056f7fdcfe88c7473aa63e333"}, + {file = "cffi-1.14.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:24a570cd11895b60829e941f2613a4f79df1a27344cbbb82164ef2e0116f09c7"}, {file = "cffi-1.14.5-cp39-cp39-win32.whl", hash = "sha256:afb29c1ba2e5a3736f1c301d9d0abe3ec8b86957d04ddfa9d7a6a42b9367e396"}, {file = "cffi-1.14.5-cp39-cp39-win_amd64.whl", hash = "sha256:f2d45f97ab6bb54753eab54fffe75aaf3de4ff2341c9daee1987ee1837636f1d"}, {file = "cffi-1.14.5.tar.gz", hash = "sha256:fd78e5fee591709f32ef6edb9a015b4aa1a5022598e36227500c8f4e02328d9c"}, @@ -2287,8 +2299,8 @@ dcicpyvcf = [ {file = "dcicpyvcf-1.0.0.tar.gz", hash = "sha256:c5bf8d585002ab3b95d13a47803376b456b931865e4189c38a18cca47b108449"}, ] dcicsnovault = [ - {file = "dcicsnovault-4.7.6-py3-none-any.whl", hash = "sha256:0ff0c37ba9201dff7f3a778f5feb2acfd8eb22f68ebbbf1e579cf6e6b664dc8e"}, - {file = "dcicsnovault-4.7.6.tar.gz", hash = "sha256:4e2a3f043b4e4632053bc7b553a2ce67c5a501de3f6af096efda3506998642ad"}, + {file = "dcicsnovault-4.8.0.0b2-py3-none-any.whl", hash = "sha256:5bf9d1000a53b250d2218ed31dfb19f0d58cb5d2de81c6baa517f03dc2803805"}, + {file = "dcicsnovault-4.8.0.0b2.tar.gz", hash = "sha256:4dcde76ab8cb82469ca3a2a4eae9476ac04396faf370e124a3114f63378cf48a"}, ] dcicutils = [ {file = "dcicutils-1.15.0-py3-none-any.whl", hash = "sha256:1ad2ce8cdae57e42fee5506669af24dc2c181a3188dd07743aba93a87024d04c"}, diff --git a/pyproject.toml b/pyproject.toml index 07996bb82e..dc2621b12b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ certifi = ">=2020.11.8" chardet = "3.0.4" colorama = "0.3.3" dcicpyvcf = "1.0.0" -dcicsnovault = "^4.7.6" +dcicsnovault = "4.8.0.0b2" dcicutils = "1.15.0" elasticsearch = "6.8.1" execnet = "1.4.1" From 97febc239b233ae46c4ded8bfb75aeac427634a1 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Wed, 2 Jun 2021 14:32:58 -0400 Subject: [PATCH 087/120] small changes, repair higlass worker --- .dockerignore | 4 ++- deploy/docker/production/Makefile | 2 +- .../docker/production/entrypoint_indexer.sh | 1 + src/encoded/visualization.py | 33 +++++++++---------- 4 files changed, 21 insertions(+), 19 deletions(-) diff --git a/.dockerignore b/.dockerignore index d4ba7704a2..209f125cd8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,8 @@ .docker-compose.yml Dockerfile -node_modules +node_modules/ .cache .pytest_cache aws-ip-ranges.json +.git +docs diff --git a/deploy/docker/production/Makefile b/deploy/docker/production/Makefile index 99aa4f6f9e..e807db661a 100644 --- a/deploy/docker/production/Makefile +++ b/deploy/docker/production/Makefile @@ -7,7 +7,7 @@ wsgi: docker push 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest indexer: - docker build -t cgap-mastertest:latest-indexer -f Dockerfile . --build-arg ENTRYPOINT=entrypoint_indexer --build-arg BUILD_PATH=deploy/docker/production + docker build -t cgap-mastertest:latest-indexer -f Dockerfile . --build-arg ENTRYPOINT=entrypoint_indexer.sh --build-arg BUILD_PATH=deploy/docker/production docker tag cgap-mastertest:latest-indexer 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-indexer docker push 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-indexer diff --git a/deploy/docker/production/entrypoint_indexer.sh b/deploy/docker/production/entrypoint_indexer.sh index 08d11d1d0b..317654b896 100644 --- a/deploy/docker/production/entrypoint_indexer.sh +++ b/deploy/docker/production/entrypoint_indexer.sh @@ -9,6 +9,7 @@ poetry run python -m assume_identity # Start indexer, run forever while true do + echo "Starting an indexing run" poetry run es-index-data production.ini --app-name app || echo "Indexing Runtime Error thrown - check previous output" sleep 3 done diff --git a/src/encoded/visualization.py b/src/encoded/visualization.py index 474bb59202..eed3df92e5 100644 --- a/src/encoded/visualization.py +++ b/src/encoded/visualization.py @@ -104,7 +104,7 @@ def get_higlass_viewconf(context, request): request(obj): Http request object. Assumes request's request is JSON and contains these keys: requesting_tab(str) : "annotation" or "bam" variant_pos_abs(int) : Center of the viewconf in abs genome coordinates - + Returns: A dictionary. success(bool) : Boolean indicating success. @@ -112,7 +112,7 @@ def get_higlass_viewconf(context, request): viewconfig(dict) : Dict representing the new viewconfig. """ - requesting_tab = request.json_body.get('requesting_tab', None) + requesting_tab = request.json_body.get('requesting_tab', None) requesting_tab = requesting_tab if requesting_tab else "annotation" viewconf_uuid = "00000000-1111-0000-1111-000000000000" if requesting_tab == "annotation" else "9146eeba-ebb8-41aa-93a8-ada8efaff64b" @@ -126,10 +126,10 @@ def get_higlass_viewconf(context, request): "success" : False, "errors": "No view config found.", "viewconfig": None - } + } # We need absolute URLs for the BAM adn GnomAD Worker - host_url = "http://localhost:6543" + host_url = "http://c4ecstrialalphaecslb-2115269186.us-east-1.elb.amazonaws.com" if request.registry.settings.get('env.name') == CGAP_ENV_WEBPROD: host_url = CGAP_PUBLIC_URL_PRD elif request.registry.settings.get('env.name') == CGAP_ENV_MASTERTEST: @@ -138,18 +138,18 @@ def get_higlass_viewconf(context, request): host_url = f"http://{CGAP_ENV_DEV}.9wzadzju3p.us-east-1.elasticbeanstalk.com" if requesting_tab == "annotation": - variant_pos = request.json_body.get('variant_pos_abs', None) + variant_pos = request.json_body.get('variant_pos_abs', None) variant_pos = variant_pos if variant_pos else 100000 window_size_small = 20 # window size for the interpretation space window_size_large = 5000 # window size for the overview # Overview higlass_viewconfig['views'][0]['initialXDomain'][0] = variant_pos - window_size_large - higlass_viewconfig['views'][0]['initialXDomain'][1] = variant_pos + window_size_large + higlass_viewconfig['views'][0]['initialXDomain'][1] = variant_pos + window_size_large # Details higlass_viewconfig['views'][1]['initialXDomain'][0] = variant_pos - window_size_small - higlass_viewconfig['views'][1]['initialXDomain'][1] = variant_pos + window_size_small + higlass_viewconfig['views'][1]['initialXDomain'][1] = variant_pos + window_size_small # Vertical rules higlass_viewconfig['views'][1]['tracks']['whole'][0]['x'] = variant_pos @@ -159,18 +159,18 @@ def get_higlass_viewconf(context, request): higlass_viewconfig['views'][1]['tracks']['top'][17]['options']['workerScriptLocation'] = host_url + wsl elif requesting_tab == "bam": - variant_pos = request.json_body.get('variant_pos_abs', None) + variant_pos = request.json_body.get('variant_pos_abs', None) variant_pos = variant_pos if variant_pos else 100000 # This is the id of the variant sample that we are currently looking at. # This should be the first file in the Higlass viewconf - bam_sample_id = request.json_body.get('bam_sample_id', None) + bam_sample_id = request.json_body.get('bam_sample_id', None) window_size_small = 20 # window size for the interpretation space window_size_large = 5000 # window size for the overview #s3_bucket = request.registry.settings.get('file_wfout_bucket') s3_bucket = "elasticbeanstalk-fourfront-cgap-wfoutput" - samples_pedigree = request.json_body.get('samples_pedigree', None) + samples_pedigree = request.json_body.get('samples_pedigree', None) samples_pedigree.sort(key=lambda x: x['sample_name'] == bam_sample_id, reverse=True) top_tracks = higlass_viewconfig['views'][1]['tracks']['top'] @@ -181,7 +181,7 @@ def get_higlass_viewconf(context, request): # Delete original tracks from the insert, replace them with adjusted data # from the sample data. If there is no data, we only show the sequence track - del top_tracks[6:10] + del top_tracks[6:10] # print(json.dumps(top_tracks, indent=2)) for sample in samples_pedigree: @@ -209,20 +209,20 @@ def get_higlass_viewconf(context, request): # Show the correct location higlass_viewconfig['views'][0]['initialXDomain'][0] = variant_pos - window_size_large - higlass_viewconfig['views'][0]['initialXDomain'][1] = variant_pos + window_size_large + higlass_viewconfig['views'][0]['initialXDomain'][1] = variant_pos + window_size_large higlass_viewconfig['views'][1]['initialXDomain'][0] = variant_pos - window_size_small - higlass_viewconfig['views'][1]['initialXDomain'][1] = variant_pos + window_size_small + higlass_viewconfig['views'][1]['initialXDomain'][1] = variant_pos + window_size_small # Vertical rules higlass_viewconfig['views'][1]['tracks']['whole'][0]['x'] = variant_pos higlass_viewconfig['views'][1]['tracks']['whole'][1]['x'] = variant_pos + 1 - + return { "success" : True, "errors": "", "viewconfig" : higlass_viewconfig - } + } def create_presigned_url(bucket_name, object_name, expiration=3600): """Generate a presigned URL to share an S3 object @@ -243,6 +243,5 @@ def create_presigned_url(bucket_name, object_name, expiration=3600): return None # The response contains the presigned URL - return response + return response - \ No newline at end of file From afc074f13386a02271122b05971f6d0ad943567d Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Wed, 2 Jun 2021 15:02:45 -0400 Subject: [PATCH 088/120] disable mpindexer --- deploy/docker/production/mastertest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/docker/production/mastertest.ini b/deploy/docker/production/mastertest.ini index 2711e88202..9d9b8832e6 100644 --- a/deploy/docker/production/mastertest.ini +++ b/deploy/docker/production/mastertest.ini @@ -16,7 +16,7 @@ encoded_version = ${PROJECT_VERSION} eb_app_version = ${APP_VERSION} snovault_version = ${SNOVAULT_VERSION} utils_version = ${UTILS_VERSION} -mpindexer = true +mpindexer = false indexer = ${INDEXER} indexer.namespace = cgap-mastertest index_server = ${INDEX_SERVER} From 7f1e62e0ab0211440f1be8db7b11ef7a0b4cbc98 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Wed, 2 Jun 2021 16:28:59 -0400 Subject: [PATCH 089/120] rate limit indexing less --- deploy/docker/production/entrypoint_indexer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/docker/production/entrypoint_indexer.sh b/deploy/docker/production/entrypoint_indexer.sh index 317654b896..e8d662991c 100644 --- a/deploy/docker/production/entrypoint_indexer.sh +++ b/deploy/docker/production/entrypoint_indexer.sh @@ -11,7 +11,7 @@ while true do echo "Starting an indexing run" poetry run es-index-data production.ini --app-name app || echo "Indexing Runtime Error thrown - check previous output" - sleep 3 + sleep 1 done exit 0 From b8054a09d2f8c066a77308cbf88b415a36b50321 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Fri, 4 Jun 2021 10:14:13 -0400 Subject: [PATCH 090/120] one dockerfile to rule them all --- Dockerfile | 40 ++++++++-------- Makefile | 2 +- deploy/docker/local/entrypoint.sh | 2 +- deploy/docker/production/Makefile | 2 +- deploy/docker/production/entrypoint.sh | 46 +++++++++++++++---- deploy/docker/production/entrypoint_portal.sh | 13 ++++++ docker-compose.yml | 9 ++-- src/encoded/__init__.py | 17 +------ src/encoded/authentication.py | 2 +- 9 files changed, 83 insertions(+), 50 deletions(-) create mode 100644 deploy/docker/production/entrypoint_portal.sh diff --git a/Dockerfile b/Dockerfile index f41dafaacd..18b046cf96 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,12 +11,8 @@ MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" # Build Arguments ARG CGAP_ENV_NAME ENV CGAP_ENV_NAME=${CGAP_ENV_NAME:-"cgap-mastertest"} -ARG BUILD_PATH -ENV BUILD_PATH=${BUILD_PATH:-"deploy/docker/production"} ARG INI_BASE ENV INI_BASE=${INI_BASE:-"mastertest.ini"} -ARG ENTRYPOINT -ENV ENTRYPOINT=${ENTRYPOINT:-"entrypoint.sh"} # Configure (global) Env ENV NGINX_USER=nginx @@ -31,7 +27,7 @@ ENV PYTHONFAULTHANDLER=1 \ POETRY_VERSION=1.1.4 # Install nginx, base system -COPY $BUILD_PATH/install_nginx.sh / +COPY deploy/docker/production/install_nginx.sh / RUN bash /install_nginx.sh && \ apt-get update && \ apt-get install -y curl vim emacs postgresql-client net-tools && \ @@ -86,7 +82,7 @@ RUN make aws-ip-ranges && \ # Remove default configuration from Nginx RUN rm /etc/nginx/nginx.conf && \ rm /etc/nginx/conf.d/default.conf -COPY $BUILD_PATH/nginx.conf /etc/nginx/nginx.conf +COPY deploy/docker/production/nginx.conf /etc/nginx/nginx.conf # nginx filesystem setup RUN chown -R nginx:nginx /var/cache/nginx && \ @@ -100,25 +96,31 @@ RUN chown -R nginx:nginx /var/cache/nginx && \ touch /var/log/nginx/error.log && \ chown -R nginx:nginx /var/log/nginx/error.log -# Provide base ini file -# will be picked up by IniFileManager -# *.ini must match the env name in secrets manager! +# Pull all required files +# Note that *.ini must match the env name in secrets manager! +# Note that deploy/docker/production/entrypoint.sh resolves which entrypoint to run +# based on env variable "application_type". # For now, this is mastertest. - Will 04/29/21 -SHELL ["/bin/bash", "-c"] -RUN if [[ $BUILD_PATH =~ .*production.* ]]; then \ - echo "Detected production build" && \ - cp $BUILD_PATH/$INI_BASE deploy/ini_files/. ; \ - else \ - echo "Detected local build" && \ - cp $BUILD_PATH/docker_development.ini development.ini ; \ - fi +COPY deploy/docker/local/docker_development.ini development.ini +COPY deploy/docker/local/entrypoint.sh entrypoint_local.sh +RUN chown nginx:nginx development.ini +RUN chmod +x entrypoint_local.sh +# Production setup RUN touch production.ini RUN chown nginx:nginx production.ini - -COPY $BUILD_PATH/$ENTRYPOINT entrypoint.sh +COPY deploy/docker/production/$INI_BASE deploy/ini_files/. +COPY deploy/docker/production/entrypoint.sh . +COPY deploy/docker/production/entrypoint_portal.sh . +COPY deploy/docker/production/entrypoint_deployment.sh . +COPY deploy/docker/production/entrypoint_indexer.sh . +COPY deploy/docker/production/entrypoint_ingester.sh . COPY deploy/docker/production/assume_identity.py . RUN chmod +x entrypoint.sh +RUN chmod +x entrypoint_deployment.sh +RUN chmod +x entrypoint_deployment.sh +RUN chmod +x entrypoint_indexer.sh +RUN chmod +x entrypoint_ingester.sh RUN chmod +x assume_identity.py EXPOSE 8000 diff --git a/Makefile b/Makefile index acefe7c9fd..01b3bdc937 100644 --- a/Makefile +++ b/Makefile @@ -173,7 +173,7 @@ update: # updates dependencies poetry update build-docker: - docker-compose build --build-arg BUILD_PATH=deploy/docker/local + docker-compose build build-docker-clean: docker-compose build --no-cache BUILD_PATH=deploy/docker/local diff --git a/deploy/docker/local/entrypoint.sh b/deploy/docker/local/entrypoint.sh index fe66a258e9..9977741427 100644 --- a/deploy/docker/local/entrypoint.sh +++ b/deploy/docker/local/entrypoint.sh @@ -5,7 +5,7 @@ if [ -z ${TEST+x} ]; then if [ ! -z ${LOAD+x} ]; then # Clear db/es since this is the local entry point - poetry run clear-db-es-contents development.ini --app-name app --env $CGAP_ENV_NAME + poetry run clear-db-es-contents development.ini --app-name app --env "$CGAP_ENV_NAME" # Create mapping poetry run create-mapping-on-deploy development.ini --app-name app diff --git a/deploy/docker/production/Makefile b/deploy/docker/production/Makefile index e807db661a..6536764ecd 100644 --- a/deploy/docker/production/Makefile +++ b/deploy/docker/production/Makefile @@ -2,7 +2,7 @@ login: aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 645819926742.dkr.ecr.us-east-1.amazonaws.com wsgi: - docker build -t cgap-mastertest:latest . --build-arg BUILD_PATH=deploy/docker/production + docker build -t cgap-mastertest:latest . docker tag cgap-mastertest:latest 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest docker push 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest diff --git a/deploy/docker/production/entrypoint.sh b/deploy/docker/production/entrypoint.sh index a974b8f407..60c583b66b 100644 --- a/deploy/docker/production/entrypoint.sh +++ b/deploy/docker/production/entrypoint.sh @@ -1,13 +1,43 @@ #!/bin/sh -echo "Starting up CGAP-Portal WSGI" -# Run assume_identity.py to access the desired deployment configuration from -# secrets manager - this builds production.ini -poetry run python -m assume_identity +# Global CGAP Application Entrypoint +# This script resolves which application type is desired based on +# the "$application_type" environment variable. Possible options are: +# * "deployment" to run the deployment +# * "ingester" to run the production ingester (forever) +# * "indexer" to run the production indexer (forever) +# * "portal" to run the production portal worker (API back-end) +# * "local" to run a local deployment + +# Note that only "local" can be run from the local machine +# but the same image build is run across the entire local/production stack. + + +deployment="deployment" +ingester="ingester" +indexer="indexer" +portal="portal" +local="local" + +echo "Resolving which entrypoint is desired" + +# shellcheck disable=SC2154 +if [ "$application_type" = deployment ]; then + sh entrypoint_deployment.sh +elif [ "$application_type" = ingester ]; then + sh entrypoint_ingester.sh +elif [ "$application_type" = indexer ]; then + sh entrypoint_indexer.sh +elif [ "$application_type" = portal ]; then + sh entrypoint_portal.sh +elif [ "$application_type" = local ]; then + sh entrypoint_local.sh +else + echo "Could not resolve entrypoint! Check that \$application_type is set." + exit 1 +fi + +exit 0 -# Start nginx proxy -service nginx start -# Start application -pserve production.ini diff --git a/deploy/docker/production/entrypoint_portal.sh b/deploy/docker/production/entrypoint_portal.sh new file mode 100644 index 0000000000..a974b8f407 --- /dev/null +++ b/deploy/docker/production/entrypoint_portal.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +echo "Starting up CGAP-Portal WSGI" + +# Run assume_identity.py to access the desired deployment configuration from +# secrets manager - this builds production.ini +poetry run python -m assume_identity + +# Start nginx proxy +service nginx start + +# Start application +pserve production.ini diff --git a/docker-compose.yml b/docker-compose.yml index f26bf32df7..61609cbc3e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -42,16 +42,19 @@ services: # * LOAD - Trigger a reload of the inserts in the database. # After running once, comment this out so the container will serve without reloading inserts. # Has no effect if TEST is specified. - app: + app-portal: build: . - container_name: cgap + container_name: cgap-portal command: "/home/nginx/cgap-portal/entrypoint.sh" environment: # Default to caller's environment AWS_ACCESS_KEY_ID: AWS_SECRET_ACCESS_KEY: - # TEST: "true" # default: run local deployment + # TEST: "true" # default: commented out to run local deployment - uncomment to run tests in the container LOAD: "true" # default: load the inserts (comment this out after first run) + application_type: "local" # use local entrypoint + ENCODED_VERSION: "local" + CGAP_ENV_NAME: "cgap-docker-will-test" # XXX: update me ports: - "8000:8000" # nginx proxy port (note application traffic is not forwarded) depends_on: diff --git a/src/encoded/__init__.py b/src/encoded/__init__.py index 4d7956c364..6c87443915 100644 --- a/src/encoded/__init__.py +++ b/src/encoded/__init__.py @@ -31,10 +31,6 @@ raise EnvironmentError("The CGAP encoded library no longer supports Python 2.") -# location of environment variables on elasticbeanstalk -BEANSTALK_ENV_PATH = "/opt/python/current/env" - - def static_resources(config): mimetypes.init() mimetypes.init([pkg_resources.resource_filename('encoded', 'static/mime.types')]) @@ -95,18 +91,7 @@ def app_version(config): if not config.registry.settings.get(APP_VERSION_REGISTRY_KEY): # we update version as part of deployment process `deploy_beanstalk.py` # but if we didn't check env then git - version = os.environ.get("ENCODED_VERSION") - if not version: - try: - version = subprocess.check_output( - ['git', '-C', os.path.dirname(__file__), 'describe']).decode('utf-8').strip() - diff = subprocess.check_output( - ['git', '-C', os.path.dirname(__file__), 'diff', '--no-ext-diff']) - if diff: - version += '-patch' + hashlib.sha1(diff).hexdigest()[:7] - except Exception: - version = "test" - + version = os.environ.get("ENCODED_VERSION", "test") config.registry.settings[APP_VERSION_REGISTRY_KEY] = version # Fourfront does GA stuff here that makes no sense in CGAP (yet). diff --git a/src/encoded/authentication.py b/src/encoded/authentication.py index f1c189c224..7a0af8da3b 100644 --- a/src/encoded/authentication.py +++ b/src/encoded/authentication.py @@ -261,7 +261,7 @@ def get_token_info(token, request): except (ValueError, jwt.exceptions.InvalidTokenError, jwt.exceptions.InvalidKeyError) as e: # Catch errors from decoding JWT - print('Invalid JWT assertion : %s (%s)', (e, type(e).__name__)) + print('Invalid JWT assertion : %s (%s)' % (e, type(e).__name__)) request.set_property(lambda r: True, 'auth0_expired') # Allow us to return 403 code &or unset cookie in renderers.py return None From d7a0cde62ece3c442ac0cdbfe98133f11f46b306 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Fri, 4 Jun 2021 13:35:13 -0400 Subject: [PATCH 091/120] integrate production build into dockerfile --- Makefile | 27 +++++++++++++++++++-------- deploy/docker/production/Makefile | 22 ---------------------- 2 files changed, 19 insertions(+), 30 deletions(-) delete mode 100644 deploy/docker/production/Makefile diff --git a/Makefile b/Makefile index 01b3bdc937..864be44d0f 100644 --- a/Makefile +++ b/Makefile @@ -172,18 +172,26 @@ remote-test-unit: # Note this does the 'indexing' tests update: # updates dependencies poetry update -build-docker: +build-local: docker-compose build -build-docker-clean: +build-local-clean: docker-compose build --no-cache BUILD_PATH=deploy/docker/local -deploy-docker: +deploy-local: docker-compose up -V -deploy-docker-daemon: +deploy-local-daemon: docker-compose up -d -V +ecr-login: + aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 645819926742.dkr.ecr.us-east-1.amazonaws.com + +build-production: + docker build -t cgap-mastertest:latest . + docker tag cgap-mastertest:latest 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest + docker push 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest + help: @make info @@ -211,7 +219,10 @@ info: $(info - Use 'make test' to run tests with normal options similar to what we use on GitHub Actions.) $(info - Use 'make test-any' to run tests without marker constraints (i.e., with no '-m' option).) $(info - Use 'make update' to update dependencies (and the lock file).) - $(info - Use 'make build-docker' to build the local Docker image.) - $(info - Use 'make build-docker-clean' to build the local Docker image with no cache.) - $(info - Use 'make deploy-docker' start up the cluster - pserve output will follow if successful.) - $(info - Use 'make deploy-docker-daemon' will start the cluster in daemon mode.) + + $(info - Use 'make build-local' to build the local Docker image.) + $(info - Use 'make build-local-clean' to build the local Docker image with no cache.) + $(info - Use 'make deploy-local' start up the cluster - pserve output will follow if successful.) + $(info - Use 'make deploy-local-daemon' will start the cluster in daemon mode.) + $(info - Use 'make ecr-login' to login to ECR with the currently sourced AWS creds.) + $(info - Use 'make build-production' to build/tag/push a production image.) diff --git a/deploy/docker/production/Makefile b/deploy/docker/production/Makefile deleted file mode 100644 index 6536764ecd..0000000000 --- a/deploy/docker/production/Makefile +++ /dev/null @@ -1,22 +0,0 @@ -login: - aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 645819926742.dkr.ecr.us-east-1.amazonaws.com - -wsgi: - docker build -t cgap-mastertest:latest . - docker tag cgap-mastertest:latest 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest - docker push 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest - -indexer: - docker build -t cgap-mastertest:latest-indexer -f Dockerfile . --build-arg ENTRYPOINT=entrypoint_indexer.sh --build-arg BUILD_PATH=deploy/docker/production - docker tag cgap-mastertest:latest-indexer 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-indexer - docker push 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-indexer - -ingester: - docker build -t cgap-mastertest:latest-ingester -f Dockerfile . --build-arg ENTRYPOINT=entrypoint_ingester.sh --build-arg BUILD_PATH=deploy/docker/production - docker tag cgap-mastertest:latest-ingester 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-ingester - docker push 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-ingester - -deployment: - docker build -t cgap-mastertest:latest-deployment -f Dockerfile . --build-arg ENTRYPOINT=entrypoint_deployment.sh --build-arg BUILD_PATH=deploy/docker/production - docker tag cgap-mastertest:latest-deployment 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-deployment - docker push 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest-deployment From 1746575d48fd92ebfdf9b7853983bf42a600ff6a Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Fri, 4 Jun 2021 13:49:49 -0400 Subject: [PATCH 092/120] fix workflow --- .github/workflows/main.yml | 2 +- Makefile | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 324ec34750..f41ddeb523 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -122,4 +122,4 @@ jobs: - name: Docker Build if: ${{ matrix.test_type == 'Docker' }} - run: make build-docker \ No newline at end of file + run: make build-local \ No newline at end of file diff --git a/Makefile b/Makefile index 864be44d0f..057c5ae16e 100644 --- a/Makefile +++ b/Makefile @@ -219,8 +219,7 @@ info: $(info - Use 'make test' to run tests with normal options similar to what we use on GitHub Actions.) $(info - Use 'make test-any' to run tests without marker constraints (i.e., with no '-m' option).) $(info - Use 'make update' to update dependencies (and the lock file).) - - $(info - Use 'make build-local' to build the local Docker image.) + $(info - Use 'make build-local' to build the local Docker image.) $(info - Use 'make build-local-clean' to build the local Docker image with no cache.) $(info - Use 'make deploy-local' start up the cluster - pserve output will follow if successful.) $(info - Use 'make deploy-local-daemon' will start the cluster in daemon mode.) From 8c5667dcf1d83c76725438469ad799b08bf50499 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Mon, 7 Jun 2021 13:19:28 -0400 Subject: [PATCH 093/120] remove wheel pin, fix main documentation --- .github/workflows/main.yml | 2 +- Dockerfile | 2 +- docs/source/index.rst | 22 +++------------------- 3 files changed, 5 insertions(+), 21 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f41ddeb523..a86cac29f7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -122,4 +122,4 @@ jobs: - name: Docker Build if: ${{ matrix.test_type == 'Docker' }} - run: make build-local \ No newline at end of file + run: make build-local diff --git a/Dockerfile b/Dockerfile index 18b046cf96..ce9d7216a9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,7 +44,7 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH" # Upgrade pip, install in layer RUN pip install --upgrade pip && \ - pip install poetry==1.1.4 wheel==0.29.0 + pip install poetry==1.1.4 # Adjust permissions RUN chown -R nginx:nginx /opt/venv && \ diff --git a/docs/source/index.rst b/docs/source/index.rst index afcac1415e..e3b09d0eba 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,24 +1,8 @@ -.. CGAP-Portal documentation master file, created by - sphinx-quickstart on Tue Oct 8 11:23:43 2019. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - +================== CGAP Documentation -======================================= - -.. image:: https://travis-ci.org/dbmi-bgm/cgap-portal.svg?branch=master - :target: https://travis-ci.org/dbmi-bgm/cgap-portal - -|Coverage|_ - -.. |Coverage| image:: https://coveralls.io/repos/github/4dn-dcic/fourfront/badge.svg?branch=master -.. _Coverage: https://coveralls.io/github/4dn-dcic/fourfront?branch=master - -|Quality|_ - -.. |Quality| image:: https://api.codacy.com/project/badge/Grade/f5fc54006b4740b5800e83eb2aeeeb43 -.. _Quality: https://www.codacy.com/app/4dn/fourfront?utm_source=github.com&utm_medium=referral&utm_content=4dn-dcic/fourfront&utm_campaign=Badge_Grade +================== +.. image:: https://github.com/dbmi-bgm/cgap-portal/actions/workflows/main.yml/badge.svg .. image:: https://readthedocs.org/projects/cgap-portal/badge/?version=latest From 1550a8ca368fec8b9b05d7aaef07efc58ee90070 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Mon, 7 Jun 2021 13:21:12 -0400 Subject: [PATCH 094/120] fix top level readme --- README.rst | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/README.rst b/README.rst index 04157ab0cd..c504dae765 100644 --- a/README.rst +++ b/README.rst @@ -2,19 +2,7 @@ CGAP PORTAL (HMS-BGM) ======================== -.. image:: https://travis-ci.org/dbmi-bgm/cgap-portal.svg?branch=master - :target: https://travis-ci.org/dbmi-bgm/cgap-portal - -|Coverage|_ - -.. |Coverage| image:: https://coveralls.io/repos/github/4dn-dcic/fourfront/badge.svg?branch=master -.. _Coverage: https://coveralls.io/github/4dn-dcic/fourfront?branch=master - -|Quality|_ - -.. |Quality| image:: https://api.codacy.com/project/badge/Grade/f5fc54006b4740b5800e83eb2aeeeb43 -.. _Quality: https://www.codacy.com/app/4dn/fourfront?utm_source=github.com&utm_medium=referral&utm_content=4dn-dcic/fourfront&utm_campaign=Badge_Grade - +.. image:: https://github.com/dbmi-bgm/cgap-portal/actions/workflows/main.yml/badge.svg .. image:: https://readthedocs.org/projects/cgap-portal/badge/?version=latest From 8379ccb684bfdb3de5f25f33fd2650803616f349 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Mon, 7 Jun 2021 13:31:40 -0400 Subject: [PATCH 095/120] little more documentation --- README.rst | 39 +++++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index c504dae765..1bc62f010f 100644 --- a/README.rst +++ b/README.rst @@ -22,9 +22,36 @@ For information on CGAP-Docker in production, see `here. <./docs/source/docker-p Navigating this Repository ^^^^^^^^^^^^^^^^^^^^^^^^^^ - * .github/workflows/ contains Github Action Workflows - * bin/ contains the few remaining executables - * deploy/docker contains Docker setups (see docker-compose.yml) - * docs/ contains documentation - * scripts/ contains misc scripts - * src/ where the code is \ No newline at end of file + +Important directories/files are outlined below. + + * ``.github/workflows/`` contains Github Action Workflows + * ``bin/`` contains the few remaining executables + * ``deploy/docker`` contains containerization related scripts/configuration + * ``docs/ contains`` documentation + * ``scripts/`` contains misc scripts + * ``src/encoded/`` where the code is + * ``.dockerignore`` specifies paths ignored by the Dockerfile + * ``Dockerfile`` contains the Docker build instructions for the cgap-portal + * ``Makefile`` contains macros for common build operations - see ``make info`` + * ``docker-compose.yml`` builds the new local deployment - see `docker-local.rst`` + * ``package.json`` and ``package-lock.json`` specify the front-end dependencies + * ``pyproject.toml`` and ``poetry.lock`` specify the back-end dependencies + * ``setup_eb.py`` performs final installation setup + +Navigating src/encoded/ +^^^^^^^^^^^^^^^^^^^^^^^ + +Top level files are modules that make up the core functionality of the back-end. Some modules differ greatly from or do +not even exist in fourfront. + + * ``annotations/`` contains mapping table and ingestion related metadata + * ``commands/`` contains Python commands that can be run on the system from the command line + * ``docs/`` contains ReadTheDocs documentation + * ``ingestion/`` contains ingestion related code, such as mapping table intake and VCF processing + * ``schemas/`` contains the metadata schemas + * ``search/`` contains the search/filter_set APIs + * ``static/`` contains front-end code + * ``tests/`` contains back-end unit tests and insert data + * ``types/`` contains metadata type definitions + * ``upgrade/`` contains collection schema version upgraders - are not functioning as intended currently From 44640039b0e6a93b8be117974146f4d071e15ac9 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Mon, 7 Jun 2021 13:32:38 -0400 Subject: [PATCH 096/120] small point on directories --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 1bc62f010f..ccc6c60bf3 100644 --- a/README.rst +++ b/README.rst @@ -43,7 +43,7 @@ Navigating src/encoded/ ^^^^^^^^^^^^^^^^^^^^^^^ Top level files are modules that make up the core functionality of the back-end. Some modules differ greatly from or do -not even exist in fourfront. +not even exist in fourfront. Directories are outlined below. * ``annotations/`` contains mapping table and ingestion related metadata * ``commands/`` contains Python commands that can be run on the system from the command line From db77687506036572a08d0ad09243c093f2764b1e Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 8 Jun 2021 14:16:08 -0400 Subject: [PATCH 097/120] update documentation for new tagging strategy --- README.rst | 4 +-- docs/source/docker-production.rst | 52 ++++++++++++++----------------- 2 files changed, 25 insertions(+), 31 deletions(-) diff --git a/README.rst b/README.rst index ccc6c60bf3..7cb1c926a6 100644 --- a/README.rst +++ b/README.rst @@ -32,9 +32,9 @@ Important directories/files are outlined below. * ``scripts/`` contains misc scripts * ``src/encoded/`` where the code is * ``.dockerignore`` specifies paths ignored by the Dockerfile - * ``Dockerfile`` contains the Docker build instructions for the cgap-portal + * ``Dockerfile`` contains the Docker build instructions for the cgap-portal - see ``docker-production.rst`` * ``Makefile`` contains macros for common build operations - see ``make info`` - * ``docker-compose.yml`` builds the new local deployment - see `docker-local.rst`` + * ``docker-compose.yml`` builds the new local deployment - see ``docker-local.rst`` * ``package.json`` and ``package-lock.json`` specify the front-end dependencies * ``pyproject.toml`` and ``poetry.lock`` specify the back-end dependencies * ``setup_eb.py`` performs final installation setup diff --git a/docs/source/docker-production.rst b/docs/source/docker-production.rst index 233ea478d4..a37d4be495 100644 --- a/docs/source/docker-production.rst +++ b/docs/source/docker-production.rst @@ -1,21 +1,21 @@ CGAP-Docker (Production) ======================== -CGAP-Docker runs in production on AWS Elastic Container Service, meant to be orchestrated from the 4dn-cloud-infra repository. End users will modify ``deploy/docker/production/Makefile`` to suite their immediate build needs with respect to target AWS Account/ECR Repository/Tagging strategy. For more information on the specifics of the ECS setup, see 4dn-cloud-infra. +CGAP-Docker runs in production on AWS Elastic Container Service, meant to be orchestrated from the 4dn-cloud-infra repository. End users will modify the ``Makefile`` to suite their immediate build needs with respect to target AWS Account/ECR Repository/Tagging strategy. For more information on the specifics of the ECS setup, see 4dn-cloud-infra. -The CGAP Application has been orchestrated into the ECS Service/Task paradigm. As of writing all core application services have their own image tag varied by passing the ``$ENTRYPOINT`` build argument. As such, they are all separate services described by the following table: +The CGAP Application has been orchestrated into the ECS Service/Task paradigm. As of writing all core application services are built into the same Docker image. Which entrypoint to run is configured by environment variable passed to the ECS Task. As such, we have 4 separate services described by the following table: +------------+--------------------------------+-----+------+------+-----+--------------------------+ | Kind | Use | Num | Spot | vCPU | Mem | Notes | +============+================================+=====+======+======+=====+==========================+ -| WSGI | Services standard API requests | 8 | Yes | .25 | 512 | Needs autoscaling. | +| Portal | Services standard API requests | 4 | Yes | .25 | 512 | Needs autoscaling. | +------------+--------------------------------+-----+------+------+-----+--------------------------+ -| Indexer | Hits /index at 3sec | 4 | Yes | .25 | 512 | Needs autoscaling. | +| Indexer | Hits /index at 1sec | 4 | Yes | .25 | 512 | Needs autoscaling. | | | intervals indefinitely. | | | | | Could use CPU% | +------------+--------------------------------+-----+------+------+-----+--------------------------+ -| Ingester | Polls SQS for ingestion tasks | 1 | No | .25 | 512 | Need API to add tasks. | +| Ingester | Polls SQS for ingestion tasks | 1 | No | .5 | 1024| Need API to add tasks. | +------------+--------------------------------+-----+------+------+-----+--------------------------+ -| Deployment | Triggers the standard | 0 | Yes | .5 | 1024| Run explicitly or (TODO) | +| Deployment | Triggers the standard | 0 | Yes | .24 | 512 | Run explicitly or (TODO) | | | deployment actions. | | | | | by API. | +------------+--------------------------------+-----+------+------+-----+--------------------------+ @@ -27,12 +27,12 @@ The production application configuration is in ``deploy/docker/production``. A d * Dockerfile - at repo top level - configurable file that builds both all local and production images. * docker-compose.yml - at repo top level - configures the local deployment - unused in production. * assume_identity.py - script for pulling application configuration from Secrets Manager. Note that this secret is meant to be generated by the Datastore stack in 4dn-cloud-infra and manually filled out. - * entrypoint.sh - WSGI entrypoint + * entrypoint.sh - resolves which entrypoint is used based on ``$application_type`` + * entrypoint_portal.sh - serves portal API requests * entrypoint_deployment.sh - deployment entrypoint * entrypoint_indexer.sh - indexer entrypoint * entrypoint_ingester.sh - ingester entrypoint * install_nginx.sh - script for pulling in nginx - * Makefile - configures builds/pushes for relevant images/tags * mastertest.ini - base ini file used to build production.ini on the server * nginx.conf - nginx configuration @@ -40,37 +40,31 @@ The production application configuration is in ``deploy/docker/production``. A d The following instructions describe how to build and push images. Note though that we assume an existing ECS setup. For instructions on how to orchestrate ECS, see 4dn-cloud-infra, but that is not the focus of this documentation. 1. Ensure the orchestrator credentials are sourced, or that your IAM user has been granted sufficient perms to push to ECR. - 2. In the Makefile, replace "cgap-mastertest" with the env.name configured for the environment. This name should match the ECR repo name if you navigate to the ECR Console. - 3. Again in the Makefile, replace the ECR Repo URL (NOT the tags) with the one from the output of the ECR stack in the account. - 4. Run ``make login``, which should pull ECR credentials using the currently active AWS credentials. - 5. Run ``make info`` for information on tags. - 6. Run the appropriate make target to build/deploy the desired version by pushing a new image to ECR. Note that the upload process may take a long time if you made application code (non-configuration) changes. + 2. In the Makefile under ``build-production``, replace "cgap-mastertest" with the env.name configured for the environment. This name should match the ECR repo name if you navigate to the ECR Console. + 3. Again in the Makefile, replace the ECR Repo URL (NOT the tag) with the one from the output of the ECR stack in the account. + 4. Run ``make ecr-login``, which should pull ECR credentials using the currently active AWS credentials. + 5. Run ``make build-production``. -Note that steps 1, 4 and 6 are all that are needed to be repeated once initial setup is done, assuming you are continuously pushing to the same location. To change which ECS orchestration you are effectively deploying to, all steps must be repeated in the relevant account. +Note that steps 4 and 6 are all that are needed to be repeated once initial setup is done, assuming you are continuously pushing to the same location. To change which ECS orchestration you are effectively deploying to, all steps must be repeated in the relevant account. Tagging Strategy ^^^^^^^^^^^^^^^^ -As stated previously, the cgap-portal consists of 4 core components that have been translated into ECS Services/Tasks. The task definitions specify image tags that differentiate each of the 4 components. Those tags are: +As stated previously, there is a single image tag, typically ``latest``, that determines the image tag that ECS will use. This tag is configurable from the 4dn-cloud-infra repository. - * ``latest`` - push to this tag to modify the WSGI image - * ``latest-indexer`` - push to this tag to modify the indexer image - * ``latest-ingester`` - push to this tag to modify the ingester image - * ``latest-deployment`` - push to this tag to modify the deployment image +After a new image version has been pushed, issue a forced deployment update to the ECS cluster through Foursight. This action will spawn a new set of tasks for all services using the newer image tags. For the portal, once the new tasks are deemed healthy by ECS and the Application Load Balancer, they will be added to the Portal Target Group and immediately begin serving requests. At that time the old tasks will begin the de-registration process from the target group, after which they will be spun down. The remaining new tasks will come online more quickly since they do not need to pass load balancer health checks. Once the old tasks have been cleaned up, it is safe to trigger a deployment task through the Deployment Service. -After all new image versions have been pushed, issue a forced deployment update to the ECS cluster. This action will spawn a new set of tasks for all services using the newer image tags. For WSGI, once the new tasks are deemed healthy by ECS and the Application Load Balancer, they will be added to the WSGI Target Group and immediately begin serving requests. At that time the old tasks will begin the de-registration process from the target group, after which they will be spun down. The remaining new tasks will come online more quickly since they do not need to pass load balancer health checks. Once the old tasks have been cleaned up, it is safe to trigger a deployment task through the Deployment Service. +Common Issues +^^^^^^^^^^^^^ -Deciding Which Image(s) to Update -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +In this section we detail some common errors and what to do about them. This section should be updated as more development in this setup occurs. -Generally speaking, all 4 tags must be pushed in order for the deployment to be considered "valid". With that said, there are a few scenarios where it may make sense to do a partial image bundle upload. Note that when uploading to production, none of the following apply. +1. ``Error: denied: User: is not authorized to perform: ecr:InitiateLayerUpload on resource: `` - * If you are only making front-end changes, feel free to push only the ``latest`` tag. - * If you are not ingesting VCF files or modifying the ingestion pipeline, you do not need to update the ``latest-ingester`` image. - * If you want to phase back-end changes in prior to releasing them on the front-end, push ``latest-indexer`` and ``latest-deployment`` images. - * If you are only modifying the deployment commands ie: ``entrypoint_deployment.sh``, then you only need to update the ``latest-deployment`` tag. +This error can happen for several reasons: + * Invalid/incorrect IAM credentials + * IAM user has insufficient permissions + * IAM credentials are valid but from a different AWS account - -Note that these images have implied dependencies on each other that are not obvious. We consider the deployment and indexer images to be bundled together in the sense that updating one without the other will trigger undefined behavior if the deployment is triggered. It is thus imperative that, at a bare minimum before triggering an ECS Cluster update, that both the ``latest-indexer`` and ``latest-deployment`` tags have been updated in the case that any back-end changes have been made, whether it be to application, it's dependencies or the Dockerfile itself. From 21c3a2bc80fa8f38e6ddcdf62c21c6ba9222f892 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 8 Jun 2021 14:30:41 -0400 Subject: [PATCH 098/120] drop in utils methods --- deploy/docker/production/assume_identity.py | 59 +++--------------- src/encoded/commands/load_access_keys.py | 67 +++------------------ 2 files changed, 16 insertions(+), 110 deletions(-) diff --git a/deploy/docker/production/assume_identity.py b/deploy/docker/production/assume_identity.py index 989c1c1a8c..831603299c 100644 --- a/deploy/docker/production/assume_identity.py +++ b/deploy/docker/production/assume_identity.py @@ -5,14 +5,10 @@ # specifically to the secret import os -import json -import boto3 import logging -import watchtower -from botocore.exceptions import ClientError -from dcicutils.beanstalk_utils import REGION from dcicutils.qa_utils import override_environ from dcicutils.deployment_utils import IniFileManager +from dcicutils.secrets_utils import assume_identity logging.basicConfig(level=logging.INFO) @@ -31,56 +27,17 @@ class CGAPDockerIniFileManager(IniFileManager): PYPROJECT_FILE_NAME = '/home/nginx/cgap-portal/pyproject.toml' -def assume_identity(): +def build_production_ini_from_global_application_configuration(): """ This function makes a request to secrets manager for the identity passed to the container. - See documentation above. + See documentation on API in dcicutils. """ - secret_name = os.environ.get('IDENTITY', 'dev/beanstalk/cgap-dev') - region_name = REGION # us-east-1 + identity = assume_identity() - # XXX: We should refactor a SecretsManager wrapper into dcicutils - session = boto3.session.Session(region_name=region_name) - # configure watchtower handler from session - logger.addHandler(watchtower.CloudWatchLogHandler(boto3_session=session)) - client = session.client( - service_name='secretsmanager', - region_name=region_name - ) + # build production.ini + with override_environ(**identity): - try: - get_secret_value_response = client.get_secret_value( - SecretId=secret_name - ) - except ClientError as e: # leaving some useful debug info to help narrow issues - if e.response['Error']['Code'] == 'DecryptionFailureException': - # Secrets Manager can't decrypt the protected secret text using the provided KMS key. - raise e - elif e.response['Error']['Code'] == 'InternalServiceErrorException': - # An error occurred on the server side. - raise e - elif e.response['Error']['Code'] == 'InvalidParameterException': - # You provided an invalid value for a parameter. - raise e - elif e.response['Error']['Code'] == 'InvalidRequestException': - # You provided a parameter value that is not valid for the current state of the resource. - raise e - elif e.response['Error']['Code'] == 'ResourceNotFoundException': - raise e - else: - raise e - else: - # Decrypts secret using the associated KMS CMK. - # Depending on whether the secret is a string or binary, one of these fields will be populated. - if 'SecretString' in get_secret_value_response: - identity = json.loads(get_secret_value_response['SecretString']) - else: - raise Exception('Got unexpected response structure from boto3') - - # build production.ini - with override_environ(**identity): - - CGAPDockerIniFileManager.main() + CGAPDockerIniFileManager.main() if __name__ == '__main__': - assume_identity() + build_production_ini_from_global_application_configuration() diff --git a/src/encoded/commands/load_access_keys.py b/src/encoded/commands/load_access_keys.py index f7ca5cc2fd..656960c0e4 100644 --- a/src/encoded/commands/load_access_keys.py +++ b/src/encoded/commands/load_access_keys.py @@ -6,9 +6,9 @@ import boto3 from pyramid.paster import get_app from webtest import AppError -from botocore.exceptions import ClientError from dcicutils.misc_utils import TestApp -from dcicutils.beanstalk_utils import get_beanstalk_real_url, REGION +from dcicutils.cloudformation_utils import get_ecs_real_url +from dcicutils.secrets_utils import assume_identity log = structlog.getLogger(__name__) EPILOG = __doc__ @@ -43,20 +43,6 @@ def get_existing_key_ids(testapp, user_uuid, key_desc): return [res['@id'] for res in search_res['@graph']] -def get_ecs_real_url(env): - """ Inspects Cloudformation stacks, looking for LB URL - TODO: pull into dcicutils - """ - cfn_client = boto3.client('cloudformation') - stacks = cfn_client.describe_stacks().get('Stacks', []) - for stack in stacks: - for output in stack['Outputs']: - if output.get('OutputKey', '') == ('ECSApplicationURL%s' % env.replace('-', '')): - return output.get('OutputValue') - log.error('Did not locate the server from Cloudformation! Check ECS Stack metadata.') - return '' - - def generate_access_key(testapp, env, user_uuid, description): """ Generate an access for given user on given environment. @@ -115,52 +101,15 @@ def main(): raise RuntimeError('load_access_keys: cannot find env.name in settings') # XXX: refactor into helper method in dcicutils, see assume_identity.py + encrypt_key = None if args.secret_name is not None: - secret_name = args.secret_name - region_name = REGION # us-east-1 - - # XXX: We should refactor a SecretsManager wrapper into dcicutils - session = boto3.session.Session(region_name=region_name) - # configure watchtower handler from session - client = session.client( - service_name='secretsmanager', - region_name=region_name - ) - - try: - get_secret_value_response = client.get_secret_value( - SecretId=secret_name - ) - except ClientError as e: # leaving some useful debug info to help narrow issues - if e.response['Error']['Code'] == 'DecryptionFailureException': - # Secrets Manager can't decrypt the protected secret text using the provided KMS key. - raise e - elif e.response['Error']['Code'] == 'InternalServiceErrorException': - # An error occurred on the server side. - raise e - elif e.response['Error']['Code'] == 'InvalidParameterException': - # You provided an invalid value for a parameter. - raise e - elif e.response['Error']['Code'] == 'InvalidRequestException': - # You provided a parameter value that is not valid for the current state of the resource. - raise e - elif e.response['Error']['Code'] == 'ResourceNotFoundException': - raise e - else: - raise e - else: - # Decrypts secret using the associated KMS CMK. - # Depending on whether the secret is a string or binary, one of these fields will be populated. - if 'SecretString' in get_secret_value_response: - identity = json.loads(get_secret_value_response['SecretString']) - encrypt_key = identity.get('S3_ENCRYPT_KEY', None) - else: - raise Exception('Got unexpected response structure from boto3') - else: - encrypt_key = os.environ.get('S3_ENCRYPT_KEY') + identity = assume_identity() # automatically detects GLOBAL_APPLICATION_CONFIGURATION + encrypt_key = identity.get('S3_ENCRYPT_KEY', None) + if not encrypt_key: + encrypt_key = os.environ.get('S3_ENCRYPT_KEY') if not encrypt_key: - raise RuntimeError('load_access_keys: must define S3_ENCRYPT_KEY in env or in passed secret') + raise RuntimeError('load_access_keys: must define S3_ENCRYPT_KEY in env or in GAC') # will need to use a dynamic region at some point (not just here) s3 = boto3.client('s3', region_name='us-east-1') From f7bddda9b66e9eef922469c2477115d0a8c0be7c Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 8 Jun 2021 15:15:09 -0400 Subject: [PATCH 099/120] remove extra indexer logging, fix higlass host url --- deploy/docker/production/entrypoint_indexer.sh | 1 - src/encoded/visualization.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/docker/production/entrypoint_indexer.sh b/deploy/docker/production/entrypoint_indexer.sh index e8d662991c..f0eacfc22d 100644 --- a/deploy/docker/production/entrypoint_indexer.sh +++ b/deploy/docker/production/entrypoint_indexer.sh @@ -9,7 +9,6 @@ poetry run python -m assume_identity # Start indexer, run forever while true do - echo "Starting an indexing run" poetry run es-index-data production.ini --app-name app || echo "Indexing Runtime Error thrown - check previous output" sleep 1 done diff --git a/src/encoded/visualization.py b/src/encoded/visualization.py index eed3df92e5..4823304432 100644 --- a/src/encoded/visualization.py +++ b/src/encoded/visualization.py @@ -129,7 +129,8 @@ def get_higlass_viewconf(context, request): } # We need absolute URLs for the BAM adn GnomAD Worker - host_url = "http://c4ecstrialalphaecslb-2115269186.us-east-1.elb.amazonaws.com" + # XXX: this needs a workaround - Will 6/8/21 + host_url = "http://c4ecstrialalphacgapmastertest-273357903.us-east-1.elb.amazonaws.com" if request.registry.settings.get('env.name') == CGAP_ENV_WEBPROD: host_url = CGAP_PUBLIC_URL_PRD elif request.registry.settings.get('env.name') == CGAP_ENV_MASTERTEST: From 2736c11ccfef56cbdd4d4024712d835c4b9589b1 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 10 Jun 2021 12:16:15 -0400 Subject: [PATCH 100/120] implement access key rotation --- src/encoded/authentication.py | 28 +++++++++++++++++----------- src/encoded/schemas/access_key.json | 6 ++++++ src/encoded/types/access_key.py | 13 ++++++++++++- 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/src/encoded/authentication.py b/src/encoded/authentication.py index 7a0af8da3b..cd2767ee82 100644 --- a/src/encoded/authentication.py +++ b/src/encoded/authentication.py @@ -2,6 +2,7 @@ import os from operator import itemgetter import jwt +import datetime from base64 import b64decode from passlib.context import CryptContext @@ -251,9 +252,9 @@ def get_token_info(token, request): request.set_property(lambda r: False, 'auth0_expired') return payload - else: # we don't have the key, let auth0 do the work for us + else: # we don't have the key, let auth0 do the work for us user_url = "https://{domain}/tokeninfo".format(domain='hms-dbmi.auth0.com') - resp = requests.post(user_url, {'id_token':token}) + resp = requests.post(user_url, {'id_token':token}) payload = resp.json() if 'email' in payload and Auth0AuthenticationPolicy.email_is_partners_or_hms(payload): request.set_property(lambda r: False, 'auth0_expired') @@ -262,7 +263,7 @@ def get_token_info(token, request): except (ValueError, jwt.exceptions.InvalidTokenError, jwt.exceptions.InvalidKeyError) as e: # Catch errors from decoding JWT print('Invalid JWT assertion : %s (%s)' % (e, type(e).__name__)) - request.set_property(lambda r: True, 'auth0_expired') # Allow us to return 403 code &or unset cookie in renderers.py + request.set_property(lambda r: True, 'auth0_expired') # Allow us to return 403 code &or unset cookie in renderers.py return None print("didn't get email or email is not verified") @@ -313,8 +314,7 @@ def login(context, request): if request_token is None: request_token = request.json_body.get("id_token", None) - - is_https = request.scheme == "https" + is_https = (request.scheme == "https") request.response.set_cookie( "jwtToken", @@ -438,6 +438,10 @@ def session_properties(context, request): def basic_auth_check(username, password, request): + """ This function implements the functionality that does the actual checking of the + access key against what is in the database. It is thus very important. Access + key expiration is implemented here - auth will fail if it has expired + """ # We may get called before the context is found and the root set root = request.registry[ROOT] collection = root['access-keys'] @@ -446,19 +450,21 @@ def basic_auth_check(username, password, request): except KeyError: return None + # Check expiration first properties = access_key.properties - hash = properties['secret_access_key_hash'] + expiration_date = datetime.datetime.fromisoformat(properties['expiration_date']) + now = datetime.datetime.utcnow() + if now > expiration_date: + return None + # If expiration valid, check hash + hash = properties['secret_access_key_hash'] crypt_context = request.registry[CRYPT_CONTEXT] valid = crypt_context.verify(password, hash) if not valid: return None - #valid, new_hash = crypt_context.verify_and_update(password, hash) - #if new_hash: - # replace_user_hash(user, new_hash) - - return [] + return [] # success @view_config(route_name='impersonate-user', request_method='POST', diff --git a/src/encoded/schemas/access_key.json b/src/encoded/schemas/access_key.json index ee00229f35..123ddd74c7 100644 --- a/src/encoded/schemas/access_key.json +++ b/src/encoded/schemas/access_key.json @@ -53,6 +53,12 @@ "title": "Secret access key Hash", "comment": "Only admins are allowed to set this value.", "type": "string" + }, + "expiration_date": { + "Title": "Expiration Date", + "comment": "Only admins are allowed to set this value.", + "type": "string", + "permission": "restricted_fields" } }, "facets": { diff --git a/src/encoded/types/access_key.py b/src/encoded/types/access_key.py index db55a1abfa..5fe8e01ee1 100644 --- a/src/encoded/types/access_key.py +++ b/src/encoded/types/access_key.py @@ -8,6 +8,7 @@ Everyone, ) from pyramid.settings import asbool +import datetime from .base import ( Item, DELETED_ACL, @@ -48,7 +49,7 @@ ]) class AccessKey(Item): """AccessKey class.""" - + ACCESS_KEY_EXPIRATION_TIME = 90 # days item_type = 'access_key' schema = load_schema('encoded:schemas/access_key.json') name_key = 'access_key_id' @@ -59,6 +60,13 @@ class AccessKey(Item): 'deleted': DELETED_ACL, } + @classmethod + def create(cls, registry, uuid, properties, sheets=None): + """ Sets the access key timeout 90 days from creation. """ + properties['expiration_date'] = (datetime.datetime.utcnow() + datetime.timedelta( + days=cls.ACCESS_KEY_EXPIRATION_TIME)).isoformat() + return super().create(registry, uuid, properties, sheets) + def __ac_local_roles__(self): """grab and return user as owner.""" owner = 'userid.%s' % self.properties['user'] @@ -77,6 +85,9 @@ def update(self, properties, sheets=None): new_properties = self.properties.copy() new_properties.update(properties) properties = new_properties + # set new expiration + properties['expiration_date'] = (datetime.datetime.utcnow() + datetime.timedelta( + days=self.ACCESS_KEY_EXPIRATION_TIME)).isoformat() self._update(properties, sheets) class Collection(Item.Collection): From 7d531b6bb28b3b73654e416afeff3ce4937d35b6 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 10 Jun 2021 13:24:57 -0400 Subject: [PATCH 101/120] address a bunch of comments --- Makefile | 20 ++++++++++---------- deploy/docker/production/entrypoint.sh | 11 +++++------ deploy/docker/production/mastertest.ini | 2 +- docs/source/docker-local.rst | 18 +++++++++--------- docs/source/docker-production.rst | 4 ++-- src/encoded/commands/load_access_keys.py | 4 ++-- src/encoded/visualization.py | 2 +- 7 files changed, 30 insertions(+), 31 deletions(-) diff --git a/Makefile b/Makefile index 057c5ae16e..b709e688fa 100644 --- a/Makefile +++ b/Makefile @@ -172,22 +172,22 @@ remote-test-unit: # Note this does the 'indexing' tests update: # updates dependencies poetry update -build-local: +build-docker-local: docker-compose build -build-local-clean: +build-docker-local-clean: docker-compose build --no-cache BUILD_PATH=deploy/docker/local -deploy-local: +deploy-docker-local: docker-compose up -V -deploy-local-daemon: +deploy-docker-local-daemon: docker-compose up -d -V ecr-login: aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 645819926742.dkr.ecr.us-east-1.amazonaws.com -build-production: +build-docker-production: docker build -t cgap-mastertest:latest . docker tag cgap-mastertest:latest 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest docker push 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest @@ -219,9 +219,9 @@ info: $(info - Use 'make test' to run tests with normal options similar to what we use on GitHub Actions.) $(info - Use 'make test-any' to run tests without marker constraints (i.e., with no '-m' option).) $(info - Use 'make update' to update dependencies (and the lock file).) - $(info - Use 'make build-local' to build the local Docker image.) - $(info - Use 'make build-local-clean' to build the local Docker image with no cache.) - $(info - Use 'make deploy-local' start up the cluster - pserve output will follow if successful.) - $(info - Use 'make deploy-local-daemon' will start the cluster in daemon mode.) + $(info - Use 'make build-docker-local' to build the local Docker image.) + $(info - Use 'make build-docker-local-clean' to build the local Docker image with no cache.) + $(info - Use 'make deploy-docker-local' start up the cluster - pserve output will follow if successful.) + $(info - Use 'make deploy-docker-local-daemon' will start the cluster in daemon mode.) $(info - Use 'make ecr-login' to login to ECR with the currently sourced AWS creds.) - $(info - Use 'make build-production' to build/tag/push a production image.) + $(info - Use 'make build-docker-production' to build/tag/push a production image.) diff --git a/deploy/docker/production/entrypoint.sh b/deploy/docker/production/entrypoint.sh index 60c583b66b..4089d44318 100644 --- a/deploy/docker/production/entrypoint.sh +++ b/deploy/docker/production/entrypoint.sh @@ -13,7 +13,6 @@ # Note that only "local" can be run from the local machine # but the same image build is run across the entire local/production stack. - deployment="deployment" ingester="ingester" indexer="indexer" @@ -23,15 +22,15 @@ local="local" echo "Resolving which entrypoint is desired" # shellcheck disable=SC2154 -if [ "$application_type" = deployment ]; then +if [ "$application_type" = $deployment ]; then sh entrypoint_deployment.sh -elif [ "$application_type" = ingester ]; then +elif [ "$application_type" = $ingester ]; then sh entrypoint_ingester.sh -elif [ "$application_type" = indexer ]; then +elif [ "$application_type" = $indexer ]; then sh entrypoint_indexer.sh -elif [ "$application_type" = portal ]; then +elif [ "$application_type" = $portal ]; then sh entrypoint_portal.sh -elif [ "$application_type" = local ]; then +elif [ "$application_type" = $local ]; then sh entrypoint_local.sh else echo "Could not resolve entrypoint! Check that \$application_type is set." diff --git a/deploy/docker/production/mastertest.ini b/deploy/docker/production/mastertest.ini index 9d9b8832e6..57a0d31ce5 100644 --- a/deploy/docker/production/mastertest.ini +++ b/deploy/docker/production/mastertest.ini @@ -82,4 +82,4 @@ level = NOTSET formatter = generic [formatter_generic] -format = %(message)s \ No newline at end of file +format = %(message)s diff --git a/docs/source/docker-local.rst b/docs/source/docker-local.rst index 89a3a60f87..84aeabefb2 100644 --- a/docs/source/docker-local.rst +++ b/docs/source/docker-local.rst @@ -33,10 +33,10 @@ Building CGAP Docker There are two new Make targets that should be sufficient for normal use. To build the image locally, ensure your AWS keys are sourced and run:: - $ make build-docker # runs docker-compose build - $ make build-docker-clean # runs a no-cache build, regenerating all layers - $ make deploy-docker # runs docker-compose up - $ make deploy-docker-daemon # runs services in background + $ make build-docker-local # runs docker-compose build + $ make build-docker-local-clean # runs a no-cache build, regenerating all layers + $ make deploy-docker-local # runs docker-compose up + $ make deploy-docker-local-daemon # runs services in background The build will take around 10 minutes the first time but will speed up dramatically after due to layer caching. In general, the rate limiting step for rebuilding is the front-end build (unless you are also updating dependencies, which will slow down the build further). Although this may seem like a drawback, the key benefit is that what you are running in Docker is essentially identical to that which is orchestrated on ECS in production. This should reduce our reliance/need for test environments. @@ -70,11 +70,11 @@ Docker Command Cheatsheet Below is a small list of useful Docker commands for advanced users:: - $ docker-compose build # will trigger a build of the local cluster - $ docker-compose build --no-cache # will trigger a fresh build of the entire cluster - $ docker-compose down # will stop cluster + $ docker-compose build # will trigger a build of the local cluster (see make build-docker-local) + $ docker-compose build --no-cache # will trigger a fresh build of the entire cluster (see make build-docker-local-clean) + $ docker-compose down # will stop cluster (can also ctrl-c) $ docker-compose down --volumes # will remove cluster volumes as well - $ docker-compose up # will start cluster and log all output to console - $ docker-compose up -d # will start cluster in background using existing containers + $ docker-compose up # will start cluster and log all output to console (see make deploy-docker-local) + $ docker-compose up -d # will start cluster in background using existing containers (see make deploy-docker-local-daemon) $ docker-compose up -d -V --build # trigger a rebuild/recreation of cluster containers $ docker system prune # will cleanup ALL unused Docker components - BE CAREFUL WITH THIS diff --git a/docs/source/docker-production.rst b/docs/source/docker-production.rst index a37d4be495..0d5645802d 100644 --- a/docs/source/docker-production.rst +++ b/docs/source/docker-production.rst @@ -40,10 +40,10 @@ The production application configuration is in ``deploy/docker/production``. A d The following instructions describe how to build and push images. Note though that we assume an existing ECS setup. For instructions on how to orchestrate ECS, see 4dn-cloud-infra, but that is not the focus of this documentation. 1. Ensure the orchestrator credentials are sourced, or that your IAM user has been granted sufficient perms to push to ECR. - 2. In the Makefile under ``build-production``, replace "cgap-mastertest" with the env.name configured for the environment. This name should match the ECR repo name if you navigate to the ECR Console. + 2. In the Makefile under ``build-docker-production``, replace "cgap-mastertest" with the env.name configured for the environment. This name should match the ECR repo name if you navigate to the ECR Console. 3. Again in the Makefile, replace the ECR Repo URL (NOT the tag) with the one from the output of the ECR stack in the account. 4. Run ``make ecr-login``, which should pull ECR credentials using the currently active AWS credentials. - 5. Run ``make build-production``. + 5. Run ``make build-docker-production``. Note that steps 4 and 6 are all that are needed to be repeated once initial setup is done, assuming you are continuously pushing to the same location. To change which ECS orchestration you are effectively deploying to, all steps must be repeated in the relevant account. diff --git a/src/encoded/commands/load_access_keys.py b/src/encoded/commands/load_access_keys.py index 656960c0e4..295ef8653c 100644 --- a/src/encoded/commands/load_access_keys.py +++ b/src/encoded/commands/load_access_keys.py @@ -100,11 +100,11 @@ def main(): if not env: raise RuntimeError('load_access_keys: cannot find env.name in settings') - # XXX: refactor into helper method in dcicutils, see assume_identity.py + # Resolve secret from environment if one is not specified encrypt_key = None if args.secret_name is not None: identity = assume_identity() # automatically detects GLOBAL_APPLICATION_CONFIGURATION - encrypt_key = identity.get('S3_ENCRYPT_KEY', None) + encrypt_key = identity.get('S3_ENCRYPT_KEY', None) # one of the secrets if not encrypt_key: encrypt_key = os.environ.get('S3_ENCRYPT_KEY') diff --git a/src/encoded/visualization.py b/src/encoded/visualization.py index 4823304432..293fd7d8b7 100644 --- a/src/encoded/visualization.py +++ b/src/encoded/visualization.py @@ -128,7 +128,7 @@ def get_higlass_viewconf(context, request): "viewconfig": None } - # We need absolute URLs for the BAM adn GnomAD Worker + # We need absolute URLs for the BAM and GnomAD Worker # XXX: this needs a workaround - Will 6/8/21 host_url = "http://c4ecstrialalphacgapmastertest-273357903.us-east-1.elb.amazonaws.com" if request.registry.settings.get('env.name') == CGAP_ENV_WEBPROD: From 8def838afacc8bebf04d1e41429ddc0a8334bfbe Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 10 Jun 2021 13:26:45 -0400 Subject: [PATCH 102/120] repair workflow --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a86cac29f7..7589c19278 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -122,4 +122,4 @@ jobs: - name: Docker Build if: ${{ matrix.test_type == 'Docker' }} - run: make build-local + run: make build-docker-local From f673d8662300b9f39aa48176cecd98fa4ec2b3b4 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Mon, 14 Jun 2021 17:10:24 -0400 Subject: [PATCH 103/120] build w node12, repair datetime constructor --- Dockerfile | 2 +- src/encoded/authentication.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index ce9d7216a9..8844f41008 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,7 @@ COPY deploy/docker/production/install_nginx.sh / RUN bash /install_nginx.sh && \ apt-get update && \ apt-get install -y curl vim emacs postgresql-client net-tools && \ - curl -sL https://deb.nodesource.com/setup_10.x | bash - && \ + curl -sL https://deb.nodesource.com/setup_12.x | bash - && \ apt-get install -y ca-certificates nodejs npm # Configure CGAP User (nginx) diff --git a/src/encoded/authentication.py b/src/encoded/authentication.py index cd2767ee82..29d97153ba 100644 --- a/src/encoded/authentication.py +++ b/src/encoded/authentication.py @@ -452,7 +452,7 @@ def basic_auth_check(username, password, request): # Check expiration first properties = access_key.properties - expiration_date = datetime.datetime.fromisoformat(properties['expiration_date']) + expiration_date = datetime.date.fromisoformat(properties['expiration_date']) now = datetime.datetime.utcnow() if now > expiration_date: return None From db0a2e3508557aae03fb7dfc928e8b6678184e01 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Mon, 14 Jun 2021 17:14:14 -0400 Subject: [PATCH 104/120] make backwards compatible (for now) --- src/encoded/authentication.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/encoded/authentication.py b/src/encoded/authentication.py index 29d97153ba..156266b0f9 100644 --- a/src/encoded/authentication.py +++ b/src/encoded/authentication.py @@ -451,11 +451,14 @@ def basic_auth_check(username, password, request): return None # Check expiration first + # Note that access keys generated awhile ago will remain valid (for now) - will 6/14/21 properties = access_key.properties - expiration_date = datetime.date.fromisoformat(properties['expiration_date']) - now = datetime.datetime.utcnow() - if now > expiration_date: - return None + expiration_date = properties.get('expiration_date') + if expiration_date: + dt = datetime.date.fromisoformat(expiration_date) + now = datetime.datetime.utcnow() + if now > dt: + return None # If expiration valid, check hash hash = properties['secret_access_key_hash'] From 740a9bc6d25016e2d75e3c1c37609e63b4b9bfa4 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 15 Jun 2021 07:49:17 -0400 Subject: [PATCH 105/120] use isoparse instead --- src/encoded/authentication.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/encoded/authentication.py b/src/encoded/authentication.py index 156266b0f9..753bb957de 100644 --- a/src/encoded/authentication.py +++ b/src/encoded/authentication.py @@ -35,6 +35,7 @@ CONNECTION, COLLECTIONS ) +from dateutil.parser import isoparse from dcicutils.misc_utils import remove_element from snovault.validation import ValidationFailure from snovault.calculated import calculate_properties @@ -455,7 +456,7 @@ def basic_auth_check(username, password, request): properties = access_key.properties expiration_date = properties.get('expiration_date') if expiration_date: - dt = datetime.date.fromisoformat(expiration_date) + dt = isoparse(expiration_date) # datetime.date.fromisoformat in Python3.7 now = datetime.datetime.utcnow() if now > dt: return None From 27f6b3f314afd8926638931ce25a1d18c8fcb03e Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 17 Jun 2021 15:25:04 -0400 Subject: [PATCH 106/120] properly install node12 with nvm --- Dockerfile | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8844f41008..5314211f51 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,15 +24,25 @@ ENV PYTHONFAULTHANDLER=1 \ PIP_NO_CACHE_DIR=off \ PIP_DISABLE_PIP_VERSION_CHECK=on \ PIP_DEFAULT_TIMEOUT=100 \ - POETRY_VERSION=1.1.4 + POETRY_VERSION=1.1.4 \ + NODE_VERSION=12.22.1 # Install nginx, base system COPY deploy/docker/production/install_nginx.sh / RUN bash /install_nginx.sh && \ apt-get update && \ - apt-get install -y curl vim emacs postgresql-client net-tools && \ - curl -sL https://deb.nodesource.com/setup_12.x | bash - && \ - apt-get install -y ca-certificates nodejs npm + apt-get install -y curl vim emacs postgresql-client net-tools ca-certificates + + +ENV NVM_DIR=/root/.nvm +RUN apt install -y curl +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash +RUN . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION} +RUN . "$NVM_DIR/nvm.sh" && nvm use v${NODE_VERSION} +RUN . "$NVM_DIR/nvm.sh" && nvm alias default v${NODE_VERSION} +ENV PATH="/root/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}" +RUN node --version +RUN npm --version # Configure CGAP User (nginx) WORKDIR /home/nginx From 19f192468039c088f46596fffce5fe3b009bfed6 Mon Sep 17 00:00:00 2001 From: Kent Pitman Date: Thu, 17 Jun 2021 16:17:08 -0400 Subject: [PATCH 107/120] Add a way to control ENV_NAME and AWS_ACCOUNT in docker commands of Makefile. --- Makefile | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index b709e688fa..a4072501fd 100644 --- a/Makefile +++ b/Makefile @@ -184,13 +184,27 @@ deploy-docker-local: deploy-docker-local-daemon: docker-compose up -d -V +ENV_NAME ?= cgap-mastertest +AWS_ACCOUNT ?= 645819926742 + ecr-login: - aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 645819926742.dkr.ecr.us-east-1.amazonaws.com + @echo "Making ecr-login AWS_ACCOUNT=${AWS_ACCOUNT} ..." + aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin ${AWS_ACCOUNT}.dkr.ecr.us-east-1.amazonaws.com + +rebuild-docker-production: + @echo "Remaking build-docker-production AWS_ACCOUNT=${AWS_ACCOUNT} ENV_NAME=${ENV_NAME} ..." + docker build -t ${ENV_NAME}:latest . --no-cache + make tag-and-push-docker-production build-docker-production: - docker build -t cgap-mastertest:latest . - docker tag cgap-mastertest:latest 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest - docker push 645819926742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest:latest + @echo "Making build-docker-production AWS_ACCOUNT=${AWS_ACCOUNT} ENV_NAME=${ENV_NAME} ..." + docker build -t ${ENV_NAME}:latest . + make tag-and-push-docker-production + +tag-and-push-docker-production: + @echo "Making tag-and-push-docker-production AWS_ACCOUNT=${AWS_ACCOUNT} ENV_NAME=${ENV_NAME} ..." + docker tag ${ENV_NAME}:latest ${AWS_ACCOUNT}.dkr.ecr.us-east-1.amazonaws.com/${ENV_NAME}:latest + docker push ${AWS_ACCOUNT}.dkr.ecr.us-east-1.amazonaws.com/${ENV_NAME}:latest help: @make info From 6423773e8f2bb2e890e82f71bf7c101f2075375c Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 17 Jun 2021 17:26:55 -0400 Subject: [PATCH 108/120] fix nvm perms --- Dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5314211f51..163614b160 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,18 +33,20 @@ RUN bash /install_nginx.sh && \ apt-get update && \ apt-get install -y curl vim emacs postgresql-client net-tools ca-certificates +# Configure CGAP User (nginx) +WORKDIR /home/nginx/.nvm -ENV NVM_DIR=/root/.nvm +# Install Node +ENV NVM_DIR=/home/nginx/.nvm RUN apt install -y curl RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash RUN . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION} RUN . "$NVM_DIR/nvm.sh" && nvm use v${NODE_VERSION} RUN . "$NVM_DIR/nvm.sh" && nvm alias default v${NODE_VERSION} -ENV PATH="/root/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}" +ENV PATH="/home/nginx/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}" RUN node --version RUN npm --version -# Configure CGAP User (nginx) WORKDIR /home/nginx # Configure venv From c78c3f54548e2aabb05ad95894f4cdf4b347f3e9 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Mon, 21 Jun 2021 11:55:09 -0400 Subject: [PATCH 109/120] restore backwards compatibility --- .ebextensions/01_apache.config | 27 ++++ .ebextensions/04_upgrade_mod_wsgi.config | 31 ++++ .ebextensions/05_set_wsgi.config | 179 +++++++++++++++++++++++ .ebextensions/09_cloudwatch.config | 26 ++++ .ebextensions/10_set_timeout.config | 4 + .ebextensions/11_logs.config | 47 ++++++ .ebextensions/13_run_npm.config | 37 +++++ .ebextensions/20_packages.config | 77 ++++++++++ parts/production-indexer/wsgi | 24 +++ parts/production-ingestion-listener/wsgi | 24 +++ parts/production/wsgi | 24 +++ src/encoded/commands/load_access_keys.py | 7 +- 12 files changed, 505 insertions(+), 2 deletions(-) create mode 100644 .ebextensions/01_apache.config create mode 100644 .ebextensions/04_upgrade_mod_wsgi.config create mode 100644 .ebextensions/05_set_wsgi.config create mode 100644 .ebextensions/09_cloudwatch.config create mode 100644 .ebextensions/10_set_timeout.config create mode 100644 .ebextensions/11_logs.config create mode 100644 .ebextensions/13_run_npm.config create mode 100644 .ebextensions/20_packages.config create mode 100644 parts/production-indexer/wsgi create mode 100644 parts/production-ingestion-listener/wsgi create mode 100755 parts/production/wsgi diff --git a/.ebextensions/01_apache.config b/.ebextensions/01_apache.config new file mode 100644 index 0000000000..394737b62c --- /dev/null +++ b/.ebextensions/01_apache.config @@ -0,0 +1,27 @@ +files: + "/etc/httpd/conf.modules.d/00-mpm.conf": + mode: "000644" + owner: root + group: root + content: | + # use prefork mpm. + # event mpm may be best for mod_wsgi, but doesn't seem to work with EB + LoadModule mpm_prefork_module modules/mod_mpm_prefork.so + + # For convenience, also set the following Apache directives here + # Find some Apache config tips for Elasticbeanstalk below: + # https://aws.amazon.com/premiumsupport/knowledge-center/apache-backend-elb/ + StartServers 20 + MinSpareServers 20 + MaxSpareServers 20 + MaxRequestWorkers 20 + ServerLimit 20 + MaxConnectionsPerChild 1000 + # set Timeout higher than idle timeout on load balancer + # should match RequestReadTimeout set in ../conf.d/mod_reqtimeout.conf + Timeout 62 + KeepAlive On + # set KeepAliveTimeout higher than Timeout + KeepAliveTimeout 63 + MaxKeepAliveRequests 100 + LogFormat "%{X-Forwarded-For}i %h %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"" combined diff --git a/.ebextensions/04_upgrade_mod_wsgi.config b/.ebextensions/04_upgrade_mod_wsgi.config new file mode 100644 index 0000000000..9db1c847bf --- /dev/null +++ b/.ebextensions/04_upgrade_mod_wsgi.config @@ -0,0 +1,31 @@ +packages: + yum: + git: [] + gcc-c++: [] + +files: + "/tmp/update-wsgi.sh" : + mode: "000755" + owner: root + group: root + content: | + # https://stackoverflow.com/questions/33818007/error-building-installing-mod-wsgi-on-aws-elasticbeanstalk-for-django-deployment + # httpd24 comes with apxs, which is needed to compile mod_wsgi -kmp 9-Mar-2020 + sudo yum install -y httpd24-devel + # update mod_wsgi version + # https://serverfault.com/a/885445 + # https://modwsgi.readthedocs.io/en/develop/user-guides/quick-installation-guide.html + cd /tmp + wget -q "https://github.com/GrahamDumpleton/mod_wsgi/archive/4.6.5.tar.gz" && \ + tar -xzf '4.6.5.tar.gz' && \ + cd mod_wsgi-4.6.5/ && \ + sudo ./configure --with-python=/opt/python/run/venv/bin/python && \ + sudo make && \ + sudo make install && \ + sudo service httpd restart + sudo make clean + +commands: + 01_mod_wsgi_update: + command: /tmp/update-wsgi.sh &> /var/log/mod_wsgi_update.log + cwd: /tmp diff --git a/.ebextensions/05_set_wsgi.config b/.ebextensions/05_set_wsgi.config new file mode 100644 index 0000000000..9bfebbede8 --- /dev/null +++ b/.ebextensions/05_set_wsgi.config @@ -0,0 +1,179 @@ +files: + "/opt/elasticbeanstalk/local/override_wsgi_conf.py": + mode: "000755" + owner: root + group: root + content: | + #!/usr/bin/env python + # This file creates a Python script that runs on appdeploy and configdeploy + # pre hooks to override the WSGI config file + # See https://forums.aws.amazon.com/thread.jspa?threadID=163369 + + import os + import sys + sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + import config + + # remove old wsgi hooks and config files + old_wsgi_hooks_conf = [ + '/opt/elasticbeanstalk/hooks/configdeploy/pre/99patchwsgi.py', + '/opt/elasticbeanstalk/hooks/appdeploy/pre/99patchwsgi.py', + '/etc/httpd/conf.d/encoded-apache.conf', + '/etc/httpd/wsgi.conf.d/extra_config.conf' + ] + for old_wsgi_file in old_wsgi_hooks_conf: + if os.path.exists(old_wsgi_file): + os.remove(old_wsgi_file) + else: + print("File %s does not exist, so needn't be deleted." % old_wsgi_file) + + MY_APACHE_TEMPLATE = ''' + # Customized wsgi.conf. If you're seeing this, good! + + # No need to load modules/mod_wsgi.so, since it's already loaded here: + # /etc/httpd/conf.modules.d/10-wsgi.conf + + WSGIPythonHome /opt/python/run/baselinenv + WSGISocketPrefix run/wsgi + WSGIRestrictEmbedded On + WSGIPassAuthorization On + + ### START originally in encoded-apache.conf + LogLevel info + + + Order deny,allow + Allow from all + + Require all granted + + + + + Order deny,allow + Allow from all + + Require all granted + + # Limit upload size to 500 MB (375MB before base64 encoding) + LimitRequestBody 524288000 + # Apache adds -gzip to outgoing ETag in mod_deflate, remove inbound. + # https://issues.apache.org/bugzilla/show_bug.cgi?id=39727 + RequestHeader edit If-Match -gzip\"$ \" + RequestHeader edit If-None-Match -gzip\"$ \" + + + # Serve static resources directly from Apache + Alias /favicon.ico /opt/python/current/app/src/encoded/static/img/favicon.ico + + # Compress JSON responses. + AddOutputFilterByType DEFLATE application/javascript application/json text/css text/html text/javascript + + # Source map type (to enable compression) + + ForceType application/json + + + RewriteEngine On + + # Proxy internal redirects for file downloads + SSLProxyEngine On + RewriteCond %{ENV:REDIRECT_STATUS} . + RewriteRule ^/_proxy/(.+)$ $1 [proxy] + + # Redirect http to https from the load balancer + # https://stackoverflow.com/a/38751749 + + # Note in 4DN the rewrite condition is: + # RewriteCond %{HTTP_HOST} ^(data\.4dnucleome\.org|testportal\.4dnucleome\.org)$ + # but if CGAP has the equivalent of a testportal, I don't know about it. -kmp 2-Sep-2020 + RewriteCond %{HTTP_HOST} ^(cgap\.hms\.harvard\.edu)$ + RewriteRule (.*) https://%{HTTP_HOST}%{REQUEST_URI} [redirect=permanent,last,qsappend] + + ### END originally in encoded-apache.conf + + + + # force use of main Python interpreter + WSGIApplicationGroup %{GLOBAL} + + ### START originally in /etc/httpd/wsgi.conf.d/extra_config.conf + Header always set Access-Control-Allow-Origin "*" + Header always set Access-Control-Allow-Methods "GET, HEAD, OPTIONS" + Header always set Access-Control-Allow-Headers "Accept, Origin, Range, X-Requested-With" + Header always set Access-Control-Expose-Headers: "Content-Length, Content-Range, Content-Type" + RewriteCond %{REQUEST_METHOD} OPTIONS + RewriteRule ^ - [redirect=200,last] + + # Indexer. Configure first to avoid catchall '/' + # Use default number of processes=1, so this is not a multiprocess daemon + WSGIDaemonProcess encoded-indexer user=wsgi group=wsgi threads=1 python-path=/opt/python/current/app:/opt/python/run/venv/lib64/python3.6/site-packages:/opt/python/run/venv/lib/python3.6/site-packages display-name=encoded-indexer + WSGIScriptAlias /_indexer /opt/python/current/app/parts/production-indexer/wsgi process-group=encoded-indexer application-group=%{GLOBAL} + + # Ingestion listener (very similar config to the indexer, but not multi-processed the same way) + WSGIDaemonProcess encoded-ingestion-listener user=wsgi group=wsgi threads=1 python-path=/opt/python/current/app:/opt/python/run/venv/lib64/python3.6/site-packages:/opt/python/run/venv/lib/python3.6/site-packages display-name=encoded-ingestion-listener + WSGIScriptAlias /_ingestion_listener /opt/python/current/app/parts/production-ingestion-listener/wsgi process-group=encoded-ingestion-listener application-group=%{GLOBAL} + + # https://github.com/GrahamDumpleton/mod_wsgi/issues/2 + # Try Graham Dumpleton's fix since we have upgraded WSGI + # Original fix (next 3 lines) + # SetEnvIf Request_Method HEAD X_REQUEST_METHOD=HEAD + # LogFormat "%v:%p %h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\" %{X-Stats}o&server_time=%D" vhost_combined_stats + # ErrorLogFormat "%M" + + ### END originally in /etc/httpd/wsgi.conf.d/extra_config.conf + + Alias /static/ /opt/python/current/app/src/encoded/static/ + + Order allow,deny + Allow from all + + + WSGIScriptAlias / /opt/python/current/app/parts/production/wsgi + + + Require all granted + + + # Customized WSGIDaemonProcess settings + # https://modwsgi.readthedocs.io/en/latest/configuration-directives/WSGIDaemonProcess.html + # Notes: + # - might want to add "request-timeout", though it would interfere with long running requests + # - "queue-timeout" should not occur since Apache servers match total WSGI threads. Set to value of Apache Timeout + # - maybe use "restart-interval" than "maximum-requests", but handle long requests + + WSGIDaemonProcess wsgi processes=5 threads=4 display-name='%{GROUP}' python-path=/opt/python/current/app:/opt/python/run/venv/lib64/python3.6/site-packages:/opt/python/run/venv/lib/python3.6/site-packages user=wsgi group=wsgi home=/opt/python/current/app graceful-timeout=30 deadlock-timeout=60 queue-timeout=62 maximum-requests=1000 + WSGIProcessGroup wsgi + + ''' + + + def main(): + if 'EB_SYSTEM_STARTUP' in os.environ: + return + try: + WSGI_STAGING_CONFIG = config.get_container_config('wsgi_staging_config') + print("WSGI_STAGING_CONFIG=", WSGI_STAGING_CONFIG) + open(WSGI_STAGING_CONFIG, 'w').write(MY_APACHE_TEMPLATE) + except Exception, e: + config.emit_error_event(config.USER_ERROR_MESSAGES['badappconfig']) + config.diagnostic("Error generating config during configdeploy/pre: %s" % str(e)) + sys.exit(1) + + + if __name__ == '__main__': + config.configure_stdout_logger() + main() + +commands: + 01_app_deploy_dir: + command: "mkdir -p /opt/elasticbeanstalk/hooks/appdeploy/pre" + + 02_config_deploy_dir: + command: "mkdir -p /opt/elasticbeanstalk/hooks/configdeploy/pre" + + 03_app_deploy_file: + command: "cp -p /opt/elasticbeanstalk/local/override_wsgi_conf.py /opt/elasticbeanstalk/hooks/appdeploy/pre/90_override_wsgi_conf.py" + + 04_config_deploy_file: + command: "cp -p /opt/elasticbeanstalk/local/override_wsgi_conf.py /opt/elasticbeanstalk/hooks/configdeploy/pre/90_override_wsgi_conf.py" diff --git a/.ebextensions/09_cloudwatch.config b/.ebextensions/09_cloudwatch.config new file mode 100644 index 0000000000..89a6f587c0 --- /dev/null +++ b/.ebextensions/09_cloudwatch.config @@ -0,0 +1,26 @@ +packages: + yum: + perl-DateTime: [] + perl-Sys-Syslog: [] + perl-LWP-Protocol-https: [] + perl-Switch: [] + perl-URI: [] + perl-Bundle-LWP: [] + +sources: + /opt/cloudwatch: https://aws-cloudwatch.s3.amazonaws.com/downloads/CloudWatchMonitoringScripts-1.2.1.zip + +container_commands: + 01-setupcron: + command: | + echo '*/5 * * * * root perl /opt/cloudwatch/aws-scripts-mon/mon-put-instance-data.pl `{"Fn::GetOptionSetting" : { "OptionName" : "CloudWatchMetrics", "DefaultValue" : "--mem-util --disk-space-util --disk-path=/" }}` > /dev/null' > /etc/cron.d/cwpump + 02-changeperm: + command: chmod 644 /etc/cron.d/cwpump + 03-changeperm: + command: chmod u+x /opt/cloudwatch/aws-scripts-mon/mon-put-instance-data.pl + +option_settings: + "aws:autoscaling:launchconfiguration" : + IamInstanceProfile : "aws-elasticbeanstalk-ec2-role" + "aws:elasticbeanstalk:customoption" : + CloudWatchMetrics : "--mem-util --mem-used --mem-avail --disk-space-util --disk-space-used --disk-space-avail --disk-path=/ --auto-scaling" diff --git a/.ebextensions/10_set_timeout.config b/.ebextensions/10_set_timeout.config new file mode 100644 index 0000000000..099702df61 --- /dev/null +++ b/.ebextensions/10_set_timeout.config @@ -0,0 +1,4 @@ +option_settings: + - namespace: aws:elasticbeanstalk:command + option_name: Timeout + value: 3300 diff --git a/.ebextensions/11_logs.config b/.ebextensions/11_logs.config new file mode 100644 index 0000000000..1953f91e11 --- /dev/null +++ b/.ebextensions/11_logs.config @@ -0,0 +1,47 @@ +files: + "/opt/elasticbeanstalk/tasks/bundlelogs.d/deploy.conf" : + mode: "000644" + owner: root + group: root + content: | + /var/log/deploy.log + "/opt/elasticbeanstalk/tasks/bundlelogs.d/create_mapping.conf" : + mode: "000644" + owner: root + group: root + content: | + /var/log/create_mapping.log + "/etc/logrotate.elasticbeanstalk.hourly/logrotate.elasticbeanstalk.httpd.conf": + mode: "000644" + owner: root + group: root + content: | + /var/log/httpd/* { + copytruncate + size 10M + missingok + notifempty + rotate 24 + dateext + dateformat -%s + olddir /var/log/httpd/rotated + } + + # Previous configuration caused problems with graceful restart + # dropping connections. Initially switched to graceful b/c reload + # caused connections to hang. Finally switch to using copytruncate + # /var/log/httpd/* { + # size 10M + # missingok + # notifempty + # rotate 5 + # sharedscripts + # compress + # dateext + # dateformat -%s + # create + # postrotate + # /sbin/service httpd graceful > /dev/null 2>/dev/null || true + # endscript + # olddir /var/log/httpd/rotated + #} diff --git a/.ebextensions/13_run_npm.config b/.ebextensions/13_run_npm.config new file mode 100644 index 0000000000..041c390660 --- /dev/null +++ b/.ebextensions/13_run_npm.config @@ -0,0 +1,37 @@ +files: + "/opt/elasticbeanstalk/hooks/appdeploy/post/99_run_npm.sh": + mode: "000755" + owner: root + group: root + content: | + #!/bin/bash + + # run npm install and npm build needed for application + # these were previously run in bin/buildout as part of + # container_commands in 20_packages.config, but that + # stopped working. + # commands taken from buildout.cfg + + cd /opt/python/current/app + + echo "Running NPM install in post-deploy hook 99_run_npm..." >> /var/log/deploy.log + echo "Running NPM build in 99_run_npm..." + + su -c "npm install --no-fund --no-progress --python=/opt/python/run/venv/bin/python" ec2-user >> /var/log/deploy.log + + echo "Running NPM build in 99_run_npm..." >> /var/log/deploy.log + echo "Running NPM build in 99_run_npm..." + + su -c "npm run build" ec2-user >> /var/log/deploy.log + + echo "Running NPM build-scss in post-deploy hook 99_run_npm..." >> /var/log/deploy.log + echo "Running NPM build in 99_run_npm..." + + su -c "npm run build-scss" ec2-user >> /var/log/deploy.log + + echo "Finished NPM build in 99_run_npm." >> /var/log/deploy.log + echo "Finished NPM build in 99_run_npm." + + echo "Restarting Apache" + + sudo service httpd restart diff --git a/.ebextensions/20_packages.config b/.ebextensions/20_packages.config new file mode 100644 index 0000000000..d9a5c858c3 --- /dev/null +++ b/.ebextensions/20_packages.config @@ -0,0 +1,77 @@ +packages: + yum: + git: [] + postgresql93-devel: [] + libffi-devel: [] + libjpeg-turbo-devel: [] + libtiff: [] + bsdtar: [] + graphviz: [] + mod24_ssl: [] + gcc-c++: [] + gcc: [] + +container_commands: + 0000_nodejs_install: + command: "curl --silent --location https://rpm.nodesource.com/setup_12.x | sudo bash - && yum install nodejs -y && node --version >> /var/log/deploy.log" + 0100_setup_wsgi_home: + command: "mkdir -p /home/wsgi && chown wsgi:wsgi /home/wsgi" + 0190_mostly_ugprade_pip: # pin a version believed to work + command: "source /opt/python/run/venv/bin/activate && pip install --upgrade pip==21.0.1 >> /var/log/deploy.log" + 0191_check_pip_version_anew: + command: "source /opt/python/run/venv/bin/activate && pip --version" + 0200_install_poetry: # pin a version believed to work + command: "source /opt/python/run/venv/bin/activate && pip install poetry==1.1.4 >> /var/log/deploy.log" + 0201_install_poetry: + command: "source /opt/python/run/venv/bin/activate && poetry --version" + 0210_superstitiously_make_clean: + command: "make clean" + 0217_check_pip_state: + command: "source /opt/python/run/venv/bin/activate && pip freeze" + 0218_uninstall_setuptools: + command: "source /opt/python/run/venv/bin/activate && pip uninstall -y setuptools" + 0219_install_setuptools: + command: "source /opt/python/run/venv/bin/activate && pip install setuptools==44.1.1" # known to work + 0220_populate_venv: + command: "source /opt/python/run/venv/bin/activate && poetry install" + 0221_get_aws_ips: + command: "make aws-ip-ranges" + 0400_superstitiously_activate_venv: + command: "source /opt/python/run/venv/bin/activate" + 0420_still_more_debugging_info: + command: "source /opt/python/run/venv/bin/activate && echo $PATH" + 0480_npm_tmp_perms: + command: "chown -R ec2-user /tmp" + 0490_app_bundle_perms: + command: "chown -R ec2-user /opt/python/bundle/" + 0500_secret_key: + command: "cat /dev/urandom | head -c 256 | base64 > session-secret.b64" + 0600_generate_production_ini: + command: "source /opt/python/run/venv/bin/activate && python deploy/generate_production_ini.py" + 0696_pip_install_encoded: + command: "source /opt/python/run/venv/bin/activate && python setup_eb.py develop --verbose && make fix-dist-info" + 0700_clear_db_es_contents: + command: "source /opt/python/run/venv/bin/activate && clear-db-es-contents production.ini --app-name app --skip-es --env fourfront-cgapdev >> /var/log/deploy.log" + leader_only: true + 0810_elastic_search_mapping: + command: "source /opt/python/run/venv/bin/activate && create-mapping-on-deploy production.ini --app-name app --clear-queue &> /var/log/create_mapping.log" + leader_only: true + 0820_load_dummy_data: + command: "source /opt/python/run/venv/bin/activate && load-data production.ini --app-name app >> /var/log/deploy.log" + leader_only: true + 0830_load_access_keys: + command: "source /opt/python/run/venv/bin/activate && load-access-keys production.ini --app-name app >> /var/log/deploy.log" + leader_only: true + 0900_restart_apache: + command: "sudo service httpd restart" + +option_settings: + "aws:elasticbeanstalk:application:environment": + "LC_ALL" : "en_US.UTF-8" + "LANG" : "en_US.UTF-8" + "aws:elasticbeanstalk:container:python:staticfiles": + "/static/": "src/encoded/static/" + "aws:elasticbeanstalk:container:python": + WSGIPath: parts/production/wsgi + NumProcesses: 5 + NumThreads: 4 diff --git a/parts/production-indexer/wsgi b/parts/production-indexer/wsgi new file mode 100644 index 0000000000..85af513622 --- /dev/null +++ b/parts/production-indexer/wsgi @@ -0,0 +1,24 @@ +import os + +from logging.config import fileConfig +from paste.deploy import loadapp + + +# This is sort of like as if we did: +# CONFIG_FILE = "/opt/python/current/app/production.ini" +# except we want to bind to the bundle name so that when links change +# between /opt/python/ondeck/app and /opt/python/current/app +# we continue to point to the right directory. +# NOTE: If this doesn't work, we might try using "../../production.ini" + +FILE_DIR = os.path.dirname(os.path.abspath(__file__)) + +PARTS_DIR = os.path.dirname(FILE_DIR) + +HOME_DIR = os.path.dirname(PARTS_DIR) + +CONFIG_FILE = os.path.join(HOME_DIR, "production.ini") + +fileConfig(CONFIG_FILE) + +application = loadapp("config:" + CONFIG_FILE, name="indexer") diff --git a/parts/production-ingestion-listener/wsgi b/parts/production-ingestion-listener/wsgi new file mode 100644 index 0000000000..6bef8f1f44 --- /dev/null +++ b/parts/production-ingestion-listener/wsgi @@ -0,0 +1,24 @@ +import os + +from logging.config import fileConfig +from paste.deploy import loadapp + + +# This is sort of like as if we did: +# CONFIG_FILE = "/opt/python/current/app/production.ini" +# except we want to bind to the bundle name so that when links change +# between /opt/python/ondeck/app and /opt/python/current/app +# we continue to point to the right directory. +# NOTE: If this doesn't work, we might try using "../../production.ini" + +FILE_DIR = os.path.dirname(os.path.abspath(__file__)) + +PARTS_DIR = os.path.dirname(FILE_DIR) + +HOME_DIR = os.path.dirname(PARTS_DIR) + +CONFIG_FILE = os.path.join(HOME_DIR, "production.ini") + +fileConfig(CONFIG_FILE) + +application = loadapp("config:" + CONFIG_FILE, name="ingester") diff --git a/parts/production/wsgi b/parts/production/wsgi new file mode 100755 index 0000000000..742fdec457 --- /dev/null +++ b/parts/production/wsgi @@ -0,0 +1,24 @@ +import os + +from logging.config import fileConfig +from paste.deploy import loadapp + + +# This is sort of like as if we did: +# CONFIG_FILE = "/opt/python/current/app/production.ini" +# except we want to bind to the bundle name so that when links change +# between /opt/python/ondeck/app and /opt/python/current/app +# we continue to point to the right directory. +# NOTE: If this doesn't work, we might try using "../../production.ini" + +FILE_DIR = os.path.dirname(os.path.abspath(__file__)) + +PARTS_DIR = os.path.dirname(FILE_DIR) + +HOME_DIR = os.path.dirname(PARTS_DIR) + +CONFIG_FILE = os.path.join(HOME_DIR, "production.ini") + +fileConfig(CONFIG_FILE) + +application = loadapp("config:" + CONFIG_FILE, name=None) diff --git a/src/encoded/commands/load_access_keys.py b/src/encoded/commands/load_access_keys.py index 295ef8653c..f292948f80 100644 --- a/src/encoded/commands/load_access_keys.py +++ b/src/encoded/commands/load_access_keys.py @@ -7,6 +7,7 @@ from pyramid.paster import get_app from webtest import AppError from dcicutils.misc_utils import TestApp +from dcicutils.beanstalk_utils import get_beanstalk_real_url from dcicutils.cloudformation_utils import get_ecs_real_url from dcicutils.secrets_utils import assume_identity @@ -56,7 +57,9 @@ def generate_access_key(testapp, env, user_uuid, description): Returns: dict: access key contents with server """ - server = get_ecs_real_url(env) # INCOMPATIBLE CHANGE; will break beanstalk -Will 5/6/21 + server = get_ecs_real_url(env) # try to grab from Cfn, if we are ECS env + if not server: # fallback if we are a beanstalk + server = get_beanstalk_real_url(env) access_key_req = {'user': user_uuid, 'description': description} res = testapp.post_json('/access_key', access_key_req).json return {'secret': res['secret_access_key'], @@ -121,7 +124,7 @@ def main(): ('foursight.app@gmail.com', 'access_key_foursight')] for email, key_name in to_generate: try: - user_props = testapp.get('/users/%s?datastore=database' % (email)).follow().json + user_props = testapp.get('/users/%s?datastore=database' % email).follow().json except Exception as exc: log.error('load_access_keys: could not get user %s. Exception: %s' % (email, exc)) continue From a63a9ddd5d8a989d5b20d19df8c583e9eb01cb43 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Mon, 21 Jun 2021 12:31:41 -0400 Subject: [PATCH 110/120] ensure s3 key is found --- .gitignore | 5 +++++ README.rst | 2 ++ src/encoded/commands/load_access_keys.py | 4 ++-- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 10d02188d8..fea42a1b4b 100644 --- a/.gitignore +++ b/.gitignore @@ -82,3 +82,8 @@ elasticsearch-*.deb # Used for some kinds of debugging in dcicutils, snovault, cgap & ff. DEBUGLOG-* + +# Elastic Beanstalk Files +.elasticbeanstalk/* +!.elasticbeanstalk/*.cfg.yml +!.elasticbeanstalk/*.global.yml diff --git a/README.rst b/README.rst index 7cb1c926a6..6739c73bac 100644 --- a/README.rst +++ b/README.rst @@ -16,6 +16,8 @@ Welcome to CGAP! We are a team of scientists, clinicians, and developers who aim Be warned that features are under active development and may not be stable! Visit the production deployment for the best experience. For installation and more information on getting started, see our `documentation page `_. +Note that at this time, CGAP is operating in hybrid model where some environments are deployed to AWS ElasticBeanstalk and others are deployed to AWS Elastic Container Service. The BS deployments are considered legacy and the ECS deployments + For information on how to run CGAP with Docker, see `here. <./docs/source/docker-local.rst>`_ For information on CGAP-Docker in production, see `here. <./docs/source/docker-production.rst>`_ diff --git a/src/encoded/commands/load_access_keys.py b/src/encoded/commands/load_access_keys.py index f292948f80..7d7dde41e4 100644 --- a/src/encoded/commands/load_access_keys.py +++ b/src/encoded/commands/load_access_keys.py @@ -108,8 +108,8 @@ def main(): if args.secret_name is not None: identity = assume_identity() # automatically detects GLOBAL_APPLICATION_CONFIGURATION encrypt_key = identity.get('S3_ENCRYPT_KEY', None) # one of the secrets - if not encrypt_key: - encrypt_key = os.environ.get('S3_ENCRYPT_KEY') + if not encrypt_key: + encrypt_key = os.environ.get('S3_ENCRYPT_KEY') if not encrypt_key: raise RuntimeError('load_access_keys: must define S3_ENCRYPT_KEY in env or in GAC') From 06165fd021a0f2ef80dc76b85be58851e4a123fb Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Mon, 21 Jun 2021 12:44:25 -0400 Subject: [PATCH 111/120] catch exception in ecs url call (that rightfully fails) --- src/encoded/commands/load_access_keys.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/encoded/commands/load_access_keys.py b/src/encoded/commands/load_access_keys.py index 7d7dde41e4..b34ba2faf1 100644 --- a/src/encoded/commands/load_access_keys.py +++ b/src/encoded/commands/load_access_keys.py @@ -57,8 +57,11 @@ def generate_access_key(testapp, env, user_uuid, description): Returns: dict: access key contents with server """ - server = get_ecs_real_url(env) # try to grab from Cfn, if we are ECS env - if not server: # fallback if we are a beanstalk + try: + server = get_ecs_real_url(env) # try to grab from Cfn, if we are ECS env + except Exception: + server = get_beanstalk_real_url(env) + if not server: server = get_beanstalk_real_url(env) access_key_req = {'user': user_uuid, 'description': description} res = testapp.post_json('/access_key', access_key_req).json From 1df4afcfd45eafe040e8a93a696cbf94165d8a6a Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 22 Jun 2021 13:45:53 -0400 Subject: [PATCH 112/120] small changes in prep for merge --- Makefile | 2 +- README.rst | 4 +++- .../docker/production/entrypoint_deployment.sh | 2 +- docs/source/docker-local.rst | 2 +- docs/source/docker-production.rst | 17 +++++++++-------- src/encoded/visualization.py | 7 +++++-- 6 files changed, 20 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index a4072501fd..7ab6856671 100644 --- a/Makefile +++ b/Makefile @@ -148,7 +148,7 @@ test-any: poetry run python -m pytest -vv -r w --timeout=200 test-npm: - poetry run python -m pytest -vv -r w --timeout=200 -m "not manual and not integratedx and not performance and not broken and not sloppy and not indexing" + poetry run python -m pytest -vv -r w --timeout=300 -m "not manual and not integratedx and not performance and not broken and not sloppy and not indexing" test-unit: poetry run python -m pytest -vv -r w --timeout=200 -m "not manual and not integratedx and not performance and not broken and not sloppy and indexing" diff --git a/README.rst b/README.rst index 6739c73bac..68195f0bfc 100644 --- a/README.rst +++ b/README.rst @@ -16,7 +16,7 @@ Welcome to CGAP! We are a team of scientists, clinicians, and developers who aim Be warned that features are under active development and may not be stable! Visit the production deployment for the best experience. For installation and more information on getting started, see our `documentation page `_. -Note that at this time, CGAP is operating in hybrid model where some environments are deployed to AWS ElasticBeanstalk and others are deployed to AWS Elastic Container Service. The BS deployments are considered legacy and the ECS deployments +Note that at this time, CGAP is operating in hybrid model where some environments are deployed to AWS ElasticBeanstalk and others are deployed to AWS Elastic Container Service. The BS deployments are referred to as "legacy deployments" and the ECS deployments are referred to as "alpha deployments". For information on how to run CGAP with Docker, see `here. <./docs/source/docker-local.rst>`_ @@ -28,9 +28,11 @@ Navigating this Repository Important directories/files are outlined below. * ``.github/workflows/`` contains Github Action Workflows + * ``.ebextensions/`` contains the Elastic Beanstalk provisioning scripts * ``bin/`` contains the few remaining executables * ``deploy/docker`` contains containerization related scripts/configuration * ``docs/ contains`` documentation + * ``parts/`` contains WSGI entry points for the Beanstalk setup * ``scripts/`` contains misc scripts * ``src/encoded/`` where the code is * ``.dockerignore`` specifies paths ignored by the Dockerfile diff --git a/deploy/docker/production/entrypoint_deployment.sh b/deploy/docker/production/entrypoint_deployment.sh index 7b0c5b2019..a9acec7a25 100644 --- a/deploy/docker/production/entrypoint_deployment.sh +++ b/deploy/docker/production/entrypoint_deployment.sh @@ -19,6 +19,6 @@ poetry run create-mapping-on-deploy production.ini --app-name app # Load access keys # Note that the secret name must match that which was created for this environment -poetry run load-access-keys production.ini --app-name app --secret-name dev/beanstalk/cgap-dev +poetry run load-access-keys production.ini --app-name app --secret-name "$IDENTITY" exit 0 diff --git a/docs/source/docker-local.rst b/docs/source/docker-local.rst index 84aeabefb2..927e0cb153 100644 --- a/docs/source/docker-local.rst +++ b/docs/source/docker-local.rst @@ -53,7 +53,7 @@ To access the running container:: Alternative Configuration with Local ElasticSearch ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -ElasticSearch is too compute intensive to virtualize on most machines. For this reason we use the CGAP test ES cluster for this deployment instead of spinning up an ES cluster in Docker. It is possible however to modify ``docker-compose.yml`` to spinup a local Elasticsearch +ElasticSearch is too compute intensive to virtualize on most machines. For this reason we use the CGAP test ES cluster for this deployment instead of spinning up an ES cluster in Docker. It is possible however to modify ``docker-compose.yml`` to spinup a local Elasticsearch. If your machine can handle this it is the ideal setup, but typically things are just too slow for it to be viable (YMMV). Common Issues diff --git a/docs/source/docker-production.rst b/docs/source/docker-production.rst index 0d5645802d..915cfdbee8 100644 --- a/docs/source/docker-production.rst +++ b/docs/source/docker-production.rst @@ -8,15 +8,15 @@ The CGAP Application has been orchestrated into the ECS Service/Task paradigm. A +------------+--------------------------------+-----+------+------+-----+--------------------------+ | Kind | Use | Num | Spot | vCPU | Mem | Notes | +============+================================+=====+======+======+=====+==========================+ -| Portal | Services standard API requests | 4 | Yes | .25 | 512 | Needs autoscaling. | +| Portal | Services standard API requests | 4 | Yes | .25 | 512 | Needs autoscaling | +------------+--------------------------------+-----+------+------+-----+--------------------------+ -| Indexer | Hits /index at 1sec | 4 | Yes | .25 | 512 | Needs autoscaling. | -| | intervals indefinitely. | | | | | Could use CPU% | +| Indexer | Hits /index at 1sec | 4 + | Yes | .25 | 512 | Can auto-scale based on | +| | intervals indefinitely. | | | | | Queue Depth | +------------+--------------------------------+-----+------+------+-----+--------------------------+ -| Ingester | Polls SQS for ingestion tasks | 1 | No | .5 | 1024| Need API to add tasks. | +| Ingester | Polls SQS for ingestion tasks | 1 | No | .5 | 1024| Need API to add tasks | +------------+--------------------------------+-----+------+------+-----+--------------------------+ | Deployment | Triggers the standard | 0 | Yes | .24 | 512 | Run explicitly or (TODO) | -| | deployment actions. | | | | | by API. | +| | deployment actions. | | | | | by API | +------------+--------------------------------+-----+------+------+-----+--------------------------+ Building an Image @@ -24,9 +24,9 @@ Building an Image The production application configuration is in ``deploy/docker/production``. A description of all the relevant files follows. - * Dockerfile - at repo top level - configurable file that builds both all local and production images. - * docker-compose.yml - at repo top level - configures the local deployment - unused in production. - * assume_identity.py - script for pulling application configuration from Secrets Manager. Note that this secret is meant to be generated by the Datastore stack in 4dn-cloud-infra and manually filled out. + * Dockerfile - at repo top level - configurable file containing the Docker build instructions for all local and production images. + * docker-compose.yml - at repo top level - configures the local deployment, unused in production. + * assume_identity.py - script for pulling global application configuration from Secrets Manager. Note that this secret is meant to be generated by the Datastore stack in 4dn-cloud-infra and manually filled out. Note that the ``$IDENTITY`` option configures which secret is used by the application workers and is passed to ECS Task definitions by 4dn-cloud-infra. * entrypoint.sh - resolves which entrypoint is used based on ``$application_type`` * entrypoint_portal.sh - serves portal API requests * entrypoint_deployment.sh - deployment entrypoint @@ -44,6 +44,7 @@ The following instructions describe how to build and push images. Note though th 3. Again in the Makefile, replace the ECR Repo URL (NOT the tag) with the one from the output of the ECR stack in the account. 4. Run ``make ecr-login``, which should pull ECR credentials using the currently active AWS credentials. 5. Run ``make build-docker-production``. + 6. Navigate to Foursight and queue the cluster update check. After around 5 minutes, the new images should be coming online. You can monitor the progress Note that steps 4 and 6 are all that are needed to be repeated once initial setup is done, assuming you are continuously pushing to the same location. To change which ECS orchestration you are effectively deploying to, all steps must be repeated in the relevant account. diff --git a/src/encoded/visualization.py b/src/encoded/visualization.py index 293fd7d8b7..da34c06f92 100644 --- a/src/encoded/visualization.py +++ b/src/encoded/visualization.py @@ -1,3 +1,4 @@ +import os from copy import ( copy, deepcopy @@ -129,8 +130,10 @@ def get_higlass_viewconf(context, request): } # We need absolute URLs for the BAM and GnomAD Worker - # XXX: this needs a workaround - Will 6/8/21 - host_url = "http://c4ecstrialalphacgapmastertest-273357903.us-east-1.elb.amazonaws.com" + # XXX: this needs a better workaround - Will June 22 2021 + host_url = 'localhost:6543' + if 'IDENTITY' in os.environ: # detect use of global application configuration + host_url = "http://c4ecstrialalphacgapmastertest-273357903.us-east-1.elb.amazonaws.com" if request.registry.settings.get('env.name') == CGAP_ENV_WEBPROD: host_url = CGAP_PUBLIC_URL_PRD elif request.registry.settings.get('env.name') == CGAP_ENV_MASTERTEST: From 54664a82c4eaea33154e5245934017fe4ae56c77 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 24 Jun 2021 13:22:42 -0400 Subject: [PATCH 113/120] bring in utils, use any.ini method --- deploy/docker/production/assume_identity.py | 2 +- poetry.lock | 703 ++++++++++---------- pyproject.toml | 2 +- 3 files changed, 356 insertions(+), 351 deletions(-) diff --git a/deploy/docker/production/assume_identity.py b/deploy/docker/production/assume_identity.py index 831603299c..7af4ebad22 100644 --- a/deploy/docker/production/assume_identity.py +++ b/deploy/docker/production/assume_identity.py @@ -36,7 +36,7 @@ def build_production_ini_from_global_application_configuration(): # build production.ini with override_environ(**identity): - CGAPDockerIniFileManager.main() + CGAPDockerIniFileManager.build_ini_file_from_template('cgap_any_alpha.ini', 'production.ini') if __name__ == '__main__': diff --git a/poetry.lock b/poetry.lock index 3342a6a33c..defd3da760 100644 --- a/poetry.lock +++ b/poetry.lock @@ -77,7 +77,7 @@ lxml = ["lxml"] [[package]] name = "bitarray" -version = "2.1.0" +version = "2.1.3" description = "efficient arrays of booleans -- C extension" category = "main" optional = false @@ -106,291 +106,293 @@ s3transfer = ">=0.3.0,<0.4.0" [[package]] name = "boto3-stubs" -version = "1.17.89" -description = "Type annotations for boto3 1.17.89, generated by mypy-boto3-buider 4.15.0" +version = "1.17.99" +description = "Type annotations for boto3 1.17.99, generated by mypy-boto3-buider 4.18.4" category = "dev" optional = false python-versions = ">=3.6" [package.dependencies] +botocore-stubs = "*" typing-extensions = {version = "*", markers = "python_version < \"3.8\""} [package.extras] -accessanalyzer = ["mypy-boto3-accessanalyzer (==1.17.89)"] -acm = ["mypy-boto3-acm (==1.17.89)"] -acm-pca = ["mypy-boto3-acm-pca (==1.17.89)"] -alexaforbusiness = ["mypy-boto3-alexaforbusiness (==1.17.89)"] -all = ["mypy-boto3-accessanalyzer (==1.17.89)", "mypy-boto3-acm (==1.17.89)", "mypy-boto3-acm-pca (==1.17.89)", "mypy-boto3-alexaforbusiness (==1.17.89)", "mypy-boto3-amp (==1.17.89)", "mypy-boto3-amplify (==1.17.89)", "mypy-boto3-amplifybackend (==1.17.89)", "mypy-boto3-apigateway (==1.17.89)", "mypy-boto3-apigatewaymanagementapi (==1.17.89)", "mypy-boto3-apigatewayv2 (==1.17.89)", "mypy-boto3-appconfig (==1.17.89)", "mypy-boto3-appflow (==1.17.89)", "mypy-boto3-appintegrations (==1.17.89)", "mypy-boto3-application-autoscaling (==1.17.89)", "mypy-boto3-application-insights (==1.17.89)", "mypy-boto3-applicationcostprofiler (==1.17.89)", "mypy-boto3-appmesh (==1.17.89)", "mypy-boto3-apprunner (==1.17.89)", "mypy-boto3-appstream (==1.17.89)", "mypy-boto3-appsync (==1.17.89)", "mypy-boto3-athena (==1.17.89)", "mypy-boto3-auditmanager (==1.17.89)", "mypy-boto3-autoscaling (==1.17.89)", "mypy-boto3-autoscaling-plans (==1.17.89)", "mypy-boto3-backup (==1.17.89)", "mypy-boto3-batch (==1.17.89)", "mypy-boto3-braket (==1.17.89)", "mypy-boto3-budgets (==1.17.89)", "mypy-boto3-ce (==1.17.89)", "mypy-boto3-chime (==1.17.89)", "mypy-boto3-cloud9 (==1.17.89)", "mypy-boto3-clouddirectory (==1.17.89)", "mypy-boto3-cloudformation (==1.17.89)", "mypy-boto3-cloudfront (==1.17.89)", "mypy-boto3-cloudhsm (==1.17.89)", "mypy-boto3-cloudhsmv2 (==1.17.89)", "mypy-boto3-cloudsearch (==1.17.89)", "mypy-boto3-cloudsearchdomain (==1.17.89)", "mypy-boto3-cloudtrail (==1.17.89)", "mypy-boto3-cloudwatch (==1.17.89)", "mypy-boto3-codeartifact (==1.17.89)", "mypy-boto3-codebuild (==1.17.89)", "mypy-boto3-codecommit (==1.17.89)", "mypy-boto3-codedeploy (==1.17.89)", "mypy-boto3-codeguru-reviewer (==1.17.89)", "mypy-boto3-codeguruprofiler (==1.17.89)", "mypy-boto3-codepipeline (==1.17.89)", "mypy-boto3-codestar (==1.17.89)", "mypy-boto3-codestar-connections (==1.17.89)", "mypy-boto3-codestar-notifications (==1.17.89)", "mypy-boto3-cognito-identity (==1.17.89)", "mypy-boto3-cognito-idp (==1.17.89)", "mypy-boto3-cognito-sync (==1.17.89)", "mypy-boto3-comprehend (==1.17.89)", "mypy-boto3-comprehendmedical (==1.17.89)", "mypy-boto3-compute-optimizer (==1.17.89)", "mypy-boto3-config (==1.17.89)", "mypy-boto3-connect (==1.17.89)", "mypy-boto3-connect-contact-lens (==1.17.89)", "mypy-boto3-connectparticipant (==1.17.89)", "mypy-boto3-cur (==1.17.89)", "mypy-boto3-customer-profiles (==1.17.89)", "mypy-boto3-databrew (==1.17.89)", "mypy-boto3-dataexchange (==1.17.89)", "mypy-boto3-datapipeline (==1.17.89)", "mypy-boto3-datasync (==1.17.89)", "mypy-boto3-dax (==1.17.89)", "mypy-boto3-detective (==1.17.89)", "mypy-boto3-devicefarm (==1.17.89)", "mypy-boto3-devops-guru (==1.17.89)", "mypy-boto3-directconnect (==1.17.89)", "mypy-boto3-discovery (==1.17.89)", "mypy-boto3-dlm (==1.17.89)", "mypy-boto3-dms (==1.17.89)", "mypy-boto3-docdb (==1.17.89)", "mypy-boto3-ds (==1.17.89)", "mypy-boto3-dynamodb (==1.17.89)", "mypy-boto3-dynamodbstreams (==1.17.89)", "mypy-boto3-ebs (==1.17.89)", "mypy-boto3-ec2 (==1.17.89)", "mypy-boto3-ec2-instance-connect (==1.17.89)", "mypy-boto3-ecr (==1.17.89)", "mypy-boto3-ecr-public (==1.17.89)", "mypy-boto3-ecs (==1.17.89)", "mypy-boto3-efs (==1.17.89)", "mypy-boto3-eks (==1.17.89)", "mypy-boto3-elastic-inference (==1.17.89)", "mypy-boto3-elasticache (==1.17.89)", "mypy-boto3-elasticbeanstalk (==1.17.89)", "mypy-boto3-elastictranscoder (==1.17.89)", "mypy-boto3-elb (==1.17.89)", "mypy-boto3-elbv2 (==1.17.89)", "mypy-boto3-emr (==1.17.89)", "mypy-boto3-emr-containers (==1.17.89)", "mypy-boto3-es (==1.17.89)", "mypy-boto3-events (==1.17.89)", "mypy-boto3-finspace (==1.17.89)", "mypy-boto3-finspace-data (==1.17.89)", "mypy-boto3-firehose (==1.17.89)", "mypy-boto3-fis (==1.17.89)", "mypy-boto3-fms (==1.17.89)", "mypy-boto3-forecast (==1.17.89)", "mypy-boto3-forecastquery (==1.17.89)", "mypy-boto3-frauddetector (==1.17.89)", "mypy-boto3-fsx (==1.17.89)", "mypy-boto3-gamelift (==1.17.89)", "mypy-boto3-glacier (==1.17.89)", "mypy-boto3-globalaccelerator (==1.17.89)", "mypy-boto3-glue (==1.17.89)", "mypy-boto3-greengrass (==1.17.89)", "mypy-boto3-greengrassv2 (==1.17.89)", "mypy-boto3-groundstation (==1.17.89)", "mypy-boto3-guardduty (==1.17.89)", "mypy-boto3-health (==1.17.89)", "mypy-boto3-healthlake (==1.17.89)", "mypy-boto3-honeycode (==1.17.89)", "mypy-boto3-iam (==1.17.89)", "mypy-boto3-identitystore (==1.17.89)", "mypy-boto3-imagebuilder (==1.17.89)", "mypy-boto3-importexport (==1.17.89)", "mypy-boto3-inspector (==1.17.89)", "mypy-boto3-iot (==1.17.89)", "mypy-boto3-iot-data (==1.17.89)", "mypy-boto3-iot-jobs-data (==1.17.89)", "mypy-boto3-iot1click-devices (==1.17.89)", "mypy-boto3-iot1click-projects (==1.17.89)", "mypy-boto3-iotanalytics (==1.17.89)", "mypy-boto3-iotdeviceadvisor (==1.17.89)", "mypy-boto3-iotevents (==1.17.89)", "mypy-boto3-iotevents-data (==1.17.89)", "mypy-boto3-iotfleethub (==1.17.89)", "mypy-boto3-iotsecuretunneling (==1.17.89)", "mypy-boto3-iotsitewise (==1.17.89)", "mypy-boto3-iotthingsgraph (==1.17.89)", "mypy-boto3-iotwireless (==1.17.89)", "mypy-boto3-ivs (==1.17.89)", "mypy-boto3-kafka (==1.17.89)", "mypy-boto3-kendra (==1.17.89)", "mypy-boto3-kinesis (==1.17.89)", "mypy-boto3-kinesis-video-archived-media (==1.17.89)", "mypy-boto3-kinesis-video-media (==1.17.89)", "mypy-boto3-kinesis-video-signaling (==1.17.89)", "mypy-boto3-kinesisanalytics (==1.17.89)", "mypy-boto3-kinesisanalyticsv2 (==1.17.89)", "mypy-boto3-kinesisvideo (==1.17.89)", "mypy-boto3-kms (==1.17.89)", "mypy-boto3-lakeformation (==1.17.89)", "mypy-boto3-lambda (==1.17.89)", "mypy-boto3-lex-models (==1.17.89)", "mypy-boto3-lex-runtime (==1.17.89)", "mypy-boto3-lexv2-models (==1.17.89)", "mypy-boto3-lexv2-runtime (==1.17.89)", "mypy-boto3-license-manager (==1.17.89)", "mypy-boto3-lightsail (==1.17.89)", "mypy-boto3-location (==1.17.89)", "mypy-boto3-logs (==1.17.89)", "mypy-boto3-lookoutequipment (==1.17.89)", "mypy-boto3-lookoutmetrics (==1.17.89)", "mypy-boto3-lookoutvision (==1.17.89)", "mypy-boto3-machinelearning (==1.17.89)", "mypy-boto3-macie (==1.17.89)", "mypy-boto3-macie2 (==1.17.89)", "mypy-boto3-managedblockchain (==1.17.89)", "mypy-boto3-marketplace-catalog (==1.17.89)", "mypy-boto3-marketplace-entitlement (==1.17.89)", "mypy-boto3-marketplacecommerceanalytics (==1.17.89)", "mypy-boto3-mediaconnect (==1.17.89)", "mypy-boto3-mediaconvert (==1.17.89)", "mypy-boto3-medialive (==1.17.89)", "mypy-boto3-mediapackage (==1.17.89)", "mypy-boto3-mediapackage-vod (==1.17.89)", "mypy-boto3-mediastore (==1.17.89)", "mypy-boto3-mediastore-data (==1.17.89)", "mypy-boto3-mediatailor (==1.17.89)", "mypy-boto3-meteringmarketplace (==1.17.89)", "mypy-boto3-mgh (==1.17.89)", "mypy-boto3-mgn (==1.17.89)", "mypy-boto3-migrationhub-config (==1.17.89)", "mypy-boto3-mobile (==1.17.89)", "mypy-boto3-mq (==1.17.89)", "mypy-boto3-mturk (==1.17.89)", "mypy-boto3-mwaa (==1.17.89)", "mypy-boto3-neptune (==1.17.89)", "mypy-boto3-network-firewall (==1.17.89)", "mypy-boto3-networkmanager (==1.17.89)", "mypy-boto3-nimble (==1.17.89)", "mypy-boto3-opsworks (==1.17.89)", "mypy-boto3-opsworkscm (==1.17.89)", "mypy-boto3-organizations (==1.17.89)", "mypy-boto3-outposts (==1.17.89)", "mypy-boto3-personalize (==1.17.89)", "mypy-boto3-personalize-events (==1.17.89)", "mypy-boto3-personalize-runtime (==1.17.89)", "mypy-boto3-pi (==1.17.89)", "mypy-boto3-pinpoint (==1.17.89)", "mypy-boto3-pinpoint-email (==1.17.89)", "mypy-boto3-pinpoint-sms-voice (==1.17.89)", "mypy-boto3-polly (==1.17.89)", "mypy-boto3-pricing (==1.17.89)", "mypy-boto3-qldb (==1.17.89)", "mypy-boto3-qldb-session (==1.17.89)", "mypy-boto3-quicksight (==1.17.89)", "mypy-boto3-ram (==1.17.89)", "mypy-boto3-rds (==1.17.89)", "mypy-boto3-rds-data (==1.17.89)", "mypy-boto3-redshift (==1.17.89)", "mypy-boto3-redshift-data (==1.17.89)", "mypy-boto3-rekognition (==1.17.89)", "mypy-boto3-resource-groups (==1.17.89)", "mypy-boto3-resourcegroupstaggingapi (==1.17.89)", "mypy-boto3-robomaker (==1.17.89)", "mypy-boto3-route53 (==1.17.89)", "mypy-boto3-route53domains (==1.17.89)", "mypy-boto3-route53resolver (==1.17.89)", "mypy-boto3-s3 (==1.17.89)", "mypy-boto3-s3control (==1.17.89)", "mypy-boto3-s3outposts (==1.17.89)", "mypy-boto3-sagemaker (==1.17.89)", "mypy-boto3-sagemaker-a2i-runtime (==1.17.89)", "mypy-boto3-sagemaker-edge (==1.17.89)", "mypy-boto3-sagemaker-featurestore-runtime (==1.17.89)", "mypy-boto3-sagemaker-runtime (==1.17.89)", "mypy-boto3-savingsplans (==1.17.89)", "mypy-boto3-schemas (==1.17.89)", "mypy-boto3-sdb (==1.17.89)", "mypy-boto3-secretsmanager (==1.17.89)", "mypy-boto3-securityhub (==1.17.89)", "mypy-boto3-serverlessrepo (==1.17.89)", "mypy-boto3-service-quotas (==1.17.89)", "mypy-boto3-servicecatalog (==1.17.89)", "mypy-boto3-servicecatalog-appregistry (==1.17.89)", "mypy-boto3-servicediscovery (==1.17.89)", "mypy-boto3-ses (==1.17.89)", "mypy-boto3-sesv2 (==1.17.89)", "mypy-boto3-shield (==1.17.89)", "mypy-boto3-signer (==1.17.89)", "mypy-boto3-sms (==1.17.89)", "mypy-boto3-sms-voice (==1.17.89)", "mypy-boto3-snowball (==1.17.89)", "mypy-boto3-sns (==1.17.89)", "mypy-boto3-sqs (==1.17.89)", "mypy-boto3-ssm (==1.17.89)", "mypy-boto3-ssm-contacts (==1.17.89)", "mypy-boto3-ssm-incidents (==1.17.89)", "mypy-boto3-sso (==1.17.89)", "mypy-boto3-sso-admin (==1.17.89)", "mypy-boto3-sso-oidc (==1.17.89)", "mypy-boto3-stepfunctions (==1.17.89)", "mypy-boto3-storagegateway (==1.17.89)", "mypy-boto3-sts (==1.17.89)", "mypy-boto3-support (==1.17.89)", "mypy-boto3-swf (==1.17.89)", "mypy-boto3-synthetics (==1.17.89)", "mypy-boto3-textract (==1.17.89)", "mypy-boto3-timestream-query (==1.17.89)", "mypy-boto3-timestream-write (==1.17.89)", "mypy-boto3-transcribe (==1.17.89)", "mypy-boto3-transfer (==1.17.89)", "mypy-boto3-translate (==1.17.89)", "mypy-boto3-waf (==1.17.89)", "mypy-boto3-waf-regional (==1.17.89)", "mypy-boto3-wafv2 (==1.17.89)", "mypy-boto3-wellarchitected (==1.17.89)", "mypy-boto3-workdocs (==1.17.89)", "mypy-boto3-worklink (==1.17.89)", "mypy-boto3-workmail (==1.17.89)", "mypy-boto3-workmailmessageflow (==1.17.89)", "mypy-boto3-workspaces (==1.17.89)", "mypy-boto3-xray (==1.17.89)"] -amp = ["mypy-boto3-amp (==1.17.89)"] -amplify = ["mypy-boto3-amplify (==1.17.89)"] -amplifybackend = ["mypy-boto3-amplifybackend (==1.17.89)"] -apigateway = ["mypy-boto3-apigateway (==1.17.89)"] -apigatewaymanagementapi = ["mypy-boto3-apigatewaymanagementapi (==1.17.89)"] -apigatewayv2 = ["mypy-boto3-apigatewayv2 (==1.17.89)"] -appconfig = ["mypy-boto3-appconfig (==1.17.89)"] -appflow = ["mypy-boto3-appflow (==1.17.89)"] -appintegrations = ["mypy-boto3-appintegrations (==1.17.89)"] -application-autoscaling = ["mypy-boto3-application-autoscaling (==1.17.89)"] -application-insights = ["mypy-boto3-application-insights (==1.17.89)"] -applicationcostprofiler = ["mypy-boto3-applicationcostprofiler (==1.17.89)"] -appmesh = ["mypy-boto3-appmesh (==1.17.89)"] -apprunner = ["mypy-boto3-apprunner (==1.17.89)"] -appstream = ["mypy-boto3-appstream (==1.17.89)"] -appsync = ["mypy-boto3-appsync (==1.17.89)"] -athena = ["mypy-boto3-athena (==1.17.89)"] -auditmanager = ["mypy-boto3-auditmanager (==1.17.89)"] -autoscaling = ["mypy-boto3-autoscaling (==1.17.89)"] -autoscaling-plans = ["mypy-boto3-autoscaling-plans (==1.17.89)"] -backup = ["mypy-boto3-backup (==1.17.89)"] -batch = ["mypy-boto3-batch (==1.17.89)"] -braket = ["mypy-boto3-braket (==1.17.89)"] -budgets = ["mypy-boto3-budgets (==1.17.89)"] -ce = ["mypy-boto3-ce (==1.17.89)"] -chime = ["mypy-boto3-chime (==1.17.89)"] -cloud9 = ["mypy-boto3-cloud9 (==1.17.89)"] -clouddirectory = ["mypy-boto3-clouddirectory (==1.17.89)"] -cloudformation = ["mypy-boto3-cloudformation (==1.17.89)"] -cloudfront = ["mypy-boto3-cloudfront (==1.17.89)"] -cloudhsm = ["mypy-boto3-cloudhsm (==1.17.89)"] -cloudhsmv2 = ["mypy-boto3-cloudhsmv2 (==1.17.89)"] -cloudsearch = ["mypy-boto3-cloudsearch (==1.17.89)"] -cloudsearchdomain = ["mypy-boto3-cloudsearchdomain (==1.17.89)"] -cloudtrail = ["mypy-boto3-cloudtrail (==1.17.89)"] -cloudwatch = ["mypy-boto3-cloudwatch (==1.17.89)"] -codeartifact = ["mypy-boto3-codeartifact (==1.17.89)"] -codebuild = ["mypy-boto3-codebuild (==1.17.89)"] -codecommit = ["mypy-boto3-codecommit (==1.17.89)"] -codedeploy = ["mypy-boto3-codedeploy (==1.17.89)"] -codeguru-reviewer = ["mypy-boto3-codeguru-reviewer (==1.17.89)"] -codeguruprofiler = ["mypy-boto3-codeguruprofiler (==1.17.89)"] -codepipeline = ["mypy-boto3-codepipeline (==1.17.89)"] -codestar = ["mypy-boto3-codestar (==1.17.89)"] -codestar-connections = ["mypy-boto3-codestar-connections (==1.17.89)"] -codestar-notifications = ["mypy-boto3-codestar-notifications (==1.17.89)"] -cognito-identity = ["mypy-boto3-cognito-identity (==1.17.89)"] -cognito-idp = ["mypy-boto3-cognito-idp (==1.17.89)"] -cognito-sync = ["mypy-boto3-cognito-sync (==1.17.89)"] -comprehend = ["mypy-boto3-comprehend (==1.17.89)"] -comprehendmedical = ["mypy-boto3-comprehendmedical (==1.17.89)"] -compute-optimizer = ["mypy-boto3-compute-optimizer (==1.17.89)"] -config = ["mypy-boto3-config (==1.17.89)"] -connect = ["mypy-boto3-connect (==1.17.89)"] -connect-contact-lens = ["mypy-boto3-connect-contact-lens (==1.17.89)"] -connectparticipant = ["mypy-boto3-connectparticipant (==1.17.89)"] -cur = ["mypy-boto3-cur (==1.17.89)"] -customer-profiles = ["mypy-boto3-customer-profiles (==1.17.89)"] -databrew = ["mypy-boto3-databrew (==1.17.89)"] -dataexchange = ["mypy-boto3-dataexchange (==1.17.89)"] -datapipeline = ["mypy-boto3-datapipeline (==1.17.89)"] -datasync = ["mypy-boto3-datasync (==1.17.89)"] -dax = ["mypy-boto3-dax (==1.17.89)"] -detective = ["mypy-boto3-detective (==1.17.89)"] -devicefarm = ["mypy-boto3-devicefarm (==1.17.89)"] -devops-guru = ["mypy-boto3-devops-guru (==1.17.89)"] -directconnect = ["mypy-boto3-directconnect (==1.17.89)"] -discovery = ["mypy-boto3-discovery (==1.17.89)"] -dlm = ["mypy-boto3-dlm (==1.17.89)"] -dms = ["mypy-boto3-dms (==1.17.89)"] -docdb = ["mypy-boto3-docdb (==1.17.89)"] -ds = ["mypy-boto3-ds (==1.17.89)"] -dynamodb = ["mypy-boto3-dynamodb (==1.17.89)"] -dynamodbstreams = ["mypy-boto3-dynamodbstreams (==1.17.89)"] -ebs = ["mypy-boto3-ebs (==1.17.89)"] -ec2 = ["mypy-boto3-ec2 (==1.17.89)"] -ec2-instance-connect = ["mypy-boto3-ec2-instance-connect (==1.17.89)"] -ecr = ["mypy-boto3-ecr (==1.17.89)"] -ecr-public = ["mypy-boto3-ecr-public (==1.17.89)"] -ecs = ["mypy-boto3-ecs (==1.17.89)"] -efs = ["mypy-boto3-efs (==1.17.89)"] -eks = ["mypy-boto3-eks (==1.17.89)"] -elastic-inference = ["mypy-boto3-elastic-inference (==1.17.89)"] -elasticache = ["mypy-boto3-elasticache (==1.17.89)"] -elasticbeanstalk = ["mypy-boto3-elasticbeanstalk (==1.17.89)"] -elastictranscoder = ["mypy-boto3-elastictranscoder (==1.17.89)"] -elb = ["mypy-boto3-elb (==1.17.89)"] -elbv2 = ["mypy-boto3-elbv2 (==1.17.89)"] -emr = ["mypy-boto3-emr (==1.17.89)"] -emr-containers = ["mypy-boto3-emr-containers (==1.17.89)"] -es = ["mypy-boto3-es (==1.17.89)"] -essential = ["mypy-boto3-cloudformation (==1.17.89)", "mypy-boto3-dynamodb (==1.17.89)", "mypy-boto3-ec2 (==1.17.89)", "mypy-boto3-lambda (==1.17.89)", "mypy-boto3-rds (==1.17.89)", "mypy-boto3-s3 (==1.17.89)", "mypy-boto3-sqs (==1.17.89)"] -events = ["mypy-boto3-events (==1.17.89)"] -finspace = ["mypy-boto3-finspace (==1.17.89)"] -finspace-data = ["mypy-boto3-finspace-data (==1.17.89)"] -firehose = ["mypy-boto3-firehose (==1.17.89)"] -fis = ["mypy-boto3-fis (==1.17.89)"] -fms = ["mypy-boto3-fms (==1.17.89)"] -forecast = ["mypy-boto3-forecast (==1.17.89)"] -forecastquery = ["mypy-boto3-forecastquery (==1.17.89)"] -frauddetector = ["mypy-boto3-frauddetector (==1.17.89)"] -fsx = ["mypy-boto3-fsx (==1.17.89)"] -gamelift = ["mypy-boto3-gamelift (==1.17.89)"] -glacier = ["mypy-boto3-glacier (==1.17.89)"] -globalaccelerator = ["mypy-boto3-globalaccelerator (==1.17.89)"] -glue = ["mypy-boto3-glue (==1.17.89)"] -greengrass = ["mypy-boto3-greengrass (==1.17.89)"] -greengrassv2 = ["mypy-boto3-greengrassv2 (==1.17.89)"] -groundstation = ["mypy-boto3-groundstation (==1.17.89)"] -guardduty = ["mypy-boto3-guardduty (==1.17.89)"] -health = ["mypy-boto3-health (==1.17.89)"] -healthlake = ["mypy-boto3-healthlake (==1.17.89)"] -honeycode = ["mypy-boto3-honeycode (==1.17.89)"] -iam = ["mypy-boto3-iam (==1.17.89)"] -identitystore = ["mypy-boto3-identitystore (==1.17.89)"] -imagebuilder = ["mypy-boto3-imagebuilder (==1.17.89)"] -importexport = ["mypy-boto3-importexport (==1.17.89)"] -inspector = ["mypy-boto3-inspector (==1.17.89)"] -iot = ["mypy-boto3-iot (==1.17.89)"] -iot-data = ["mypy-boto3-iot-data (==1.17.89)"] -iot-jobs-data = ["mypy-boto3-iot-jobs-data (==1.17.89)"] -iot1click-devices = ["mypy-boto3-iot1click-devices (==1.17.89)"] -iot1click-projects = ["mypy-boto3-iot1click-projects (==1.17.89)"] -iotanalytics = ["mypy-boto3-iotanalytics (==1.17.89)"] -iotdeviceadvisor = ["mypy-boto3-iotdeviceadvisor (==1.17.89)"] -iotevents = ["mypy-boto3-iotevents (==1.17.89)"] -iotevents-data = ["mypy-boto3-iotevents-data (==1.17.89)"] -iotfleethub = ["mypy-boto3-iotfleethub (==1.17.89)"] -iotsecuretunneling = ["mypy-boto3-iotsecuretunneling (==1.17.89)"] -iotsitewise = ["mypy-boto3-iotsitewise (==1.17.89)"] -iotthingsgraph = ["mypy-boto3-iotthingsgraph (==1.17.89)"] -iotwireless = ["mypy-boto3-iotwireless (==1.17.89)"] -ivs = ["mypy-boto3-ivs (==1.17.89)"] -kafka = ["mypy-boto3-kafka (==1.17.89)"] -kendra = ["mypy-boto3-kendra (==1.17.89)"] -kinesis = ["mypy-boto3-kinesis (==1.17.89)"] -kinesis-video-archived-media = ["mypy-boto3-kinesis-video-archived-media (==1.17.89)"] -kinesis-video-media = ["mypy-boto3-kinesis-video-media (==1.17.89)"] -kinesis-video-signaling = ["mypy-boto3-kinesis-video-signaling (==1.17.89)"] -kinesisanalytics = ["mypy-boto3-kinesisanalytics (==1.17.89)"] -kinesisanalyticsv2 = ["mypy-boto3-kinesisanalyticsv2 (==1.17.89)"] -kinesisvideo = ["mypy-boto3-kinesisvideo (==1.17.89)"] -kms = ["mypy-boto3-kms (==1.17.89)"] -lakeformation = ["mypy-boto3-lakeformation (==1.17.89)"] -lambda = ["mypy-boto3-lambda (==1.17.89)"] -lex-models = ["mypy-boto3-lex-models (==1.17.89)"] -lex-runtime = ["mypy-boto3-lex-runtime (==1.17.89)"] -lexv2-models = ["mypy-boto3-lexv2-models (==1.17.89)"] -lexv2-runtime = ["mypy-boto3-lexv2-runtime (==1.17.89)"] -license-manager = ["mypy-boto3-license-manager (==1.17.89)"] -lightsail = ["mypy-boto3-lightsail (==1.17.89)"] -location = ["mypy-boto3-location (==1.17.89)"] -logs = ["mypy-boto3-logs (==1.17.89)"] -lookoutequipment = ["mypy-boto3-lookoutequipment (==1.17.89)"] -lookoutmetrics = ["mypy-boto3-lookoutmetrics (==1.17.89)"] -lookoutvision = ["mypy-boto3-lookoutvision (==1.17.89)"] -machinelearning = ["mypy-boto3-machinelearning (==1.17.89)"] -macie = ["mypy-boto3-macie (==1.17.89)"] -macie2 = ["mypy-boto3-macie2 (==1.17.89)"] -managedblockchain = ["mypy-boto3-managedblockchain (==1.17.89)"] -marketplace-catalog = ["mypy-boto3-marketplace-catalog (==1.17.89)"] -marketplace-entitlement = ["mypy-boto3-marketplace-entitlement (==1.17.89)"] -marketplacecommerceanalytics = ["mypy-boto3-marketplacecommerceanalytics (==1.17.89)"] -mediaconnect = ["mypy-boto3-mediaconnect (==1.17.89)"] -mediaconvert = ["mypy-boto3-mediaconvert (==1.17.89)"] -medialive = ["mypy-boto3-medialive (==1.17.89)"] -mediapackage = ["mypy-boto3-mediapackage (==1.17.89)"] -mediapackage-vod = ["mypy-boto3-mediapackage-vod (==1.17.89)"] -mediastore = ["mypy-boto3-mediastore (==1.17.89)"] -mediastore-data = ["mypy-boto3-mediastore-data (==1.17.89)"] -mediatailor = ["mypy-boto3-mediatailor (==1.17.89)"] -meteringmarketplace = ["mypy-boto3-meteringmarketplace (==1.17.89)"] -mgh = ["mypy-boto3-mgh (==1.17.89)"] -mgn = ["mypy-boto3-mgn (==1.17.89)"] -migrationhub-config = ["mypy-boto3-migrationhub-config (==1.17.89)"] -mobile = ["mypy-boto3-mobile (==1.17.89)"] -mq = ["mypy-boto3-mq (==1.17.89)"] -mturk = ["mypy-boto3-mturk (==1.17.89)"] -mwaa = ["mypy-boto3-mwaa (==1.17.89)"] -neptune = ["mypy-boto3-neptune (==1.17.89)"] -network-firewall = ["mypy-boto3-network-firewall (==1.17.89)"] -networkmanager = ["mypy-boto3-networkmanager (==1.17.89)"] -nimble = ["mypy-boto3-nimble (==1.17.89)"] -opsworks = ["mypy-boto3-opsworks (==1.17.89)"] -opsworkscm = ["mypy-boto3-opsworkscm (==1.17.89)"] -organizations = ["mypy-boto3-organizations (==1.17.89)"] -outposts = ["mypy-boto3-outposts (==1.17.89)"] -personalize = ["mypy-boto3-personalize (==1.17.89)"] -personalize-events = ["mypy-boto3-personalize-events (==1.17.89)"] -personalize-runtime = ["mypy-boto3-personalize-runtime (==1.17.89)"] -pi = ["mypy-boto3-pi (==1.17.89)"] -pinpoint = ["mypy-boto3-pinpoint (==1.17.89)"] -pinpoint-email = ["mypy-boto3-pinpoint-email (==1.17.89)"] -pinpoint-sms-voice = ["mypy-boto3-pinpoint-sms-voice (==1.17.89)"] -polly = ["mypy-boto3-polly (==1.17.89)"] -pricing = ["mypy-boto3-pricing (==1.17.89)"] -qldb = ["mypy-boto3-qldb (==1.17.89)"] -qldb-session = ["mypy-boto3-qldb-session (==1.17.89)"] -quicksight = ["mypy-boto3-quicksight (==1.17.89)"] -ram = ["mypy-boto3-ram (==1.17.89)"] -rds = ["mypy-boto3-rds (==1.17.89)"] -rds-data = ["mypy-boto3-rds-data (==1.17.89)"] -redshift = ["mypy-boto3-redshift (==1.17.89)"] -redshift-data = ["mypy-boto3-redshift-data (==1.17.89)"] -rekognition = ["mypy-boto3-rekognition (==1.17.89)"] -resource-groups = ["mypy-boto3-resource-groups (==1.17.89)"] -resourcegroupstaggingapi = ["mypy-boto3-resourcegroupstaggingapi (==1.17.89)"] -robomaker = ["mypy-boto3-robomaker (==1.17.89)"] -route53 = ["mypy-boto3-route53 (==1.17.89)"] -route53domains = ["mypy-boto3-route53domains (==1.17.89)"] -route53resolver = ["mypy-boto3-route53resolver (==1.17.89)"] -s3 = ["mypy-boto3-s3 (==1.17.89)"] -s3control = ["mypy-boto3-s3control (==1.17.89)"] -s3outposts = ["mypy-boto3-s3outposts (==1.17.89)"] -sagemaker = ["mypy-boto3-sagemaker (==1.17.89)"] -sagemaker-a2i-runtime = ["mypy-boto3-sagemaker-a2i-runtime (==1.17.89)"] -sagemaker-edge = ["mypy-boto3-sagemaker-edge (==1.17.89)"] -sagemaker-featurestore-runtime = ["mypy-boto3-sagemaker-featurestore-runtime (==1.17.89)"] -sagemaker-runtime = ["mypy-boto3-sagemaker-runtime (==1.17.89)"] -savingsplans = ["mypy-boto3-savingsplans (==1.17.89)"] -schemas = ["mypy-boto3-schemas (==1.17.89)"] -sdb = ["mypy-boto3-sdb (==1.17.89)"] -secretsmanager = ["mypy-boto3-secretsmanager (==1.17.89)"] -securityhub = ["mypy-boto3-securityhub (==1.17.89)"] -serverlessrepo = ["mypy-boto3-serverlessrepo (==1.17.89)"] -service-quotas = ["mypy-boto3-service-quotas (==1.17.89)"] -servicecatalog = ["mypy-boto3-servicecatalog (==1.17.89)"] -servicecatalog-appregistry = ["mypy-boto3-servicecatalog-appregistry (==1.17.89)"] -servicediscovery = ["mypy-boto3-servicediscovery (==1.17.89)"] -ses = ["mypy-boto3-ses (==1.17.89)"] -sesv2 = ["mypy-boto3-sesv2 (==1.17.89)"] -shield = ["mypy-boto3-shield (==1.17.89)"] -signer = ["mypy-boto3-signer (==1.17.89)"] -sms = ["mypy-boto3-sms (==1.17.89)"] -sms-voice = ["mypy-boto3-sms-voice (==1.17.89)"] -snowball = ["mypy-boto3-snowball (==1.17.89)"] -sns = ["mypy-boto3-sns (==1.17.89)"] -sqs = ["mypy-boto3-sqs (==1.17.89)"] -ssm = ["mypy-boto3-ssm (==1.17.89)"] -ssm-contacts = ["mypy-boto3-ssm-contacts (==1.17.89)"] -ssm-incidents = ["mypy-boto3-ssm-incidents (==1.17.89)"] -sso = ["mypy-boto3-sso (==1.17.89)"] -sso-admin = ["mypy-boto3-sso-admin (==1.17.89)"] -sso-oidc = ["mypy-boto3-sso-oidc (==1.17.89)"] -stepfunctions = ["mypy-boto3-stepfunctions (==1.17.89)"] -storagegateway = ["mypy-boto3-storagegateway (==1.17.89)"] -sts = ["mypy-boto3-sts (==1.17.89)"] -support = ["mypy-boto3-support (==1.17.89)"] -swf = ["mypy-boto3-swf (==1.17.89)"] -synthetics = ["mypy-boto3-synthetics (==1.17.89)"] -textract = ["mypy-boto3-textract (==1.17.89)"] -timestream-query = ["mypy-boto3-timestream-query (==1.17.89)"] -timestream-write = ["mypy-boto3-timestream-write (==1.17.89)"] -transcribe = ["mypy-boto3-transcribe (==1.17.89)"] -transfer = ["mypy-boto3-transfer (==1.17.89)"] -translate = ["mypy-boto3-translate (==1.17.89)"] -waf = ["mypy-boto3-waf (==1.17.89)"] -waf-regional = ["mypy-boto3-waf-regional (==1.17.89)"] -wafv2 = ["mypy-boto3-wafv2 (==1.17.89)"] -wellarchitected = ["mypy-boto3-wellarchitected (==1.17.89)"] -workdocs = ["mypy-boto3-workdocs (==1.17.89)"] -worklink = ["mypy-boto3-worklink (==1.17.89)"] -workmail = ["mypy-boto3-workmail (==1.17.89)"] -workmailmessageflow = ["mypy-boto3-workmailmessageflow (==1.17.89)"] -workspaces = ["mypy-boto3-workspaces (==1.17.89)"] -xray = ["mypy-boto3-xray (==1.17.89)"] +accessanalyzer = ["mypy-boto3-accessanalyzer (==1.17.99)"] +acm = ["mypy-boto3-acm (==1.17.99)"] +acm-pca = ["mypy-boto3-acm-pca (==1.17.99)"] +alexaforbusiness = ["mypy-boto3-alexaforbusiness (==1.17.99)"] +all = ["mypy-boto3-accessanalyzer (==1.17.99)", "mypy-boto3-acm (==1.17.99)", "mypy-boto3-acm-pca (==1.17.99)", "mypy-boto3-alexaforbusiness (==1.17.99)", "mypy-boto3-amp (==1.17.99)", "mypy-boto3-amplify (==1.17.99)", "mypy-boto3-amplifybackend (==1.17.99)", "mypy-boto3-apigateway (==1.17.99)", "mypy-boto3-apigatewaymanagementapi (==1.17.99)", "mypy-boto3-apigatewayv2 (==1.17.99)", "mypy-boto3-appconfig (==1.17.99)", "mypy-boto3-appflow (==1.17.99)", "mypy-boto3-appintegrations (==1.17.99)", "mypy-boto3-application-autoscaling (==1.17.99)", "mypy-boto3-application-insights (==1.17.99)", "mypy-boto3-applicationcostprofiler (==1.17.99)", "mypy-boto3-appmesh (==1.17.99)", "mypy-boto3-apprunner (==1.17.99)", "mypy-boto3-appstream (==1.17.99)", "mypy-boto3-appsync (==1.17.99)", "mypy-boto3-athena (==1.17.99)", "mypy-boto3-auditmanager (==1.17.99)", "mypy-boto3-autoscaling (==1.17.99)", "mypy-boto3-autoscaling-plans (==1.17.99)", "mypy-boto3-backup (==1.17.99)", "mypy-boto3-batch (==1.17.99)", "mypy-boto3-braket (==1.17.99)", "mypy-boto3-budgets (==1.17.99)", "mypy-boto3-ce (==1.17.99)", "mypy-boto3-chime (==1.17.99)", "mypy-boto3-cloud9 (==1.17.99)", "mypy-boto3-clouddirectory (==1.17.99)", "mypy-boto3-cloudformation (==1.17.99)", "mypy-boto3-cloudfront (==1.17.99)", "mypy-boto3-cloudhsm (==1.17.99)", "mypy-boto3-cloudhsmv2 (==1.17.99)", "mypy-boto3-cloudsearch (==1.17.99)", "mypy-boto3-cloudsearchdomain (==1.17.99)", "mypy-boto3-cloudtrail (==1.17.99)", "mypy-boto3-cloudwatch (==1.17.99)", "mypy-boto3-codeartifact (==1.17.99)", "mypy-boto3-codebuild (==1.17.99)", "mypy-boto3-codecommit (==1.17.99)", "mypy-boto3-codedeploy (==1.17.99)", "mypy-boto3-codeguru-reviewer (==1.17.99)", "mypy-boto3-codeguruprofiler (==1.17.99)", "mypy-boto3-codepipeline (==1.17.99)", "mypy-boto3-codestar (==1.17.99)", "mypy-boto3-codestar-connections (==1.17.99)", "mypy-boto3-codestar-notifications (==1.17.99)", "mypy-boto3-cognito-identity (==1.17.99)", "mypy-boto3-cognito-idp (==1.17.99)", "mypy-boto3-cognito-sync (==1.17.99)", "mypy-boto3-comprehend (==1.17.99)", "mypy-boto3-comprehendmedical (==1.17.99)", "mypy-boto3-compute-optimizer (==1.17.99)", "mypy-boto3-config (==1.17.99)", "mypy-boto3-connect (==1.17.99)", "mypy-boto3-connect-contact-lens (==1.17.99)", "mypy-boto3-connectparticipant (==1.17.99)", "mypy-boto3-cur (==1.17.99)", "mypy-boto3-customer-profiles (==1.17.99)", "mypy-boto3-databrew (==1.17.99)", "mypy-boto3-dataexchange (==1.17.99)", "mypy-boto3-datapipeline (==1.17.99)", "mypy-boto3-datasync (==1.17.99)", "mypy-boto3-dax (==1.17.99)", "mypy-boto3-detective (==1.17.99)", "mypy-boto3-devicefarm (==1.17.99)", "mypy-boto3-devops-guru (==1.17.99)", "mypy-boto3-directconnect (==1.17.99)", "mypy-boto3-discovery (==1.17.99)", "mypy-boto3-dlm (==1.17.99)", "mypy-boto3-dms (==1.17.99)", "mypy-boto3-docdb (==1.17.99)", "mypy-boto3-ds (==1.17.99)", "mypy-boto3-dynamodb (==1.17.99)", "mypy-boto3-dynamodbstreams (==1.17.99)", "mypy-boto3-ebs (==1.17.99)", "mypy-boto3-ec2 (==1.17.99)", "mypy-boto3-ec2-instance-connect (==1.17.99)", "mypy-boto3-ecr (==1.17.99)", "mypy-boto3-ecr-public (==1.17.99)", "mypy-boto3-ecs (==1.17.99)", "mypy-boto3-efs (==1.17.99)", "mypy-boto3-eks (==1.17.99)", "mypy-boto3-elastic-inference (==1.17.99)", "mypy-boto3-elasticache (==1.17.99)", "mypy-boto3-elasticbeanstalk (==1.17.99)", "mypy-boto3-elastictranscoder (==1.17.99)", "mypy-boto3-elb (==1.17.99)", "mypy-boto3-elbv2 (==1.17.99)", "mypy-boto3-emr (==1.17.99)", "mypy-boto3-emr-containers (==1.17.99)", "mypy-boto3-es (==1.17.99)", "mypy-boto3-events (==1.17.99)", "mypy-boto3-finspace (==1.17.99)", "mypy-boto3-finspace-data (==1.17.99)", "mypy-boto3-firehose (==1.17.99)", "mypy-boto3-fis (==1.17.99)", "mypy-boto3-fms (==1.17.99)", "mypy-boto3-forecast (==1.17.99)", "mypy-boto3-forecastquery (==1.17.99)", "mypy-boto3-frauddetector (==1.17.99)", "mypy-boto3-fsx (==1.17.99)", "mypy-boto3-gamelift (==1.17.99)", "mypy-boto3-glacier (==1.17.99)", "mypy-boto3-globalaccelerator (==1.17.99)", "mypy-boto3-glue (==1.17.99)", "mypy-boto3-greengrass (==1.17.99)", "mypy-boto3-greengrassv2 (==1.17.99)", "mypy-boto3-groundstation (==1.17.99)", "mypy-boto3-guardduty (==1.17.99)", "mypy-boto3-health (==1.17.99)", "mypy-boto3-healthlake (==1.17.99)", "mypy-boto3-honeycode (==1.17.99)", "mypy-boto3-iam (==1.17.99)", "mypy-boto3-identitystore (==1.17.99)", "mypy-boto3-imagebuilder (==1.17.99)", "mypy-boto3-importexport (==1.17.99)", "mypy-boto3-inspector (==1.17.99)", "mypy-boto3-iot (==1.17.99)", "mypy-boto3-iot-data (==1.17.99)", "mypy-boto3-iot-jobs-data (==1.17.99)", "mypy-boto3-iot1click-devices (==1.17.99)", "mypy-boto3-iot1click-projects (==1.17.99)", "mypy-boto3-iotanalytics (==1.17.99)", "mypy-boto3-iotdeviceadvisor (==1.17.99)", "mypy-boto3-iotevents (==1.17.99)", "mypy-boto3-iotevents-data (==1.17.99)", "mypy-boto3-iotfleethub (==1.17.99)", "mypy-boto3-iotsecuretunneling (==1.17.99)", "mypy-boto3-iotsitewise (==1.17.99)", "mypy-boto3-iotthingsgraph (==1.17.99)", "mypy-boto3-iotwireless (==1.17.99)", "mypy-boto3-ivs (==1.17.99)", "mypy-boto3-kafka (==1.17.99)", "mypy-boto3-kendra (==1.17.99)", "mypy-boto3-kinesis (==1.17.99)", "mypy-boto3-kinesis-video-archived-media (==1.17.99)", "mypy-boto3-kinesis-video-media (==1.17.99)", "mypy-boto3-kinesis-video-signaling (==1.17.99)", "mypy-boto3-kinesisanalytics (==1.17.99)", "mypy-boto3-kinesisanalyticsv2 (==1.17.99)", "mypy-boto3-kinesisvideo (==1.17.99)", "mypy-boto3-kms (==1.17.99)", "mypy-boto3-lakeformation (==1.17.99)", "mypy-boto3-lambda (==1.17.99)", "mypy-boto3-lex-models (==1.17.99)", "mypy-boto3-lex-runtime (==1.17.99)", "mypy-boto3-lexv2-models (==1.17.99)", "mypy-boto3-lexv2-runtime (==1.17.99)", "mypy-boto3-license-manager (==1.17.99)", "mypy-boto3-lightsail (==1.17.99)", "mypy-boto3-location (==1.17.99)", "mypy-boto3-logs (==1.17.99)", "mypy-boto3-lookoutequipment (==1.17.99)", "mypy-boto3-lookoutmetrics (==1.17.99)", "mypy-boto3-lookoutvision (==1.17.99)", "mypy-boto3-machinelearning (==1.17.99)", "mypy-boto3-macie (==1.17.99)", "mypy-boto3-macie2 (==1.17.99)", "mypy-boto3-managedblockchain (==1.17.99)", "mypy-boto3-marketplace-catalog (==1.17.99)", "mypy-boto3-marketplace-entitlement (==1.17.99)", "mypy-boto3-marketplacecommerceanalytics (==1.17.99)", "mypy-boto3-mediaconnect (==1.17.99)", "mypy-boto3-mediaconvert (==1.17.99)", "mypy-boto3-medialive (==1.17.99)", "mypy-boto3-mediapackage (==1.17.99)", "mypy-boto3-mediapackage-vod (==1.17.99)", "mypy-boto3-mediastore (==1.17.99)", "mypy-boto3-mediastore-data (==1.17.99)", "mypy-boto3-mediatailor (==1.17.99)", "mypy-boto3-meteringmarketplace (==1.17.99)", "mypy-boto3-mgh (==1.17.99)", "mypy-boto3-mgn (==1.17.99)", "mypy-boto3-migrationhub-config (==1.17.99)", "mypy-boto3-mobile (==1.17.99)", "mypy-boto3-mq (==1.17.99)", "mypy-boto3-mturk (==1.17.99)", "mypy-boto3-mwaa (==1.17.99)", "mypy-boto3-neptune (==1.17.99)", "mypy-boto3-network-firewall (==1.17.99)", "mypy-boto3-networkmanager (==1.17.99)", "mypy-boto3-nimble (==1.17.99)", "mypy-boto3-opsworks (==1.17.99)", "mypy-boto3-opsworkscm (==1.17.99)", "mypy-boto3-organizations (==1.17.99)", "mypy-boto3-outposts (==1.17.99)", "mypy-boto3-personalize (==1.17.99)", "mypy-boto3-personalize-events (==1.17.99)", "mypy-boto3-personalize-runtime (==1.17.99)", "mypy-boto3-pi (==1.17.99)", "mypy-boto3-pinpoint (==1.17.99)", "mypy-boto3-pinpoint-email (==1.17.99)", "mypy-boto3-pinpoint-sms-voice (==1.17.99)", "mypy-boto3-polly (==1.17.99)", "mypy-boto3-pricing (==1.17.99)", "mypy-boto3-proton (==1.17.99)", "mypy-boto3-qldb (==1.17.99)", "mypy-boto3-qldb-session (==1.17.99)", "mypy-boto3-quicksight (==1.17.99)", "mypy-boto3-ram (==1.17.99)", "mypy-boto3-rds (==1.17.99)", "mypy-boto3-rds-data (==1.17.99)", "mypy-boto3-redshift (==1.17.99)", "mypy-boto3-redshift-data (==1.17.99)", "mypy-boto3-rekognition (==1.17.99)", "mypy-boto3-resource-groups (==1.17.99)", "mypy-boto3-resourcegroupstaggingapi (==1.17.99)", "mypy-boto3-robomaker (==1.17.99)", "mypy-boto3-route53 (==1.17.99)", "mypy-boto3-route53domains (==1.17.99)", "mypy-boto3-route53resolver (==1.17.99)", "mypy-boto3-s3 (==1.17.99)", "mypy-boto3-s3control (==1.17.99)", "mypy-boto3-s3outposts (==1.17.99)", "mypy-boto3-sagemaker (==1.17.99)", "mypy-boto3-sagemaker-a2i-runtime (==1.17.99)", "mypy-boto3-sagemaker-edge (==1.17.99)", "mypy-boto3-sagemaker-featurestore-runtime (==1.17.99)", "mypy-boto3-sagemaker-runtime (==1.17.99)", "mypy-boto3-savingsplans (==1.17.99)", "mypy-boto3-schemas (==1.17.99)", "mypy-boto3-sdb (==1.17.99)", "mypy-boto3-secretsmanager (==1.17.99)", "mypy-boto3-securityhub (==1.17.99)", "mypy-boto3-serverlessrepo (==1.17.99)", "mypy-boto3-service-quotas (==1.17.99)", "mypy-boto3-servicecatalog (==1.17.99)", "mypy-boto3-servicecatalog-appregistry (==1.17.99)", "mypy-boto3-servicediscovery (==1.17.99)", "mypy-boto3-ses (==1.17.99)", "mypy-boto3-sesv2 (==1.17.99)", "mypy-boto3-shield (==1.17.99)", "mypy-boto3-signer (==1.17.99)", "mypy-boto3-sms (==1.17.99)", "mypy-boto3-sms-voice (==1.17.99)", "mypy-boto3-snowball (==1.17.99)", "mypy-boto3-sns (==1.17.99)", "mypy-boto3-sqs (==1.17.99)", "mypy-boto3-ssm (==1.17.99)", "mypy-boto3-ssm-contacts (==1.17.99)", "mypy-boto3-ssm-incidents (==1.17.99)", "mypy-boto3-sso (==1.17.99)", "mypy-boto3-sso-admin (==1.17.99)", "mypy-boto3-sso-oidc (==1.17.99)", "mypy-boto3-stepfunctions (==1.17.99)", "mypy-boto3-storagegateway (==1.17.99)", "mypy-boto3-sts (==1.17.99)", "mypy-boto3-support (==1.17.99)", "mypy-boto3-swf (==1.17.99)", "mypy-boto3-synthetics (==1.17.99)", "mypy-boto3-textract (==1.17.99)", "mypy-boto3-timestream-query (==1.17.99)", "mypy-boto3-timestream-write (==1.17.99)", "mypy-boto3-transcribe (==1.17.99)", "mypy-boto3-transfer (==1.17.99)", "mypy-boto3-translate (==1.17.99)", "mypy-boto3-waf (==1.17.99)", "mypy-boto3-waf-regional (==1.17.99)", "mypy-boto3-wafv2 (==1.17.99)", "mypy-boto3-wellarchitected (==1.17.99)", "mypy-boto3-workdocs (==1.17.99)", "mypy-boto3-worklink (==1.17.99)", "mypy-boto3-workmail (==1.17.99)", "mypy-boto3-workmailmessageflow (==1.17.99)", "mypy-boto3-workspaces (==1.17.99)", "mypy-boto3-xray (==1.17.99)"] +amp = ["mypy-boto3-amp (==1.17.99)"] +amplify = ["mypy-boto3-amplify (==1.17.99)"] +amplifybackend = ["mypy-boto3-amplifybackend (==1.17.99)"] +apigateway = ["mypy-boto3-apigateway (==1.17.99)"] +apigatewaymanagementapi = ["mypy-boto3-apigatewaymanagementapi (==1.17.99)"] +apigatewayv2 = ["mypy-boto3-apigatewayv2 (==1.17.99)"] +appconfig = ["mypy-boto3-appconfig (==1.17.99)"] +appflow = ["mypy-boto3-appflow (==1.17.99)"] +appintegrations = ["mypy-boto3-appintegrations (==1.17.99)"] +application-autoscaling = ["mypy-boto3-application-autoscaling (==1.17.99)"] +application-insights = ["mypy-boto3-application-insights (==1.17.99)"] +applicationcostprofiler = ["mypy-boto3-applicationcostprofiler (==1.17.99)"] +appmesh = ["mypy-boto3-appmesh (==1.17.99)"] +apprunner = ["mypy-boto3-apprunner (==1.17.99)"] +appstream = ["mypy-boto3-appstream (==1.17.99)"] +appsync = ["mypy-boto3-appsync (==1.17.99)"] +athena = ["mypy-boto3-athena (==1.17.99)"] +auditmanager = ["mypy-boto3-auditmanager (==1.17.99)"] +autoscaling = ["mypy-boto3-autoscaling (==1.17.99)"] +autoscaling-plans = ["mypy-boto3-autoscaling-plans (==1.17.99)"] +backup = ["mypy-boto3-backup (==1.17.99)"] +batch = ["mypy-boto3-batch (==1.17.99)"] +braket = ["mypy-boto3-braket (==1.17.99)"] +budgets = ["mypy-boto3-budgets (==1.17.99)"] +ce = ["mypy-boto3-ce (==1.17.99)"] +chime = ["mypy-boto3-chime (==1.17.99)"] +cloud9 = ["mypy-boto3-cloud9 (==1.17.99)"] +clouddirectory = ["mypy-boto3-clouddirectory (==1.17.99)"] +cloudformation = ["mypy-boto3-cloudformation (==1.17.99)"] +cloudfront = ["mypy-boto3-cloudfront (==1.17.99)"] +cloudhsm = ["mypy-boto3-cloudhsm (==1.17.99)"] +cloudhsmv2 = ["mypy-boto3-cloudhsmv2 (==1.17.99)"] +cloudsearch = ["mypy-boto3-cloudsearch (==1.17.99)"] +cloudsearchdomain = ["mypy-boto3-cloudsearchdomain (==1.17.99)"] +cloudtrail = ["mypy-boto3-cloudtrail (==1.17.99)"] +cloudwatch = ["mypy-boto3-cloudwatch (==1.17.99)"] +codeartifact = ["mypy-boto3-codeartifact (==1.17.99)"] +codebuild = ["mypy-boto3-codebuild (==1.17.99)"] +codecommit = ["mypy-boto3-codecommit (==1.17.99)"] +codedeploy = ["mypy-boto3-codedeploy (==1.17.99)"] +codeguru-reviewer = ["mypy-boto3-codeguru-reviewer (==1.17.99)"] +codeguruprofiler = ["mypy-boto3-codeguruprofiler (==1.17.99)"] +codepipeline = ["mypy-boto3-codepipeline (==1.17.99)"] +codestar = ["mypy-boto3-codestar (==1.17.99)"] +codestar-connections = ["mypy-boto3-codestar-connections (==1.17.99)"] +codestar-notifications = ["mypy-boto3-codestar-notifications (==1.17.99)"] +cognito-identity = ["mypy-boto3-cognito-identity (==1.17.99)"] +cognito-idp = ["mypy-boto3-cognito-idp (==1.17.99)"] +cognito-sync = ["mypy-boto3-cognito-sync (==1.17.99)"] +comprehend = ["mypy-boto3-comprehend (==1.17.99)"] +comprehendmedical = ["mypy-boto3-comprehendmedical (==1.17.99)"] +compute-optimizer = ["mypy-boto3-compute-optimizer (==1.17.99)"] +config = ["mypy-boto3-config (==1.17.99)"] +connect = ["mypy-boto3-connect (==1.17.99)"] +connect-contact-lens = ["mypy-boto3-connect-contact-lens (==1.17.99)"] +connectparticipant = ["mypy-boto3-connectparticipant (==1.17.99)"] +cur = ["mypy-boto3-cur (==1.17.99)"] +customer-profiles = ["mypy-boto3-customer-profiles (==1.17.99)"] +databrew = ["mypy-boto3-databrew (==1.17.99)"] +dataexchange = ["mypy-boto3-dataexchange (==1.17.99)"] +datapipeline = ["mypy-boto3-datapipeline (==1.17.99)"] +datasync = ["mypy-boto3-datasync (==1.17.99)"] +dax = ["mypy-boto3-dax (==1.17.99)"] +detective = ["mypy-boto3-detective (==1.17.99)"] +devicefarm = ["mypy-boto3-devicefarm (==1.17.99)"] +devops-guru = ["mypy-boto3-devops-guru (==1.17.99)"] +directconnect = ["mypy-boto3-directconnect (==1.17.99)"] +discovery = ["mypy-boto3-discovery (==1.17.99)"] +dlm = ["mypy-boto3-dlm (==1.17.99)"] +dms = ["mypy-boto3-dms (==1.17.99)"] +docdb = ["mypy-boto3-docdb (==1.17.99)"] +ds = ["mypy-boto3-ds (==1.17.99)"] +dynamodb = ["mypy-boto3-dynamodb (==1.17.99)"] +dynamodbstreams = ["mypy-boto3-dynamodbstreams (==1.17.99)"] +ebs = ["mypy-boto3-ebs (==1.17.99)"] +ec2 = ["mypy-boto3-ec2 (==1.17.99)"] +ec2-instance-connect = ["mypy-boto3-ec2-instance-connect (==1.17.99)"] +ecr = ["mypy-boto3-ecr (==1.17.99)"] +ecr-public = ["mypy-boto3-ecr-public (==1.17.99)"] +ecs = ["mypy-boto3-ecs (==1.17.99)"] +efs = ["mypy-boto3-efs (==1.17.99)"] +eks = ["mypy-boto3-eks (==1.17.99)"] +elastic-inference = ["mypy-boto3-elastic-inference (==1.17.99)"] +elasticache = ["mypy-boto3-elasticache (==1.17.99)"] +elasticbeanstalk = ["mypy-boto3-elasticbeanstalk (==1.17.99)"] +elastictranscoder = ["mypy-boto3-elastictranscoder (==1.17.99)"] +elb = ["mypy-boto3-elb (==1.17.99)"] +elbv2 = ["mypy-boto3-elbv2 (==1.17.99)"] +emr = ["mypy-boto3-emr (==1.17.99)"] +emr-containers = ["mypy-boto3-emr-containers (==1.17.99)"] +es = ["mypy-boto3-es (==1.17.99)"] +essential = ["mypy-boto3-cloudformation (==1.17.99)", "mypy-boto3-dynamodb (==1.17.99)", "mypy-boto3-ec2 (==1.17.99)", "mypy-boto3-lambda (==1.17.99)", "mypy-boto3-rds (==1.17.99)", "mypy-boto3-s3 (==1.17.99)", "mypy-boto3-sqs (==1.17.99)"] +events = ["mypy-boto3-events (==1.17.99)"] +finspace = ["mypy-boto3-finspace (==1.17.99)"] +finspace-data = ["mypy-boto3-finspace-data (==1.17.99)"] +firehose = ["mypy-boto3-firehose (==1.17.99)"] +fis = ["mypy-boto3-fis (==1.17.99)"] +fms = ["mypy-boto3-fms (==1.17.99)"] +forecast = ["mypy-boto3-forecast (==1.17.99)"] +forecastquery = ["mypy-boto3-forecastquery (==1.17.99)"] +frauddetector = ["mypy-boto3-frauddetector (==1.17.99)"] +fsx = ["mypy-boto3-fsx (==1.17.99)"] +gamelift = ["mypy-boto3-gamelift (==1.17.99)"] +glacier = ["mypy-boto3-glacier (==1.17.99)"] +globalaccelerator = ["mypy-boto3-globalaccelerator (==1.17.99)"] +glue = ["mypy-boto3-glue (==1.17.99)"] +greengrass = ["mypy-boto3-greengrass (==1.17.99)"] +greengrassv2 = ["mypy-boto3-greengrassv2 (==1.17.99)"] +groundstation = ["mypy-boto3-groundstation (==1.17.99)"] +guardduty = ["mypy-boto3-guardduty (==1.17.99)"] +health = ["mypy-boto3-health (==1.17.99)"] +healthlake = ["mypy-boto3-healthlake (==1.17.99)"] +honeycode = ["mypy-boto3-honeycode (==1.17.99)"] +iam = ["mypy-boto3-iam (==1.17.99)"] +identitystore = ["mypy-boto3-identitystore (==1.17.99)"] +imagebuilder = ["mypy-boto3-imagebuilder (==1.17.99)"] +importexport = ["mypy-boto3-importexport (==1.17.99)"] +inspector = ["mypy-boto3-inspector (==1.17.99)"] +iot = ["mypy-boto3-iot (==1.17.99)"] +iot-data = ["mypy-boto3-iot-data (==1.17.99)"] +iot-jobs-data = ["mypy-boto3-iot-jobs-data (==1.17.99)"] +iot1click-devices = ["mypy-boto3-iot1click-devices (==1.17.99)"] +iot1click-projects = ["mypy-boto3-iot1click-projects (==1.17.99)"] +iotanalytics = ["mypy-boto3-iotanalytics (==1.17.99)"] +iotdeviceadvisor = ["mypy-boto3-iotdeviceadvisor (==1.17.99)"] +iotevents = ["mypy-boto3-iotevents (==1.17.99)"] +iotevents-data = ["mypy-boto3-iotevents-data (==1.17.99)"] +iotfleethub = ["mypy-boto3-iotfleethub (==1.17.99)"] +iotsecuretunneling = ["mypy-boto3-iotsecuretunneling (==1.17.99)"] +iotsitewise = ["mypy-boto3-iotsitewise (==1.17.99)"] +iotthingsgraph = ["mypy-boto3-iotthingsgraph (==1.17.99)"] +iotwireless = ["mypy-boto3-iotwireless (==1.17.99)"] +ivs = ["mypy-boto3-ivs (==1.17.99)"] +kafka = ["mypy-boto3-kafka (==1.17.99)"] +kendra = ["mypy-boto3-kendra (==1.17.99)"] +kinesis = ["mypy-boto3-kinesis (==1.17.99)"] +kinesis-video-archived-media = ["mypy-boto3-kinesis-video-archived-media (==1.17.99)"] +kinesis-video-media = ["mypy-boto3-kinesis-video-media (==1.17.99)"] +kinesis-video-signaling = ["mypy-boto3-kinesis-video-signaling (==1.17.99)"] +kinesisanalytics = ["mypy-boto3-kinesisanalytics (==1.17.99)"] +kinesisanalyticsv2 = ["mypy-boto3-kinesisanalyticsv2 (==1.17.99)"] +kinesisvideo = ["mypy-boto3-kinesisvideo (==1.17.99)"] +kms = ["mypy-boto3-kms (==1.17.99)"] +lakeformation = ["mypy-boto3-lakeformation (==1.17.99)"] +lambda = ["mypy-boto3-lambda (==1.17.99)"] +lex-models = ["mypy-boto3-lex-models (==1.17.99)"] +lex-runtime = ["mypy-boto3-lex-runtime (==1.17.99)"] +lexv2-models = ["mypy-boto3-lexv2-models (==1.17.99)"] +lexv2-runtime = ["mypy-boto3-lexv2-runtime (==1.17.99)"] +license-manager = ["mypy-boto3-license-manager (==1.17.99)"] +lightsail = ["mypy-boto3-lightsail (==1.17.99)"] +location = ["mypy-boto3-location (==1.17.99)"] +logs = ["mypy-boto3-logs (==1.17.99)"] +lookoutequipment = ["mypy-boto3-lookoutequipment (==1.17.99)"] +lookoutmetrics = ["mypy-boto3-lookoutmetrics (==1.17.99)"] +lookoutvision = ["mypy-boto3-lookoutvision (==1.17.99)"] +machinelearning = ["mypy-boto3-machinelearning (==1.17.99)"] +macie = ["mypy-boto3-macie (==1.17.99)"] +macie2 = ["mypy-boto3-macie2 (==1.17.99)"] +managedblockchain = ["mypy-boto3-managedblockchain (==1.17.99)"] +marketplace-catalog = ["mypy-boto3-marketplace-catalog (==1.17.99)"] +marketplace-entitlement = ["mypy-boto3-marketplace-entitlement (==1.17.99)"] +marketplacecommerceanalytics = ["mypy-boto3-marketplacecommerceanalytics (==1.17.99)"] +mediaconnect = ["mypy-boto3-mediaconnect (==1.17.99)"] +mediaconvert = ["mypy-boto3-mediaconvert (==1.17.99)"] +medialive = ["mypy-boto3-medialive (==1.17.99)"] +mediapackage = ["mypy-boto3-mediapackage (==1.17.99)"] +mediapackage-vod = ["mypy-boto3-mediapackage-vod (==1.17.99)"] +mediastore = ["mypy-boto3-mediastore (==1.17.99)"] +mediastore-data = ["mypy-boto3-mediastore-data (==1.17.99)"] +mediatailor = ["mypy-boto3-mediatailor (==1.17.99)"] +meteringmarketplace = ["mypy-boto3-meteringmarketplace (==1.17.99)"] +mgh = ["mypy-boto3-mgh (==1.17.99)"] +mgn = ["mypy-boto3-mgn (==1.17.99)"] +migrationhub-config = ["mypy-boto3-migrationhub-config (==1.17.99)"] +mobile = ["mypy-boto3-mobile (==1.17.99)"] +mq = ["mypy-boto3-mq (==1.17.99)"] +mturk = ["mypy-boto3-mturk (==1.17.99)"] +mwaa = ["mypy-boto3-mwaa (==1.17.99)"] +neptune = ["mypy-boto3-neptune (==1.17.99)"] +network-firewall = ["mypy-boto3-network-firewall (==1.17.99)"] +networkmanager = ["mypy-boto3-networkmanager (==1.17.99)"] +nimble = ["mypy-boto3-nimble (==1.17.99)"] +opsworks = ["mypy-boto3-opsworks (==1.17.99)"] +opsworkscm = ["mypy-boto3-opsworkscm (==1.17.99)"] +organizations = ["mypy-boto3-organizations (==1.17.99)"] +outposts = ["mypy-boto3-outposts (==1.17.99)"] +personalize = ["mypy-boto3-personalize (==1.17.99)"] +personalize-events = ["mypy-boto3-personalize-events (==1.17.99)"] +personalize-runtime = ["mypy-boto3-personalize-runtime (==1.17.99)"] +pi = ["mypy-boto3-pi (==1.17.99)"] +pinpoint = ["mypy-boto3-pinpoint (==1.17.99)"] +pinpoint-email = ["mypy-boto3-pinpoint-email (==1.17.99)"] +pinpoint-sms-voice = ["mypy-boto3-pinpoint-sms-voice (==1.17.99)"] +polly = ["mypy-boto3-polly (==1.17.99)"] +pricing = ["mypy-boto3-pricing (==1.17.99)"] +proton = ["mypy-boto3-proton (==1.17.99)"] +qldb = ["mypy-boto3-qldb (==1.17.99)"] +qldb-session = ["mypy-boto3-qldb-session (==1.17.99)"] +quicksight = ["mypy-boto3-quicksight (==1.17.99)"] +ram = ["mypy-boto3-ram (==1.17.99)"] +rds = ["mypy-boto3-rds (==1.17.99)"] +rds-data = ["mypy-boto3-rds-data (==1.17.99)"] +redshift = ["mypy-boto3-redshift (==1.17.99)"] +redshift-data = ["mypy-boto3-redshift-data (==1.17.99)"] +rekognition = ["mypy-boto3-rekognition (==1.17.99)"] +resource-groups = ["mypy-boto3-resource-groups (==1.17.99)"] +resourcegroupstaggingapi = ["mypy-boto3-resourcegroupstaggingapi (==1.17.99)"] +robomaker = ["mypy-boto3-robomaker (==1.17.99)"] +route53 = ["mypy-boto3-route53 (==1.17.99)"] +route53domains = ["mypy-boto3-route53domains (==1.17.99)"] +route53resolver = ["mypy-boto3-route53resolver (==1.17.99)"] +s3 = ["mypy-boto3-s3 (==1.17.99)"] +s3control = ["mypy-boto3-s3control (==1.17.99)"] +s3outposts = ["mypy-boto3-s3outposts (==1.17.99)"] +sagemaker = ["mypy-boto3-sagemaker (==1.17.99)"] +sagemaker-a2i-runtime = ["mypy-boto3-sagemaker-a2i-runtime (==1.17.99)"] +sagemaker-edge = ["mypy-boto3-sagemaker-edge (==1.17.99)"] +sagemaker-featurestore-runtime = ["mypy-boto3-sagemaker-featurestore-runtime (==1.17.99)"] +sagemaker-runtime = ["mypy-boto3-sagemaker-runtime (==1.17.99)"] +savingsplans = ["mypy-boto3-savingsplans (==1.17.99)"] +schemas = ["mypy-boto3-schemas (==1.17.99)"] +sdb = ["mypy-boto3-sdb (==1.17.99)"] +secretsmanager = ["mypy-boto3-secretsmanager (==1.17.99)"] +securityhub = ["mypy-boto3-securityhub (==1.17.99)"] +serverlessrepo = ["mypy-boto3-serverlessrepo (==1.17.99)"] +service-quotas = ["mypy-boto3-service-quotas (==1.17.99)"] +servicecatalog = ["mypy-boto3-servicecatalog (==1.17.99)"] +servicecatalog-appregistry = ["mypy-boto3-servicecatalog-appregistry (==1.17.99)"] +servicediscovery = ["mypy-boto3-servicediscovery (==1.17.99)"] +ses = ["mypy-boto3-ses (==1.17.99)"] +sesv2 = ["mypy-boto3-sesv2 (==1.17.99)"] +shield = ["mypy-boto3-shield (==1.17.99)"] +signer = ["mypy-boto3-signer (==1.17.99)"] +sms = ["mypy-boto3-sms (==1.17.99)"] +sms-voice = ["mypy-boto3-sms-voice (==1.17.99)"] +snowball = ["mypy-boto3-snowball (==1.17.99)"] +sns = ["mypy-boto3-sns (==1.17.99)"] +sqs = ["mypy-boto3-sqs (==1.17.99)"] +ssm = ["mypy-boto3-ssm (==1.17.99)"] +ssm-contacts = ["mypy-boto3-ssm-contacts (==1.17.99)"] +ssm-incidents = ["mypy-boto3-ssm-incidents (==1.17.99)"] +sso = ["mypy-boto3-sso (==1.17.99)"] +sso-admin = ["mypy-boto3-sso-admin (==1.17.99)"] +sso-oidc = ["mypy-boto3-sso-oidc (==1.17.99)"] +stepfunctions = ["mypy-boto3-stepfunctions (==1.17.99)"] +storagegateway = ["mypy-boto3-storagegateway (==1.17.99)"] +sts = ["mypy-boto3-sts (==1.17.99)"] +support = ["mypy-boto3-support (==1.17.99)"] +swf = ["mypy-boto3-swf (==1.17.99)"] +synthetics = ["mypy-boto3-synthetics (==1.17.99)"] +textract = ["mypy-boto3-textract (==1.17.99)"] +timestream-query = ["mypy-boto3-timestream-query (==1.17.99)"] +timestream-write = ["mypy-boto3-timestream-write (==1.17.99)"] +transcribe = ["mypy-boto3-transcribe (==1.17.99)"] +transfer = ["mypy-boto3-transfer (==1.17.99)"] +translate = ["mypy-boto3-translate (==1.17.99)"] +waf = ["mypy-boto3-waf (==1.17.99)"] +waf-regional = ["mypy-boto3-waf-regional (==1.17.99)"] +wafv2 = ["mypy-boto3-wafv2 (==1.17.99)"] +wellarchitected = ["mypy-boto3-wellarchitected (==1.17.99)"] +workdocs = ["mypy-boto3-workdocs (==1.17.99)"] +worklink = ["mypy-boto3-worklink (==1.17.99)"] +workmail = ["mypy-boto3-workmail (==1.17.99)"] +workmailmessageflow = ["mypy-boto3-workmailmessageflow (==1.17.99)"] +workspaces = ["mypy-boto3-workspaces (==1.17.99)"] +xray = ["mypy-boto3-xray (==1.17.99)"] [[package]] name = "botocore" -version = "1.20.89" +version = "1.20.99" description = "Low-level, data-driven core of boto 3." category = "main" optional = false @@ -402,7 +404,18 @@ python-dateutil = ">=2.1,<3.0.0" urllib3 = ">=1.25.4,<1.27" [package.extras] -crt = ["awscrt (==0.11.15)"] +crt = ["awscrt (==0.11.24)"] + +[[package]] +name = "botocore-stubs" +version = "1.20.99" +description = "Type annotations for botocore 1.20.99, generated by mypy-boto3-buider 4.18.4" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +typing-extensions = {version = "*", markers = "python_version < \"3.8\""} [[package]] name = "cached-property" @@ -592,7 +605,7 @@ xlrd = ">=1.0.0,<2.0.0" [[package]] name = "dcicutils" -version = "1.18.0" +version = "1.18.1.2b2" description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources" category = "main" optional = false @@ -774,11 +787,11 @@ smmap = ">=3.0.1,<5" [[package]] name = "gitpython" -version = "3.1.17" +version = "3.1.18" description = "Python Git Library" category = "main" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" [package.dependencies] gitdb = ">=4.0.1,<5" @@ -1218,19 +1231,19 @@ test = ["ipaddress", "mock", "unittest2", "enum34", "pywin32", "wmi"] [[package]] name = "psycopg2" -version = "2.8.6" +version = "2.9.1" description = "psycopg2 - Python-PostgreSQL Database Adapter" category = "main" optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" +python-versions = ">=3.6" [[package]] name = "psycopg2-binary" -version = "2.8.6" +version = "2.9.1" description = "psycopg2 - Python-PostgreSQL Database Adapter" category = "main" optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" +python-versions = ">=3.6" [[package]] name = "ptable" @@ -1854,7 +1867,7 @@ python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "tqdm" -version = "4.61.0" +version = "4.61.1" description = "Fast, Extensible Progress Meter" category = "main" optional = false @@ -1967,7 +1980,7 @@ testing = ["pytest (>=3.1.0)", "coverage", "pytest-cov", "pytest-xdist"] [[package]] name = "websocket-client" -version = "1.0.1" +version = "1.1.0" description = "WebSocket client for Python with low level API options" category = "main" optional = false @@ -2095,7 +2108,7 @@ test = ["zope.testing"] [metadata] lock-version = "1.1" python-versions = ">=3.6.1,<3.7" -content-hash = "d842fa6e10cd09c99c0f855bfa538ddb50e451092ba1a6330929c8133a544c89" +content-hash = "8eccb570d4ce85e64a03653a48c06b436fb4aab207da54f7304ae399bdfe533d" [metadata.files] apipkg = [ @@ -2128,7 +2141,7 @@ beautifulsoup4 = [ {file = "beautifulsoup4-4.9.3.tar.gz", hash = "sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25"}, ] bitarray = [ - {file = "bitarray-2.1.0.tar.gz", hash = "sha256:97224a19325ecee49a3bf4df3ee0531d3af9cf288b67d089a7ef44a3c4ea3839"}, + {file = "bitarray-2.1.3.tar.gz", hash = "sha256:a24aff72a7f1b09571b5daf9dbfcffd98481be1fe085ae5ef662cf11452a97e0"}, ] boto = [ {file = "boto-2.49.0-py2.py3-none-any.whl", hash = "sha256:147758d41ae7240dc989f0039f27da8ca0d53734be0eb869ef16e3adcfa462e8"}, @@ -2139,12 +2152,16 @@ boto3 = [ {file = "boto3-1.17.53.tar.gz", hash = "sha256:1d26f6e7ae3c940cb07119077ac42485dcf99164350da0ab50d0f5ad345800cd"}, ] boto3-stubs = [ - {file = "boto3-stubs-1.17.89.tar.gz", hash = "sha256:e65cf658c94cc3270d14039fda50f41599c9bd18d426cd0b81af6edfd166a4a6"}, - {file = "boto3_stubs-1.17.89-py3-none-any.whl", hash = "sha256:444aa2a680fb13bb3f2ee0ed09716f1b8bff495f5d311de3298ef30cd939db0f"}, + {file = "boto3-stubs-1.17.99.tar.gz", hash = "sha256:17f32726e1f1b318b3a9ba6683a4dbd5873b6e143eea632c66042d7586b5911f"}, + {file = "boto3_stubs-1.17.99-py3-none-any.whl", hash = "sha256:f69706063ad49fade04d63d4e2e8b1b750fb1066de2281a7b994cd0003a5680c"}, ] botocore = [ - {file = "botocore-1.20.89-py2.py3-none-any.whl", hash = "sha256:e112f9a45db1c5a42f787e4b228a35da6e823bcba70f43f43005b4fb58066446"}, - {file = "botocore-1.20.89.tar.gz", hash = "sha256:ce0fa8bc260ad187824052805d224cee239d953bb4bfb1e52cf35ad79481b316"}, + {file = "botocore-1.20.99-py2.py3-none-any.whl", hash = "sha256:a236bb890e2b25f0db1b9bb4dd49e2d825b051ba953830c7cd7be7200f5aecbf"}, + {file = "botocore-1.20.99.tar.gz", hash = "sha256:683c7cc7d01c94a6e593694d1d7bcdd3ea5f59c00421fa7e34500458175b9346"}, +] +botocore-stubs = [ + {file = "botocore-stubs-1.20.99.tar.gz", hash = "sha256:cce3a22449ada550d621c7350636c5b6730033857f250c14f633e52ea5e6e50e"}, + {file = "botocore_stubs-1.20.99-py3-none-any.whl", hash = "sha256:f9a6e1977ab0b6a651cdb956594192fd7d465757825823bba2109b14292223d6"}, ] cached-property = [ {file = "cached-property-1.5.2.tar.gz", hash = "sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130"}, @@ -2304,8 +2321,8 @@ dcicsnovault = [ {file = "dcicsnovault-4.8.0.0b2.tar.gz", hash = "sha256:4dcde76ab8cb82469ca3a2a4eae9476ac04396faf370e124a3114f63378cf48a"}, ] dcicutils = [ - {file = "dcicutils-1.18.0-py3-none-any.whl", hash = "sha256:e41b7ca87687d9ed41f2aa517f5338df9aae79aaae1acc35662d925f77c6fe10"}, - {file = "dcicutils-1.18.0.tar.gz", hash = "sha256:f5113c59b7565d77cd42985de7e15eb1c2475c640fb491b81b0833cfaf46ac1a"}, + {file = "dcicutils-1.18.1.2b2-py3-none-any.whl", hash = "sha256:906c126d5d8a205e8c5ee6fc3915e888f1e0d7ea5f36677366507f3841f18680"}, + {file = "dcicutils-1.18.1.2b2.tar.gz", hash = "sha256:d557c6a121daed98973f9ac43d927bf487edd6c52525fa4a362285e5e8abc964"}, ] docker = [ {file = "docker-4.4.4-py2.py3-none-any.whl", hash = "sha256:f3607d5695be025fa405a12aca2e5df702a57db63790c73b927eb6a94aac60af"}, @@ -2359,8 +2376,8 @@ gitdb = [ {file = "gitdb-4.0.7.tar.gz", hash = "sha256:96bf5c08b157a666fec41129e6d327235284cca4c81e92109260f353ba138005"}, ] gitpython = [ - {file = "GitPython-3.1.17-py3-none-any.whl", hash = "sha256:29fe82050709760081f588dd50ce83504feddbebdc4da6956d02351552b1c135"}, - {file = "GitPython-3.1.17.tar.gz", hash = "sha256:ee24bdc93dce357630764db659edaf6b8d664d4ff5447ccfeedd2dc5c253f41e"}, + {file = "GitPython-3.1.18-py3-none-any.whl", hash = "sha256:fce760879cd2aebd2991b3542876dc5c4a909b30c9d69dfc488e504a8db37ee8"}, + {file = "GitPython-3.1.18.tar.gz", hash = "sha256:b838a895977b45ab6f0cc926a9045c8d1c44e2b653c1fcc39fe91f42c6e8f05b"}, ] granite-suite = [ {file = "granite-suite-0.1.11b0.tar.gz", hash = "sha256:8d5b893ff8658d42cd16e7b2013c2456dad6216aa836fea45767bee07a9aaf97"}, @@ -2655,58 +2672,46 @@ psutil = [ {file = "psutil-5.8.0.tar.gz", hash = "sha256:0c9ccb99ab76025f2f0bbecf341d4656e9c1351db8cc8a03ccd62e318ab4b5c6"}, ] psycopg2 = [ - {file = "psycopg2-2.8.6-cp27-cp27m-win32.whl", hash = "sha256:068115e13c70dc5982dfc00c5d70437fe37c014c808acce119b5448361c03725"}, - {file = "psycopg2-2.8.6-cp27-cp27m-win_amd64.whl", hash = "sha256:d160744652e81c80627a909a0e808f3c6653a40af435744de037e3172cf277f5"}, - {file = "psycopg2-2.8.6-cp34-cp34m-win32.whl", hash = "sha256:b8cae8b2f022efa1f011cc753adb9cbadfa5a184431d09b273fb49b4167561ad"}, - {file = "psycopg2-2.8.6-cp34-cp34m-win_amd64.whl", hash = "sha256:f22ea9b67aea4f4a1718300908a2fb62b3e4276cf00bd829a97ab5894af42ea3"}, - {file = "psycopg2-2.8.6-cp35-cp35m-win32.whl", hash = "sha256:26e7fd115a6db75267b325de0fba089b911a4a12ebd3d0b5e7acb7028bc46821"}, - {file = "psycopg2-2.8.6-cp35-cp35m-win_amd64.whl", hash = "sha256:00195b5f6832dbf2876b8bf77f12bdce648224c89c880719c745b90515233301"}, - {file = "psycopg2-2.8.6-cp36-cp36m-win32.whl", hash = "sha256:a49833abfdede8985ba3f3ec641f771cca215479f41523e99dace96d5b8cce2a"}, - {file = "psycopg2-2.8.6-cp36-cp36m-win_amd64.whl", hash = "sha256:f974c96fca34ae9e4f49839ba6b78addf0346777b46c4da27a7bf54f48d3057d"}, - {file = "psycopg2-2.8.6-cp37-cp37m-win32.whl", hash = "sha256:6a3d9efb6f36f1fe6aa8dbb5af55e067db802502c55a9defa47c5a1dad41df84"}, - {file = "psycopg2-2.8.6-cp37-cp37m-win_amd64.whl", hash = "sha256:56fee7f818d032f802b8eed81ef0c1232b8b42390df189cab9cfa87573fe52c5"}, - {file = "psycopg2-2.8.6-cp38-cp38-win32.whl", hash = "sha256:ad2fe8a37be669082e61fb001c185ffb58867fdbb3e7a6b0b0d2ffe232353a3e"}, - {file = "psycopg2-2.8.6-cp38-cp38-win_amd64.whl", hash = "sha256:56007a226b8e95aa980ada7abdea6b40b75ce62a433bd27cec7a8178d57f4051"}, - {file = "psycopg2-2.8.6-cp39-cp39-win32.whl", hash = "sha256:2c93d4d16933fea5bbacbe1aaf8fa8c1348740b2e50b3735d1b0bf8154cbf0f3"}, - {file = "psycopg2-2.8.6-cp39-cp39-win_amd64.whl", hash = "sha256:d5062ae50b222da28253059880a871dc87e099c25cb68acf613d9d227413d6f7"}, - {file = "psycopg2-2.8.6.tar.gz", hash = "sha256:fb23f6c71107c37fd667cb4ea363ddeb936b348bbd6449278eb92c189699f543"}, + {file = "psycopg2-2.9.1-cp36-cp36m-win32.whl", hash = "sha256:7f91312f065df517187134cce8e395ab37f5b601a42446bdc0f0d51773621854"}, + {file = "psycopg2-2.9.1-cp36-cp36m-win_amd64.whl", hash = "sha256:830c8e8dddab6b6716a4bf73a09910c7954a92f40cf1d1e702fb93c8a919cc56"}, + {file = "psycopg2-2.9.1-cp37-cp37m-win32.whl", hash = "sha256:89409d369f4882c47f7ea20c42c5046879ce22c1e4ea20ef3b00a4dfc0a7f188"}, + {file = "psycopg2-2.9.1-cp37-cp37m-win_amd64.whl", hash = "sha256:7640e1e4d72444ef012e275e7b53204d7fab341fb22bc76057ede22fe6860b25"}, + {file = "psycopg2-2.9.1-cp38-cp38-win32.whl", hash = "sha256:079d97fc22de90da1d370c90583659a9f9a6ee4007355f5825e5f1c70dffc1fa"}, + {file = "psycopg2-2.9.1-cp38-cp38-win_amd64.whl", hash = "sha256:2c992196719fadda59f72d44603ee1a2fdcc67de097eea38d41c7ad9ad246e62"}, + {file = "psycopg2-2.9.1-cp39-cp39-win32.whl", hash = "sha256:2087013c159a73e09713294a44d0c8008204d06326006b7f652bef5ace66eebb"}, + {file = "psycopg2-2.9.1-cp39-cp39-win_amd64.whl", hash = "sha256:bf35a25f1aaa8a3781195595577fcbb59934856ee46b4f252f56ad12b8043bcf"}, + {file = "psycopg2-2.9.1.tar.gz", hash = "sha256:de5303a6f1d0a7a34b9d40e4d3bef684ccc44a49bbe3eb85e3c0bffb4a131b7c"}, ] psycopg2-binary = [ - {file = "psycopg2-binary-2.8.6.tar.gz", hash = "sha256:11b9c0ebce097180129e422379b824ae21c8f2a6596b159c7659e2e5a00e1aa0"}, - {file = "psycopg2_binary-2.8.6-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl", hash = "sha256:d14b140a4439d816e3b1229a4a525df917d6ea22a0771a2a78332273fd9528a4"}, - {file = "psycopg2_binary-2.8.6-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:1fabed9ea2acc4efe4671b92c669a213db744d2af8a9fc5d69a8e9bc14b7a9db"}, - {file = "psycopg2_binary-2.8.6-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:f5ab93a2cb2d8338b1674be43b442a7f544a0971da062a5da774ed40587f18f5"}, - {file = "psycopg2_binary-2.8.6-cp27-cp27m-win32.whl", hash = "sha256:b4afc542c0ac0db720cf516dd20c0846f71c248d2b3d21013aa0d4ef9c71ca25"}, - {file = "psycopg2_binary-2.8.6-cp27-cp27m-win_amd64.whl", hash = "sha256:e74a55f6bad0e7d3968399deb50f61f4db1926acf4a6d83beaaa7df986f48b1c"}, - {file = "psycopg2_binary-2.8.6-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:0deac2af1a587ae12836aa07970f5cb91964f05a7c6cdb69d8425ff4c15d4e2c"}, - {file = "psycopg2_binary-2.8.6-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ad20d2eb875aaa1ea6d0f2916949f5c08a19c74d05b16ce6ebf6d24f2c9f75d1"}, - {file = "psycopg2_binary-2.8.6-cp34-cp34m-win32.whl", hash = "sha256:950bc22bb56ee6ff142a2cb9ee980b571dd0912b0334aa3fe0fe3788d860bea2"}, - {file = "psycopg2_binary-2.8.6-cp34-cp34m-win_amd64.whl", hash = "sha256:b8a3715b3c4e604bcc94c90a825cd7f5635417453b253499664f784fc4da0152"}, - {file = "psycopg2_binary-2.8.6-cp35-cp35m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl", hash = "sha256:d1b4ab59e02d9008efe10ceabd0b31e79519da6fb67f7d8e8977118832d0f449"}, - {file = "psycopg2_binary-2.8.6-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:ac0c682111fbf404525dfc0f18a8b5f11be52657d4f96e9fcb75daf4f3984859"}, - {file = "psycopg2_binary-2.8.6-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:7d92a09b788cbb1aec325af5fcba9fed7203897bbd9269d5691bb1e3bce29550"}, - {file = "psycopg2_binary-2.8.6-cp35-cp35m-win32.whl", hash = "sha256:aaa4213c862f0ef00022751161df35804127b78adf4a2755b9f991a507e425fd"}, - {file = "psycopg2_binary-2.8.6-cp35-cp35m-win_amd64.whl", hash = "sha256:c2507d796fca339c8fb03216364cca68d87e037c1f774977c8fc377627d01c71"}, - {file = "psycopg2_binary-2.8.6-cp36-cp36m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl", hash = "sha256:ee69dad2c7155756ad114c02db06002f4cded41132cc51378e57aad79cc8e4f4"}, - {file = "psycopg2_binary-2.8.6-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:e82aba2188b9ba309fd8e271702bd0d0fc9148ae3150532bbb474f4590039ffb"}, - {file = "psycopg2_binary-2.8.6-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:d5227b229005a696cc67676e24c214740efd90b148de5733419ac9aaba3773da"}, - {file = "psycopg2_binary-2.8.6-cp36-cp36m-win32.whl", hash = "sha256:a0eb43a07386c3f1f1ebb4dc7aafb13f67188eab896e7397aa1ee95a9c884eb2"}, - {file = "psycopg2_binary-2.8.6-cp36-cp36m-win_amd64.whl", hash = "sha256:e1f57aa70d3f7cc6947fd88636a481638263ba04a742b4a37dd25c373e41491a"}, - {file = "psycopg2_binary-2.8.6-cp37-cp37m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl", hash = "sha256:833709a5c66ca52f1d21d41865a637223b368c0ee76ea54ca5bad6f2526c7679"}, - {file = "psycopg2_binary-2.8.6-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:ba28584e6bca48c59eecbf7efb1576ca214b47f05194646b081717fa628dfddf"}, - {file = "psycopg2_binary-2.8.6-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:6a32f3a4cb2f6e1a0b15215f448e8ce2da192fd4ff35084d80d5e39da683e79b"}, - {file = "psycopg2_binary-2.8.6-cp37-cp37m-win32.whl", hash = "sha256:0e4dc3d5996760104746e6cfcdb519d9d2cd27c738296525d5867ea695774e67"}, - {file = "psycopg2_binary-2.8.6-cp37-cp37m-win_amd64.whl", hash = "sha256:cec7e622ebc545dbb4564e483dd20e4e404da17ae07e06f3e780b2dacd5cee66"}, - {file = "psycopg2_binary-2.8.6-cp38-cp38-macosx_10_9_x86_64.macosx_10_9_intel.macosx_10_10_intel.macosx_10_10_x86_64.whl", hash = "sha256:ba381aec3a5dc29634f20692349d73f2d21f17653bda1decf0b52b11d694541f"}, - {file = "psycopg2_binary-2.8.6-cp38-cp38-manylinux1_i686.whl", hash = "sha256:a0c50db33c32594305b0ef9abc0cb7db13de7621d2cadf8392a1d9b3c437ef77"}, - {file = "psycopg2_binary-2.8.6-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:2dac98e85565d5688e8ab7bdea5446674a83a3945a8f416ad0110018d1501b94"}, - {file = "psycopg2_binary-2.8.6-cp38-cp38-win32.whl", hash = "sha256:bd1be66dde2b82f80afb9459fc618216753f67109b859a361cf7def5c7968729"}, - {file = "psycopg2_binary-2.8.6-cp38-cp38-win_amd64.whl", hash = "sha256:8cd0fb36c7412996859cb4606a35969dd01f4ea34d9812a141cd920c3b18be77"}, - {file = "psycopg2_binary-2.8.6-cp39-cp39-macosx_10_9_x86_64.macosx_10_9_intel.macosx_10_10_intel.macosx_10_10_x86_64.whl", hash = "sha256:89705f45ce07b2dfa806ee84439ec67c5d9a0ef20154e0e475e2b2ed392a5b83"}, - {file = "psycopg2_binary-2.8.6-cp39-cp39-manylinux1_i686.whl", hash = "sha256:42ec1035841b389e8cc3692277a0bd81cdfe0b65d575a2c8862cec7a80e62e52"}, - {file = "psycopg2_binary-2.8.6-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:7312e931b90fe14f925729cde58022f5d034241918a5c4f9797cac62f6b3a9dd"}, - {file = "psycopg2_binary-2.8.6-cp39-cp39-win32.whl", hash = "sha256:6422f2ff0919fd720195f64ffd8f924c1395d30f9a495f31e2392c2efafb5056"}, - {file = "psycopg2_binary-2.8.6-cp39-cp39-win_amd64.whl", hash = "sha256:15978a1fbd225583dd8cdaf37e67ccc278b5abecb4caf6b2d6b8e2b948e953f6"}, + {file = "psycopg2-binary-2.9.1.tar.gz", hash = "sha256:b0221ca5a9837e040ebf61f48899926b5783668b7807419e4adae8175a31f773"}, + {file = "psycopg2_binary-2.9.1-cp36-cp36m-macosx_10_14_x86_64.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl", hash = "sha256:c250a7ec489b652c892e4f0a5d122cc14c3780f9f643e1a326754aedf82d9a76"}, + {file = "psycopg2_binary-2.9.1-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aef9aee84ec78af51107181d02fe8773b100b01c5dfde351184ad9223eab3698"}, + {file = "psycopg2_binary-2.9.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:123c3fb684e9abfc47218d3784c7b4c47c8587951ea4dd5bc38b6636ac57f616"}, + {file = "psycopg2_binary-2.9.1-cp36-cp36m-manylinux_2_24_aarch64.whl", hash = "sha256:995fc41ebda5a7a663a254a1dcac52638c3e847f48307b5416ee373da15075d7"}, + {file = "psycopg2_binary-2.9.1-cp36-cp36m-manylinux_2_24_ppc64le.whl", hash = "sha256:fbb42a541b1093385a2d8c7eec94d26d30437d0e77c1d25dae1dcc46741a385e"}, + {file = "psycopg2_binary-2.9.1-cp36-cp36m-win32.whl", hash = "sha256:20f1ab44d8c352074e2d7ca67dc00843067788791be373e67a0911998787ce7d"}, + {file = "psycopg2_binary-2.9.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f6fac64a38f6768e7bc7b035b9e10d8a538a9fadce06b983fb3e6fa55ac5f5ce"}, + {file = "psycopg2_binary-2.9.1-cp37-cp37m-macosx_10_14_x86_64.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl", hash = "sha256:1e3a362790edc0a365385b1ac4cc0acc429a0c0d662d829a50b6ce743ae61b5a"}, + {file = "psycopg2_binary-2.9.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f8559617b1fcf59a9aedba2c9838b5b6aa211ffedecabca412b92a1ff75aac1a"}, + {file = "psycopg2_binary-2.9.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a36c7eb6152ba5467fb264d73844877be8b0847874d4822b7cf2d3c0cb8cdcb0"}, + {file = "psycopg2_binary-2.9.1-cp37-cp37m-manylinux_2_24_aarch64.whl", hash = "sha256:2f62c207d1740b0bde5c4e949f857b044818f734a3d57f1d0d0edc65050532ed"}, + {file = "psycopg2_binary-2.9.1-cp37-cp37m-manylinux_2_24_ppc64le.whl", hash = "sha256:cfc523edecddaef56f6740d7de1ce24a2fdf94fd5e704091856a201872e37f9f"}, + {file = "psycopg2_binary-2.9.1-cp37-cp37m-win32.whl", hash = "sha256:1e85b74cbbb3056e3656f1cc4781294df03383127a8114cbc6531e8b8367bf1e"}, + {file = "psycopg2_binary-2.9.1-cp37-cp37m-win_amd64.whl", hash = "sha256:1473c0215b0613dd938db54a653f68251a45a78b05f6fc21af4326f40e8360a2"}, + {file = "psycopg2_binary-2.9.1-cp38-cp38-macosx_10_14_x86_64.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl", hash = "sha256:35c4310f8febe41f442d3c65066ca93cccefd75013df3d8c736c5b93ec288140"}, + {file = "psycopg2_binary-2.9.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c13d72ed6af7fd2c8acbd95661cf9477f94e381fce0792c04981a8283b52917"}, + {file = "psycopg2_binary-2.9.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14db1752acdd2187d99cb2ca0a1a6dfe57fc65c3281e0f20e597aac8d2a5bd90"}, + {file = "psycopg2_binary-2.9.1-cp38-cp38-manylinux_2_24_aarch64.whl", hash = "sha256:aed4a9a7e3221b3e252c39d0bf794c438dc5453bc2963e8befe9d4cd324dff72"}, + {file = "psycopg2_binary-2.9.1-cp38-cp38-manylinux_2_24_ppc64le.whl", hash = "sha256:da113b70f6ec40e7d81b43d1b139b9db6a05727ab8be1ee559f3a69854a69d34"}, + {file = "psycopg2_binary-2.9.1-cp38-cp38-win32.whl", hash = "sha256:4235f9d5ddcab0b8dbd723dca56ea2922b485ea00e1dafacf33b0c7e840b3d32"}, + {file = "psycopg2_binary-2.9.1-cp38-cp38-win_amd64.whl", hash = "sha256:988b47ac70d204aed01589ed342303da7c4d84b56c2f4c4b8b00deda123372bf"}, + {file = "psycopg2_binary-2.9.1-cp39-cp39-macosx_10_14_x86_64.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl", hash = "sha256:7360647ea04db2e7dff1648d1da825c8cf68dc5fbd80b8fb5b3ee9f068dcd21a"}, + {file = "psycopg2_binary-2.9.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca86db5b561b894f9e5f115d6a159fff2a2570a652e07889d8a383b5fae66eb4"}, + {file = "psycopg2_binary-2.9.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ced67f1e34e1a450cdb48eb53ca73b60aa0af21c46b9b35ac3e581cf9f00e31"}, + {file = "psycopg2_binary-2.9.1-cp39-cp39-manylinux_2_24_aarch64.whl", hash = "sha256:0f2e04bd2a2ab54fa44ee67fe2d002bb90cee1c0f1cc0ebc3148af7b02034cbd"}, + {file = "psycopg2_binary-2.9.1-cp39-cp39-manylinux_2_24_ppc64le.whl", hash = "sha256:3242b9619de955ab44581a03a64bdd7d5e470cc4183e8fcadd85ab9d3756ce7a"}, + {file = "psycopg2_binary-2.9.1-cp39-cp39-win32.whl", hash = "sha256:0b7dae87f0b729922e06f85f667de7bf16455d411971b2043bbd9577af9d1975"}, + {file = "psycopg2_binary-2.9.1-cp39-cp39-win_amd64.whl", hash = "sha256:b4d7679a08fea64573c969f6994a2631908bb2c0e69a7235648642f3d2e39a68"}, ] ptable = [ {file = "PTable-0.9.2.tar.gz", hash = "sha256:aa7fc151cb40f2dabcd2275ba6f7fd0ff8577a86be3365cd3fb297cbe09cc292"}, @@ -3016,8 +3021,8 @@ toml = [ {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] tqdm = [ - {file = "tqdm-4.61.0-py2.py3-none-any.whl", hash = "sha256:736524215c690621b06fc89d0310a49822d75e599fcd0feb7cc742b98d692493"}, - {file = "tqdm-4.61.0.tar.gz", hash = "sha256:cd5791b5d7c3f2f1819efc81d36eb719a38e0906a7380365c556779f585ea042"}, + {file = "tqdm-4.61.1-py2.py3-none-any.whl", hash = "sha256:aa0c29f03f298951ac6318f7c8ce584e48fa22ec26396e6411e43d038243bdb2"}, + {file = "tqdm-4.61.1.tar.gz", hash = "sha256:24be966933e942be5f074c29755a95b315c69a91f839a29139bf26ffffe2d3fd"}, ] transaction = [ {file = "transaction-2.4.0-py2.py3-none-any.whl", hash = "sha256:b96a5e9aaa73f905759bc9ccf0021bf4864c01ac36666e0d28395e871f6d584a"}, @@ -3056,8 +3061,8 @@ webob = [ {file = "WebOb-1.8.7.tar.gz", hash = "sha256:b64ef5141be559cfade448f044fa45c2260351edcb6a8ef6b7e00c7dcef0c323"}, ] websocket-client = [ - {file = "websocket-client-1.0.1.tar.gz", hash = "sha256:3e2bf58191d4619b161389a95bdce84ce9e0b24eb8107e7e590db682c2d0ca81"}, - {file = "websocket_client-1.0.1-py2.py3-none-any.whl", hash = "sha256:abf306dc6351dcef07f4d40453037e51cc5d9da2ef60d0fc5d0fe3bcda255372"}, + {file = "websocket-client-1.1.0.tar.gz", hash = "sha256:b68e4959d704768fa20e35c9d508c8dc2bbc041fd8d267c0d7345cffe2824568"}, + {file = "websocket_client-1.1.0-py2.py3-none-any.whl", hash = "sha256:e5c333bfa9fa739538b652b6f8c8fc2559f1d364243c8a689d7c0e1d41c2e611"}, ] webtest = [ {file = "WebTest-2.0.35-py2.py3-none-any.whl", hash = "sha256:44ddfe99b5eca4cf07675e7222c81dd624d22f9a26035d2b93dc8862dc1153c6"}, diff --git a/pyproject.toml b/pyproject.toml index 08616b964b..586cec5c87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,7 @@ chardet = "3.0.4" colorama = "0.3.3" dcicpyvcf = "1.0.0" dcicsnovault = "4.8.0.0b2" -dcicutils = "^1.15.0" +dcicutils = "1.18.1.2b2" elasticsearch = "6.8.1" execnet = "1.4.1" future = "^0.15.2" From f6fdc625329bc6892cb275add2054f8cd7d5509a Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 24 Jun 2021 13:51:58 -0400 Subject: [PATCH 114/120] refactor to use any_alpha, remove env dockerfile arg --- Dockerfile | 4 +-- .../{mastertest.ini => cgap_any_alpha.ini} | 26 +++++++++---------- 2 files changed, 13 insertions(+), 17 deletions(-) rename deploy/docker/production/{mastertest.ini => cgap_any_alpha.ini} (74%) diff --git a/Dockerfile b/Dockerfile index 163614b160..c59b3d5d01 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,10 +9,8 @@ FROM python@sha256:db248d2d0494973550d323dd6b82af7fc2f4c1e0365769a758abd7fac2aa7 MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu" # Build Arguments -ARG CGAP_ENV_NAME -ENV CGAP_ENV_NAME=${CGAP_ENV_NAME:-"cgap-mastertest"} ARG INI_BASE -ENV INI_BASE=${INI_BASE:-"mastertest.ini"} +ENV INI_BASE=${INI_BASE:-"cgap_any_alpha.ini"} # Configure (global) Env ENV NGINX_USER=nginx diff --git a/deploy/docker/production/mastertest.ini b/deploy/docker/production/cgap_any_alpha.ini similarity index 74% rename from deploy/docker/production/mastertest.ini rename to deploy/docker/production/cgap_any_alpha.ini index 57a0d31ce5..4ca53b80f9 100644 --- a/deploy/docker/production/mastertest.ini +++ b/deploy/docker/production/cgap_any_alpha.ini @@ -1,36 +1,34 @@ [app:app] -beanstalk_app_version = c4_519 use = config:base.ini#app session.secret = %(here)s/session-secret.b64 -file_upload_bucket = ${ENCODED_FILES_BUCKET} -file_wfout_bucket = ${ENCODED_WFOUT_BUCKET} -blob_bucket = ${ENCODED_BLOBS_BUCKET} -system_bucket = ${ENCODED_SYSTEM_BUCKET} -metadata_bundles_bucket = ${ENCODED_METADATA_BUNDLE_BUCKET} +file_upload_bucket = ${FILE_UPLOAD_BUCKET} +file_wfout_bucket = ${FILE_WFOUT_BUCKET} +blob_bucket = ${BLOB_BUCKET} +system_bucket = ${SYSTEM_BUCKET} +metadata_bundles_bucket = ${METADATA_BUNDLES_BUCKET} sentry_dsn = ${SENTRY_DSN} +# blob_store_profile_name = encoded-4dn-files accession_factory = encoded.server_defaults.enc_accession elasticsearch.server = ${ES_SERVER} -snovault.app_version = ${APP_VERSION} -env.name = cgap-mastertest +snovault.app_version = ask-pip +env.name = ${BS_ENV} +mirror.env.name = ${BS_MIRROR_ENV} encoded_version = ${PROJECT_VERSION} eb_app_version = ${APP_VERSION} snovault_version = ${SNOVAULT_VERSION} utils_version = ${UTILS_VERSION} mpindexer = false indexer = ${INDEXER} -indexer.namespace = cgap-mastertest +indexer.namespace = ${ES_NAMESPACE} index_server = ${INDEX_SERVER} elasticsearch.aws_auth = true production = true -load_test_data = encoded.loadxl:load_test_data +load_test_data = encoded.loadxl:load_${DATA_SET}_data sqlalchemy.url = postgresql://${RDS_USERNAME}:${RDS_PASSWORD}@${RDS_HOSTNAME}:${RDS_PORT}/${RDS_DB_NAME} [composite:indexer] use = config:base.ini#indexer -[composite:ingester] -use = config:base.ini#ingester - [pipeline:main] pipeline = config:base.ini#memlimit @@ -82,4 +80,4 @@ level = NOTSET formatter = generic [formatter_generic] -format = %(message)s +format = %(message)s \ No newline at end of file From 1b388e8ffa7b3084adb8ec73b09781975eb7895a Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 29 Jun 2021 11:11:10 -0400 Subject: [PATCH 115/120] pass absolute paths --- deploy/docker/production/assume_identity.py | 7 +++++-- src/encoded/tests/test_post_put_patch.py | 10 +++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/deploy/docker/production/assume_identity.py b/deploy/docker/production/assume_identity.py index 7af4ebad22..239175b3a1 100644 --- a/deploy/docker/production/assume_identity.py +++ b/deploy/docker/production/assume_identity.py @@ -6,6 +6,7 @@ import os import logging +import subprocess from dcicutils.qa_utils import override_environ from dcicutils.deployment_utils import IniFileManager from dcicutils.secrets_utils import assume_identity @@ -19,7 +20,6 @@ class CGAPDockerIniFileManager(IniFileManager): """ This runs at top level, so path is slightly different. """ # should work but doesn't (missing cgap-portal): os.path.join(os.path.dirname(_MY_DIR), "pyproject.toml") - # TODO: repair via # expected = # actual = # assert actual == expected, "The actual value %s was not what we expected, %s." % (actual, expected) @@ -36,7 +36,10 @@ def build_production_ini_from_global_application_configuration(): # build production.ini with override_environ(**identity): - CGAPDockerIniFileManager.build_ini_file_from_template('cgap_any_alpha.ini', 'production.ini') + CGAPDockerIniFileManager.build_ini_file_from_template( + '/home/nginx/cgap-portal/deploy/ini_files/cgap_any_alpha.ini', + '/home/nginx/cgap-portal/production.ini' + ) if __name__ == '__main__': diff --git a/src/encoded/tests/test_post_put_patch.py b/src/encoded/tests/test_post_put_patch.py index b5609cb6d1..9549d49ac8 100644 --- a/src/encoded/tests/test_post_put_patch.py +++ b/src/encoded/tests/test_post_put_patch.py @@ -248,7 +248,7 @@ def test_post_check_only_invalid_data(testapp, disorder_data): ''' note theese test should work on any object ''' - disorder_data['taxon_id'] = 24; + disorder_data['taxon_id'] = 24 testapp.post_json('/disorder/?check_only=true', disorder_data, status=422) @@ -327,7 +327,7 @@ def test_patch_delete_fields_restricted_fields_admin(link_targets, testapp): res = testapp.post_json(COLLECTION_URL, item_with_link[0], status=201) url = res.location assert res.json['@graph'][0]['protected_link'] - res = testapp.patch_json(url + "?delete_fields=protected_link", {}, status=200) + testapp.patch_json(url + "?delete_fields=protected_link", {}, status=200) def test_patch_delete_fields_restricted_fields_submitter(content, testapp, submitter_testapp): @@ -344,10 +344,14 @@ def test_patch_delete_fields_restricted_fields_submitter(content, testapp, submi res1 = submitter_testapp.patch_json(url + "?delete_fields=protected", {}, status=200) assert res1.json['@graph'][0]['protected'] == 'protected default' - # change protected value + # submitter cannot change value + submitter_testapp.patch_json(url, {'protected': 'protected new'}, status=422) + + # admin can change protected value res = testapp.patch_json(url, {'protected': 'protected new'}, status=200) assert res.json['@graph'][0]['protected'] == 'protected new' + # results in a delta in the protected field, reject res2 = submitter_testapp.patch_json(url + "?delete_fields=protected", {}, status=422) res_errors = res2.json['errors'] assert len(res_errors) == 2 From ce69033df045f31846d44e87d5aefabe65ffaf26 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 29 Jun 2021 12:02:59 -0400 Subject: [PATCH 116/120] pull in botocore, snovault w fix --- poetry.lock | 595 +++++++++++++++++++++++++------------------------ pyproject.toml | 4 +- 2 files changed, 300 insertions(+), 299 deletions(-) diff --git a/poetry.lock b/poetry.lock index defd3da760..3ce70693db 100644 --- a/poetry.lock +++ b/poetry.lock @@ -106,8 +106,8 @@ s3transfer = ">=0.3.0,<0.4.0" [[package]] name = "boto3-stubs" -version = "1.17.99" -description = "Type annotations for boto3 1.17.99, generated by mypy-boto3-buider 4.18.4" +version = "1.17.102.post1" +description = "Type annotations for boto3 1.17.102, generated by mypy-boto3-buider 4.21.0" category = "dev" optional = false python-versions = ">=3.6" @@ -117,282 +117,282 @@ botocore-stubs = "*" typing-extensions = {version = "*", markers = "python_version < \"3.8\""} [package.extras] -accessanalyzer = ["mypy-boto3-accessanalyzer (==1.17.99)"] -acm = ["mypy-boto3-acm (==1.17.99)"] -acm-pca = ["mypy-boto3-acm-pca (==1.17.99)"] -alexaforbusiness = ["mypy-boto3-alexaforbusiness (==1.17.99)"] -all = ["mypy-boto3-accessanalyzer (==1.17.99)", "mypy-boto3-acm (==1.17.99)", "mypy-boto3-acm-pca (==1.17.99)", "mypy-boto3-alexaforbusiness (==1.17.99)", "mypy-boto3-amp (==1.17.99)", "mypy-boto3-amplify (==1.17.99)", "mypy-boto3-amplifybackend (==1.17.99)", "mypy-boto3-apigateway (==1.17.99)", "mypy-boto3-apigatewaymanagementapi (==1.17.99)", "mypy-boto3-apigatewayv2 (==1.17.99)", "mypy-boto3-appconfig (==1.17.99)", "mypy-boto3-appflow (==1.17.99)", "mypy-boto3-appintegrations (==1.17.99)", "mypy-boto3-application-autoscaling (==1.17.99)", "mypy-boto3-application-insights (==1.17.99)", "mypy-boto3-applicationcostprofiler (==1.17.99)", "mypy-boto3-appmesh (==1.17.99)", "mypy-boto3-apprunner (==1.17.99)", "mypy-boto3-appstream (==1.17.99)", "mypy-boto3-appsync (==1.17.99)", "mypy-boto3-athena (==1.17.99)", "mypy-boto3-auditmanager (==1.17.99)", "mypy-boto3-autoscaling (==1.17.99)", "mypy-boto3-autoscaling-plans (==1.17.99)", "mypy-boto3-backup (==1.17.99)", "mypy-boto3-batch (==1.17.99)", "mypy-boto3-braket (==1.17.99)", "mypy-boto3-budgets (==1.17.99)", "mypy-boto3-ce (==1.17.99)", "mypy-boto3-chime (==1.17.99)", "mypy-boto3-cloud9 (==1.17.99)", "mypy-boto3-clouddirectory (==1.17.99)", "mypy-boto3-cloudformation (==1.17.99)", "mypy-boto3-cloudfront (==1.17.99)", "mypy-boto3-cloudhsm (==1.17.99)", "mypy-boto3-cloudhsmv2 (==1.17.99)", "mypy-boto3-cloudsearch (==1.17.99)", "mypy-boto3-cloudsearchdomain (==1.17.99)", "mypy-boto3-cloudtrail (==1.17.99)", "mypy-boto3-cloudwatch (==1.17.99)", "mypy-boto3-codeartifact (==1.17.99)", "mypy-boto3-codebuild (==1.17.99)", "mypy-boto3-codecommit (==1.17.99)", "mypy-boto3-codedeploy (==1.17.99)", "mypy-boto3-codeguru-reviewer (==1.17.99)", "mypy-boto3-codeguruprofiler (==1.17.99)", "mypy-boto3-codepipeline (==1.17.99)", "mypy-boto3-codestar (==1.17.99)", "mypy-boto3-codestar-connections (==1.17.99)", "mypy-boto3-codestar-notifications (==1.17.99)", "mypy-boto3-cognito-identity (==1.17.99)", "mypy-boto3-cognito-idp (==1.17.99)", "mypy-boto3-cognito-sync (==1.17.99)", "mypy-boto3-comprehend (==1.17.99)", "mypy-boto3-comprehendmedical (==1.17.99)", "mypy-boto3-compute-optimizer (==1.17.99)", "mypy-boto3-config (==1.17.99)", "mypy-boto3-connect (==1.17.99)", "mypy-boto3-connect-contact-lens (==1.17.99)", "mypy-boto3-connectparticipant (==1.17.99)", "mypy-boto3-cur (==1.17.99)", "mypy-boto3-customer-profiles (==1.17.99)", "mypy-boto3-databrew (==1.17.99)", "mypy-boto3-dataexchange (==1.17.99)", "mypy-boto3-datapipeline (==1.17.99)", "mypy-boto3-datasync (==1.17.99)", "mypy-boto3-dax (==1.17.99)", "mypy-boto3-detective (==1.17.99)", "mypy-boto3-devicefarm (==1.17.99)", "mypy-boto3-devops-guru (==1.17.99)", "mypy-boto3-directconnect (==1.17.99)", "mypy-boto3-discovery (==1.17.99)", "mypy-boto3-dlm (==1.17.99)", "mypy-boto3-dms (==1.17.99)", "mypy-boto3-docdb (==1.17.99)", "mypy-boto3-ds (==1.17.99)", "mypy-boto3-dynamodb (==1.17.99)", "mypy-boto3-dynamodbstreams (==1.17.99)", "mypy-boto3-ebs (==1.17.99)", "mypy-boto3-ec2 (==1.17.99)", "mypy-boto3-ec2-instance-connect (==1.17.99)", "mypy-boto3-ecr (==1.17.99)", "mypy-boto3-ecr-public (==1.17.99)", "mypy-boto3-ecs (==1.17.99)", "mypy-boto3-efs (==1.17.99)", "mypy-boto3-eks (==1.17.99)", "mypy-boto3-elastic-inference (==1.17.99)", "mypy-boto3-elasticache (==1.17.99)", "mypy-boto3-elasticbeanstalk (==1.17.99)", "mypy-boto3-elastictranscoder (==1.17.99)", "mypy-boto3-elb (==1.17.99)", "mypy-boto3-elbv2 (==1.17.99)", "mypy-boto3-emr (==1.17.99)", "mypy-boto3-emr-containers (==1.17.99)", "mypy-boto3-es (==1.17.99)", "mypy-boto3-events (==1.17.99)", "mypy-boto3-finspace (==1.17.99)", "mypy-boto3-finspace-data (==1.17.99)", "mypy-boto3-firehose (==1.17.99)", "mypy-boto3-fis (==1.17.99)", "mypy-boto3-fms (==1.17.99)", "mypy-boto3-forecast (==1.17.99)", "mypy-boto3-forecastquery (==1.17.99)", "mypy-boto3-frauddetector (==1.17.99)", "mypy-boto3-fsx (==1.17.99)", "mypy-boto3-gamelift (==1.17.99)", "mypy-boto3-glacier (==1.17.99)", "mypy-boto3-globalaccelerator (==1.17.99)", "mypy-boto3-glue (==1.17.99)", "mypy-boto3-greengrass (==1.17.99)", "mypy-boto3-greengrassv2 (==1.17.99)", "mypy-boto3-groundstation (==1.17.99)", "mypy-boto3-guardduty (==1.17.99)", "mypy-boto3-health (==1.17.99)", "mypy-boto3-healthlake (==1.17.99)", "mypy-boto3-honeycode (==1.17.99)", "mypy-boto3-iam (==1.17.99)", "mypy-boto3-identitystore (==1.17.99)", "mypy-boto3-imagebuilder (==1.17.99)", "mypy-boto3-importexport (==1.17.99)", "mypy-boto3-inspector (==1.17.99)", "mypy-boto3-iot (==1.17.99)", "mypy-boto3-iot-data (==1.17.99)", "mypy-boto3-iot-jobs-data (==1.17.99)", "mypy-boto3-iot1click-devices (==1.17.99)", "mypy-boto3-iot1click-projects (==1.17.99)", "mypy-boto3-iotanalytics (==1.17.99)", "mypy-boto3-iotdeviceadvisor (==1.17.99)", "mypy-boto3-iotevents (==1.17.99)", "mypy-boto3-iotevents-data (==1.17.99)", "mypy-boto3-iotfleethub (==1.17.99)", "mypy-boto3-iotsecuretunneling (==1.17.99)", "mypy-boto3-iotsitewise (==1.17.99)", "mypy-boto3-iotthingsgraph (==1.17.99)", "mypy-boto3-iotwireless (==1.17.99)", "mypy-boto3-ivs (==1.17.99)", "mypy-boto3-kafka (==1.17.99)", "mypy-boto3-kendra (==1.17.99)", "mypy-boto3-kinesis (==1.17.99)", "mypy-boto3-kinesis-video-archived-media (==1.17.99)", "mypy-boto3-kinesis-video-media (==1.17.99)", "mypy-boto3-kinesis-video-signaling (==1.17.99)", "mypy-boto3-kinesisanalytics (==1.17.99)", "mypy-boto3-kinesisanalyticsv2 (==1.17.99)", "mypy-boto3-kinesisvideo (==1.17.99)", "mypy-boto3-kms (==1.17.99)", "mypy-boto3-lakeformation (==1.17.99)", "mypy-boto3-lambda (==1.17.99)", "mypy-boto3-lex-models (==1.17.99)", "mypy-boto3-lex-runtime (==1.17.99)", "mypy-boto3-lexv2-models (==1.17.99)", "mypy-boto3-lexv2-runtime (==1.17.99)", "mypy-boto3-license-manager (==1.17.99)", "mypy-boto3-lightsail (==1.17.99)", "mypy-boto3-location (==1.17.99)", "mypy-boto3-logs (==1.17.99)", "mypy-boto3-lookoutequipment (==1.17.99)", "mypy-boto3-lookoutmetrics (==1.17.99)", "mypy-boto3-lookoutvision (==1.17.99)", "mypy-boto3-machinelearning (==1.17.99)", "mypy-boto3-macie (==1.17.99)", "mypy-boto3-macie2 (==1.17.99)", "mypy-boto3-managedblockchain (==1.17.99)", "mypy-boto3-marketplace-catalog (==1.17.99)", "mypy-boto3-marketplace-entitlement (==1.17.99)", "mypy-boto3-marketplacecommerceanalytics (==1.17.99)", "mypy-boto3-mediaconnect (==1.17.99)", "mypy-boto3-mediaconvert (==1.17.99)", "mypy-boto3-medialive (==1.17.99)", "mypy-boto3-mediapackage (==1.17.99)", "mypy-boto3-mediapackage-vod (==1.17.99)", "mypy-boto3-mediastore (==1.17.99)", "mypy-boto3-mediastore-data (==1.17.99)", "mypy-boto3-mediatailor (==1.17.99)", "mypy-boto3-meteringmarketplace (==1.17.99)", "mypy-boto3-mgh (==1.17.99)", "mypy-boto3-mgn (==1.17.99)", "mypy-boto3-migrationhub-config (==1.17.99)", "mypy-boto3-mobile (==1.17.99)", "mypy-boto3-mq (==1.17.99)", "mypy-boto3-mturk (==1.17.99)", "mypy-boto3-mwaa (==1.17.99)", "mypy-boto3-neptune (==1.17.99)", "mypy-boto3-network-firewall (==1.17.99)", "mypy-boto3-networkmanager (==1.17.99)", "mypy-boto3-nimble (==1.17.99)", "mypy-boto3-opsworks (==1.17.99)", "mypy-boto3-opsworkscm (==1.17.99)", "mypy-boto3-organizations (==1.17.99)", "mypy-boto3-outposts (==1.17.99)", "mypy-boto3-personalize (==1.17.99)", "mypy-boto3-personalize-events (==1.17.99)", "mypy-boto3-personalize-runtime (==1.17.99)", "mypy-boto3-pi (==1.17.99)", "mypy-boto3-pinpoint (==1.17.99)", "mypy-boto3-pinpoint-email (==1.17.99)", "mypy-boto3-pinpoint-sms-voice (==1.17.99)", "mypy-boto3-polly (==1.17.99)", "mypy-boto3-pricing (==1.17.99)", "mypy-boto3-proton (==1.17.99)", "mypy-boto3-qldb (==1.17.99)", "mypy-boto3-qldb-session (==1.17.99)", "mypy-boto3-quicksight (==1.17.99)", "mypy-boto3-ram (==1.17.99)", "mypy-boto3-rds (==1.17.99)", "mypy-boto3-rds-data (==1.17.99)", "mypy-boto3-redshift (==1.17.99)", "mypy-boto3-redshift-data (==1.17.99)", "mypy-boto3-rekognition (==1.17.99)", "mypy-boto3-resource-groups (==1.17.99)", "mypy-boto3-resourcegroupstaggingapi (==1.17.99)", "mypy-boto3-robomaker (==1.17.99)", "mypy-boto3-route53 (==1.17.99)", "mypy-boto3-route53domains (==1.17.99)", "mypy-boto3-route53resolver (==1.17.99)", "mypy-boto3-s3 (==1.17.99)", "mypy-boto3-s3control (==1.17.99)", "mypy-boto3-s3outposts (==1.17.99)", "mypy-boto3-sagemaker (==1.17.99)", "mypy-boto3-sagemaker-a2i-runtime (==1.17.99)", "mypy-boto3-sagemaker-edge (==1.17.99)", "mypy-boto3-sagemaker-featurestore-runtime (==1.17.99)", "mypy-boto3-sagemaker-runtime (==1.17.99)", "mypy-boto3-savingsplans (==1.17.99)", "mypy-boto3-schemas (==1.17.99)", "mypy-boto3-sdb (==1.17.99)", "mypy-boto3-secretsmanager (==1.17.99)", "mypy-boto3-securityhub (==1.17.99)", "mypy-boto3-serverlessrepo (==1.17.99)", "mypy-boto3-service-quotas (==1.17.99)", "mypy-boto3-servicecatalog (==1.17.99)", "mypy-boto3-servicecatalog-appregistry (==1.17.99)", "mypy-boto3-servicediscovery (==1.17.99)", "mypy-boto3-ses (==1.17.99)", "mypy-boto3-sesv2 (==1.17.99)", "mypy-boto3-shield (==1.17.99)", "mypy-boto3-signer (==1.17.99)", "mypy-boto3-sms (==1.17.99)", "mypy-boto3-sms-voice (==1.17.99)", "mypy-boto3-snowball (==1.17.99)", "mypy-boto3-sns (==1.17.99)", "mypy-boto3-sqs (==1.17.99)", "mypy-boto3-ssm (==1.17.99)", "mypy-boto3-ssm-contacts (==1.17.99)", "mypy-boto3-ssm-incidents (==1.17.99)", "mypy-boto3-sso (==1.17.99)", "mypy-boto3-sso-admin (==1.17.99)", "mypy-boto3-sso-oidc (==1.17.99)", "mypy-boto3-stepfunctions (==1.17.99)", "mypy-boto3-storagegateway (==1.17.99)", "mypy-boto3-sts (==1.17.99)", "mypy-boto3-support (==1.17.99)", "mypy-boto3-swf (==1.17.99)", "mypy-boto3-synthetics (==1.17.99)", "mypy-boto3-textract (==1.17.99)", "mypy-boto3-timestream-query (==1.17.99)", "mypy-boto3-timestream-write (==1.17.99)", "mypy-boto3-transcribe (==1.17.99)", "mypy-boto3-transfer (==1.17.99)", "mypy-boto3-translate (==1.17.99)", "mypy-boto3-waf (==1.17.99)", "mypy-boto3-waf-regional (==1.17.99)", "mypy-boto3-wafv2 (==1.17.99)", "mypy-boto3-wellarchitected (==1.17.99)", "mypy-boto3-workdocs (==1.17.99)", "mypy-boto3-worklink (==1.17.99)", "mypy-boto3-workmail (==1.17.99)", "mypy-boto3-workmailmessageflow (==1.17.99)", "mypy-boto3-workspaces (==1.17.99)", "mypy-boto3-xray (==1.17.99)"] -amp = ["mypy-boto3-amp (==1.17.99)"] -amplify = ["mypy-boto3-amplify (==1.17.99)"] -amplifybackend = ["mypy-boto3-amplifybackend (==1.17.99)"] -apigateway = ["mypy-boto3-apigateway (==1.17.99)"] -apigatewaymanagementapi = ["mypy-boto3-apigatewaymanagementapi (==1.17.99)"] -apigatewayv2 = ["mypy-boto3-apigatewayv2 (==1.17.99)"] -appconfig = ["mypy-boto3-appconfig (==1.17.99)"] -appflow = ["mypy-boto3-appflow (==1.17.99)"] -appintegrations = ["mypy-boto3-appintegrations (==1.17.99)"] -application-autoscaling = ["mypy-boto3-application-autoscaling (==1.17.99)"] -application-insights = ["mypy-boto3-application-insights (==1.17.99)"] -applicationcostprofiler = ["mypy-boto3-applicationcostprofiler (==1.17.99)"] -appmesh = ["mypy-boto3-appmesh (==1.17.99)"] -apprunner = ["mypy-boto3-apprunner (==1.17.99)"] -appstream = ["mypy-boto3-appstream (==1.17.99)"] -appsync = ["mypy-boto3-appsync (==1.17.99)"] -athena = ["mypy-boto3-athena (==1.17.99)"] -auditmanager = ["mypy-boto3-auditmanager (==1.17.99)"] -autoscaling = ["mypy-boto3-autoscaling (==1.17.99)"] -autoscaling-plans = ["mypy-boto3-autoscaling-plans (==1.17.99)"] -backup = ["mypy-boto3-backup (==1.17.99)"] -batch = ["mypy-boto3-batch (==1.17.99)"] -braket = ["mypy-boto3-braket (==1.17.99)"] -budgets = ["mypy-boto3-budgets (==1.17.99)"] -ce = ["mypy-boto3-ce (==1.17.99)"] -chime = ["mypy-boto3-chime (==1.17.99)"] -cloud9 = ["mypy-boto3-cloud9 (==1.17.99)"] -clouddirectory = ["mypy-boto3-clouddirectory (==1.17.99)"] -cloudformation = ["mypy-boto3-cloudformation (==1.17.99)"] -cloudfront = ["mypy-boto3-cloudfront (==1.17.99)"] -cloudhsm = ["mypy-boto3-cloudhsm (==1.17.99)"] -cloudhsmv2 = ["mypy-boto3-cloudhsmv2 (==1.17.99)"] -cloudsearch = ["mypy-boto3-cloudsearch (==1.17.99)"] -cloudsearchdomain = ["mypy-boto3-cloudsearchdomain (==1.17.99)"] -cloudtrail = ["mypy-boto3-cloudtrail (==1.17.99)"] -cloudwatch = ["mypy-boto3-cloudwatch (==1.17.99)"] -codeartifact = ["mypy-boto3-codeartifact (==1.17.99)"] -codebuild = ["mypy-boto3-codebuild (==1.17.99)"] -codecommit = ["mypy-boto3-codecommit (==1.17.99)"] -codedeploy = ["mypy-boto3-codedeploy (==1.17.99)"] -codeguru-reviewer = ["mypy-boto3-codeguru-reviewer (==1.17.99)"] -codeguruprofiler = ["mypy-boto3-codeguruprofiler (==1.17.99)"] -codepipeline = ["mypy-boto3-codepipeline (==1.17.99)"] -codestar = ["mypy-boto3-codestar (==1.17.99)"] -codestar-connections = ["mypy-boto3-codestar-connections (==1.17.99)"] -codestar-notifications = ["mypy-boto3-codestar-notifications (==1.17.99)"] -cognito-identity = ["mypy-boto3-cognito-identity (==1.17.99)"] -cognito-idp = ["mypy-boto3-cognito-idp (==1.17.99)"] -cognito-sync = ["mypy-boto3-cognito-sync (==1.17.99)"] -comprehend = ["mypy-boto3-comprehend (==1.17.99)"] -comprehendmedical = ["mypy-boto3-comprehendmedical (==1.17.99)"] -compute-optimizer = ["mypy-boto3-compute-optimizer (==1.17.99)"] -config = ["mypy-boto3-config (==1.17.99)"] -connect = ["mypy-boto3-connect (==1.17.99)"] -connect-contact-lens = ["mypy-boto3-connect-contact-lens (==1.17.99)"] -connectparticipant = ["mypy-boto3-connectparticipant (==1.17.99)"] -cur = ["mypy-boto3-cur (==1.17.99)"] -customer-profiles = ["mypy-boto3-customer-profiles (==1.17.99)"] -databrew = ["mypy-boto3-databrew (==1.17.99)"] -dataexchange = ["mypy-boto3-dataexchange (==1.17.99)"] -datapipeline = ["mypy-boto3-datapipeline (==1.17.99)"] -datasync = ["mypy-boto3-datasync (==1.17.99)"] -dax = ["mypy-boto3-dax (==1.17.99)"] -detective = ["mypy-boto3-detective (==1.17.99)"] -devicefarm = ["mypy-boto3-devicefarm (==1.17.99)"] -devops-guru = ["mypy-boto3-devops-guru (==1.17.99)"] -directconnect = ["mypy-boto3-directconnect (==1.17.99)"] -discovery = ["mypy-boto3-discovery (==1.17.99)"] -dlm = ["mypy-boto3-dlm (==1.17.99)"] -dms = ["mypy-boto3-dms (==1.17.99)"] -docdb = ["mypy-boto3-docdb (==1.17.99)"] -ds = ["mypy-boto3-ds (==1.17.99)"] -dynamodb = ["mypy-boto3-dynamodb (==1.17.99)"] -dynamodbstreams = ["mypy-boto3-dynamodbstreams (==1.17.99)"] -ebs = ["mypy-boto3-ebs (==1.17.99)"] -ec2 = ["mypy-boto3-ec2 (==1.17.99)"] -ec2-instance-connect = ["mypy-boto3-ec2-instance-connect (==1.17.99)"] -ecr = ["mypy-boto3-ecr (==1.17.99)"] -ecr-public = ["mypy-boto3-ecr-public (==1.17.99)"] -ecs = ["mypy-boto3-ecs (==1.17.99)"] -efs = ["mypy-boto3-efs (==1.17.99)"] -eks = ["mypy-boto3-eks (==1.17.99)"] -elastic-inference = ["mypy-boto3-elastic-inference (==1.17.99)"] -elasticache = ["mypy-boto3-elasticache (==1.17.99)"] -elasticbeanstalk = ["mypy-boto3-elasticbeanstalk (==1.17.99)"] -elastictranscoder = ["mypy-boto3-elastictranscoder (==1.17.99)"] -elb = ["mypy-boto3-elb (==1.17.99)"] -elbv2 = ["mypy-boto3-elbv2 (==1.17.99)"] -emr = ["mypy-boto3-emr (==1.17.99)"] -emr-containers = ["mypy-boto3-emr-containers (==1.17.99)"] -es = ["mypy-boto3-es (==1.17.99)"] -essential = ["mypy-boto3-cloudformation (==1.17.99)", "mypy-boto3-dynamodb (==1.17.99)", "mypy-boto3-ec2 (==1.17.99)", "mypy-boto3-lambda (==1.17.99)", "mypy-boto3-rds (==1.17.99)", "mypy-boto3-s3 (==1.17.99)", "mypy-boto3-sqs (==1.17.99)"] -events = ["mypy-boto3-events (==1.17.99)"] -finspace = ["mypy-boto3-finspace (==1.17.99)"] -finspace-data = ["mypy-boto3-finspace-data (==1.17.99)"] -firehose = ["mypy-boto3-firehose (==1.17.99)"] -fis = ["mypy-boto3-fis (==1.17.99)"] -fms = ["mypy-boto3-fms (==1.17.99)"] -forecast = ["mypy-boto3-forecast (==1.17.99)"] -forecastquery = ["mypy-boto3-forecastquery (==1.17.99)"] -frauddetector = ["mypy-boto3-frauddetector (==1.17.99)"] -fsx = ["mypy-boto3-fsx (==1.17.99)"] -gamelift = ["mypy-boto3-gamelift (==1.17.99)"] -glacier = ["mypy-boto3-glacier (==1.17.99)"] -globalaccelerator = ["mypy-boto3-globalaccelerator (==1.17.99)"] -glue = ["mypy-boto3-glue (==1.17.99)"] -greengrass = ["mypy-boto3-greengrass (==1.17.99)"] -greengrassv2 = ["mypy-boto3-greengrassv2 (==1.17.99)"] -groundstation = ["mypy-boto3-groundstation (==1.17.99)"] -guardduty = ["mypy-boto3-guardduty (==1.17.99)"] -health = ["mypy-boto3-health (==1.17.99)"] -healthlake = ["mypy-boto3-healthlake (==1.17.99)"] -honeycode = ["mypy-boto3-honeycode (==1.17.99)"] -iam = ["mypy-boto3-iam (==1.17.99)"] -identitystore = ["mypy-boto3-identitystore (==1.17.99)"] -imagebuilder = ["mypy-boto3-imagebuilder (==1.17.99)"] -importexport = ["mypy-boto3-importexport (==1.17.99)"] -inspector = ["mypy-boto3-inspector (==1.17.99)"] -iot = ["mypy-boto3-iot (==1.17.99)"] -iot-data = ["mypy-boto3-iot-data (==1.17.99)"] -iot-jobs-data = ["mypy-boto3-iot-jobs-data (==1.17.99)"] -iot1click-devices = ["mypy-boto3-iot1click-devices (==1.17.99)"] -iot1click-projects = ["mypy-boto3-iot1click-projects (==1.17.99)"] -iotanalytics = ["mypy-boto3-iotanalytics (==1.17.99)"] -iotdeviceadvisor = ["mypy-boto3-iotdeviceadvisor (==1.17.99)"] -iotevents = ["mypy-boto3-iotevents (==1.17.99)"] -iotevents-data = ["mypy-boto3-iotevents-data (==1.17.99)"] -iotfleethub = ["mypy-boto3-iotfleethub (==1.17.99)"] -iotsecuretunneling = ["mypy-boto3-iotsecuretunneling (==1.17.99)"] -iotsitewise = ["mypy-boto3-iotsitewise (==1.17.99)"] -iotthingsgraph = ["mypy-boto3-iotthingsgraph (==1.17.99)"] -iotwireless = ["mypy-boto3-iotwireless (==1.17.99)"] -ivs = ["mypy-boto3-ivs (==1.17.99)"] -kafka = ["mypy-boto3-kafka (==1.17.99)"] -kendra = ["mypy-boto3-kendra (==1.17.99)"] -kinesis = ["mypy-boto3-kinesis (==1.17.99)"] -kinesis-video-archived-media = ["mypy-boto3-kinesis-video-archived-media (==1.17.99)"] -kinesis-video-media = ["mypy-boto3-kinesis-video-media (==1.17.99)"] -kinesis-video-signaling = ["mypy-boto3-kinesis-video-signaling (==1.17.99)"] -kinesisanalytics = ["mypy-boto3-kinesisanalytics (==1.17.99)"] -kinesisanalyticsv2 = ["mypy-boto3-kinesisanalyticsv2 (==1.17.99)"] -kinesisvideo = ["mypy-boto3-kinesisvideo (==1.17.99)"] -kms = ["mypy-boto3-kms (==1.17.99)"] -lakeformation = ["mypy-boto3-lakeformation (==1.17.99)"] -lambda = ["mypy-boto3-lambda (==1.17.99)"] -lex-models = ["mypy-boto3-lex-models (==1.17.99)"] -lex-runtime = ["mypy-boto3-lex-runtime (==1.17.99)"] -lexv2-models = ["mypy-boto3-lexv2-models (==1.17.99)"] -lexv2-runtime = ["mypy-boto3-lexv2-runtime (==1.17.99)"] -license-manager = ["mypy-boto3-license-manager (==1.17.99)"] -lightsail = ["mypy-boto3-lightsail (==1.17.99)"] -location = ["mypy-boto3-location (==1.17.99)"] -logs = ["mypy-boto3-logs (==1.17.99)"] -lookoutequipment = ["mypy-boto3-lookoutequipment (==1.17.99)"] -lookoutmetrics = ["mypy-boto3-lookoutmetrics (==1.17.99)"] -lookoutvision = ["mypy-boto3-lookoutvision (==1.17.99)"] -machinelearning = ["mypy-boto3-machinelearning (==1.17.99)"] -macie = ["mypy-boto3-macie (==1.17.99)"] -macie2 = ["mypy-boto3-macie2 (==1.17.99)"] -managedblockchain = ["mypy-boto3-managedblockchain (==1.17.99)"] -marketplace-catalog = ["mypy-boto3-marketplace-catalog (==1.17.99)"] -marketplace-entitlement = ["mypy-boto3-marketplace-entitlement (==1.17.99)"] -marketplacecommerceanalytics = ["mypy-boto3-marketplacecommerceanalytics (==1.17.99)"] -mediaconnect = ["mypy-boto3-mediaconnect (==1.17.99)"] -mediaconvert = ["mypy-boto3-mediaconvert (==1.17.99)"] -medialive = ["mypy-boto3-medialive (==1.17.99)"] -mediapackage = ["mypy-boto3-mediapackage (==1.17.99)"] -mediapackage-vod = ["mypy-boto3-mediapackage-vod (==1.17.99)"] -mediastore = ["mypy-boto3-mediastore (==1.17.99)"] -mediastore-data = ["mypy-boto3-mediastore-data (==1.17.99)"] -mediatailor = ["mypy-boto3-mediatailor (==1.17.99)"] -meteringmarketplace = ["mypy-boto3-meteringmarketplace (==1.17.99)"] -mgh = ["mypy-boto3-mgh (==1.17.99)"] -mgn = ["mypy-boto3-mgn (==1.17.99)"] -migrationhub-config = ["mypy-boto3-migrationhub-config (==1.17.99)"] -mobile = ["mypy-boto3-mobile (==1.17.99)"] -mq = ["mypy-boto3-mq (==1.17.99)"] -mturk = ["mypy-boto3-mturk (==1.17.99)"] -mwaa = ["mypy-boto3-mwaa (==1.17.99)"] -neptune = ["mypy-boto3-neptune (==1.17.99)"] -network-firewall = ["mypy-boto3-network-firewall (==1.17.99)"] -networkmanager = ["mypy-boto3-networkmanager (==1.17.99)"] -nimble = ["mypy-boto3-nimble (==1.17.99)"] -opsworks = ["mypy-boto3-opsworks (==1.17.99)"] -opsworkscm = ["mypy-boto3-opsworkscm (==1.17.99)"] -organizations = ["mypy-boto3-organizations (==1.17.99)"] -outposts = ["mypy-boto3-outposts (==1.17.99)"] -personalize = ["mypy-boto3-personalize (==1.17.99)"] -personalize-events = ["mypy-boto3-personalize-events (==1.17.99)"] -personalize-runtime = ["mypy-boto3-personalize-runtime (==1.17.99)"] -pi = ["mypy-boto3-pi (==1.17.99)"] -pinpoint = ["mypy-boto3-pinpoint (==1.17.99)"] -pinpoint-email = ["mypy-boto3-pinpoint-email (==1.17.99)"] -pinpoint-sms-voice = ["mypy-boto3-pinpoint-sms-voice (==1.17.99)"] -polly = ["mypy-boto3-polly (==1.17.99)"] -pricing = ["mypy-boto3-pricing (==1.17.99)"] -proton = ["mypy-boto3-proton (==1.17.99)"] -qldb = ["mypy-boto3-qldb (==1.17.99)"] -qldb-session = ["mypy-boto3-qldb-session (==1.17.99)"] -quicksight = ["mypy-boto3-quicksight (==1.17.99)"] -ram = ["mypy-boto3-ram (==1.17.99)"] -rds = ["mypy-boto3-rds (==1.17.99)"] -rds-data = ["mypy-boto3-rds-data (==1.17.99)"] -redshift = ["mypy-boto3-redshift (==1.17.99)"] -redshift-data = ["mypy-boto3-redshift-data (==1.17.99)"] -rekognition = ["mypy-boto3-rekognition (==1.17.99)"] -resource-groups = ["mypy-boto3-resource-groups (==1.17.99)"] -resourcegroupstaggingapi = ["mypy-boto3-resourcegroupstaggingapi (==1.17.99)"] -robomaker = ["mypy-boto3-robomaker (==1.17.99)"] -route53 = ["mypy-boto3-route53 (==1.17.99)"] -route53domains = ["mypy-boto3-route53domains (==1.17.99)"] -route53resolver = ["mypy-boto3-route53resolver (==1.17.99)"] -s3 = ["mypy-boto3-s3 (==1.17.99)"] -s3control = ["mypy-boto3-s3control (==1.17.99)"] -s3outposts = ["mypy-boto3-s3outposts (==1.17.99)"] -sagemaker = ["mypy-boto3-sagemaker (==1.17.99)"] -sagemaker-a2i-runtime = ["mypy-boto3-sagemaker-a2i-runtime (==1.17.99)"] -sagemaker-edge = ["mypy-boto3-sagemaker-edge (==1.17.99)"] -sagemaker-featurestore-runtime = ["mypy-boto3-sagemaker-featurestore-runtime (==1.17.99)"] -sagemaker-runtime = ["mypy-boto3-sagemaker-runtime (==1.17.99)"] -savingsplans = ["mypy-boto3-savingsplans (==1.17.99)"] -schemas = ["mypy-boto3-schemas (==1.17.99)"] -sdb = ["mypy-boto3-sdb (==1.17.99)"] -secretsmanager = ["mypy-boto3-secretsmanager (==1.17.99)"] -securityhub = ["mypy-boto3-securityhub (==1.17.99)"] -serverlessrepo = ["mypy-boto3-serverlessrepo (==1.17.99)"] -service-quotas = ["mypy-boto3-service-quotas (==1.17.99)"] -servicecatalog = ["mypy-boto3-servicecatalog (==1.17.99)"] -servicecatalog-appregistry = ["mypy-boto3-servicecatalog-appregistry (==1.17.99)"] -servicediscovery = ["mypy-boto3-servicediscovery (==1.17.99)"] -ses = ["mypy-boto3-ses (==1.17.99)"] -sesv2 = ["mypy-boto3-sesv2 (==1.17.99)"] -shield = ["mypy-boto3-shield (==1.17.99)"] -signer = ["mypy-boto3-signer (==1.17.99)"] -sms = ["mypy-boto3-sms (==1.17.99)"] -sms-voice = ["mypy-boto3-sms-voice (==1.17.99)"] -snowball = ["mypy-boto3-snowball (==1.17.99)"] -sns = ["mypy-boto3-sns (==1.17.99)"] -sqs = ["mypy-boto3-sqs (==1.17.99)"] -ssm = ["mypy-boto3-ssm (==1.17.99)"] -ssm-contacts = ["mypy-boto3-ssm-contacts (==1.17.99)"] -ssm-incidents = ["mypy-boto3-ssm-incidents (==1.17.99)"] -sso = ["mypy-boto3-sso (==1.17.99)"] -sso-admin = ["mypy-boto3-sso-admin (==1.17.99)"] -sso-oidc = ["mypy-boto3-sso-oidc (==1.17.99)"] -stepfunctions = ["mypy-boto3-stepfunctions (==1.17.99)"] -storagegateway = ["mypy-boto3-storagegateway (==1.17.99)"] -sts = ["mypy-boto3-sts (==1.17.99)"] -support = ["mypy-boto3-support (==1.17.99)"] -swf = ["mypy-boto3-swf (==1.17.99)"] -synthetics = ["mypy-boto3-synthetics (==1.17.99)"] -textract = ["mypy-boto3-textract (==1.17.99)"] -timestream-query = ["mypy-boto3-timestream-query (==1.17.99)"] -timestream-write = ["mypy-boto3-timestream-write (==1.17.99)"] -transcribe = ["mypy-boto3-transcribe (==1.17.99)"] -transfer = ["mypy-boto3-transfer (==1.17.99)"] -translate = ["mypy-boto3-translate (==1.17.99)"] -waf = ["mypy-boto3-waf (==1.17.99)"] -waf-regional = ["mypy-boto3-waf-regional (==1.17.99)"] -wafv2 = ["mypy-boto3-wafv2 (==1.17.99)"] -wellarchitected = ["mypy-boto3-wellarchitected (==1.17.99)"] -workdocs = ["mypy-boto3-workdocs (==1.17.99)"] -worklink = ["mypy-boto3-worklink (==1.17.99)"] -workmail = ["mypy-boto3-workmail (==1.17.99)"] -workmailmessageflow = ["mypy-boto3-workmailmessageflow (==1.17.99)"] -workspaces = ["mypy-boto3-workspaces (==1.17.99)"] -xray = ["mypy-boto3-xray (==1.17.99)"] +accessanalyzer = ["mypy-boto3-accessanalyzer (==1.17.102.post1)"] +acm = ["mypy-boto3-acm (==1.17.102.post1)"] +acm-pca = ["mypy-boto3-acm-pca (==1.17.102.post1)"] +alexaforbusiness = ["mypy-boto3-alexaforbusiness (==1.17.102.post1)"] +all = ["mypy-boto3-accessanalyzer (==1.17.102.post1)", "mypy-boto3-acm (==1.17.102.post1)", "mypy-boto3-acm-pca (==1.17.102.post1)", "mypy-boto3-alexaforbusiness (==1.17.102.post1)", "mypy-boto3-amp (==1.17.102.post1)", "mypy-boto3-amplify (==1.17.102.post1)", "mypy-boto3-amplifybackend (==1.17.102.post1)", "mypy-boto3-apigateway (==1.17.102.post1)", "mypy-boto3-apigatewaymanagementapi (==1.17.102.post1)", "mypy-boto3-apigatewayv2 (==1.17.102.post1)", "mypy-boto3-appconfig (==1.17.102.post1)", "mypy-boto3-appflow (==1.17.102.post1)", "mypy-boto3-appintegrations (==1.17.102.post1)", "mypy-boto3-application-autoscaling (==1.17.102.post1)", "mypy-boto3-application-insights (==1.17.102.post1)", "mypy-boto3-applicationcostprofiler (==1.17.102.post1)", "mypy-boto3-appmesh (==1.17.102.post1)", "mypy-boto3-apprunner (==1.17.102.post1)", "mypy-boto3-appstream (==1.17.102.post1)", "mypy-boto3-appsync (==1.17.102.post1)", "mypy-boto3-athena (==1.17.102.post1)", "mypy-boto3-auditmanager (==1.17.102.post1)", "mypy-boto3-autoscaling (==1.17.102.post1)", "mypy-boto3-autoscaling-plans (==1.17.102.post1)", "mypy-boto3-backup (==1.17.102.post1)", "mypy-boto3-batch (==1.17.102.post1)", "mypy-boto3-braket (==1.17.102.post1)", "mypy-boto3-budgets (==1.17.102.post1)", "mypy-boto3-ce (==1.17.102.post1)", "mypy-boto3-chime (==1.17.102.post1)", "mypy-boto3-cloud9 (==1.17.102.post1)", "mypy-boto3-clouddirectory (==1.17.102.post1)", "mypy-boto3-cloudformation (==1.17.102.post1)", "mypy-boto3-cloudfront (==1.17.102.post1)", "mypy-boto3-cloudhsm (==1.17.102.post1)", "mypy-boto3-cloudhsmv2 (==1.17.102.post1)", "mypy-boto3-cloudsearch (==1.17.102.post1)", "mypy-boto3-cloudsearchdomain (==1.17.102.post1)", "mypy-boto3-cloudtrail (==1.17.102.post1)", "mypy-boto3-cloudwatch (==1.17.102.post1)", "mypy-boto3-codeartifact (==1.17.102.post1)", "mypy-boto3-codebuild (==1.17.102.post1)", "mypy-boto3-codecommit (==1.17.102.post1)", "mypy-boto3-codedeploy (==1.17.102.post1)", "mypy-boto3-codeguru-reviewer (==1.17.102.post1)", "mypy-boto3-codeguruprofiler (==1.17.102.post1)", "mypy-boto3-codepipeline (==1.17.102.post1)", "mypy-boto3-codestar (==1.17.102.post1)", "mypy-boto3-codestar-connections (==1.17.102.post1)", "mypy-boto3-codestar-notifications (==1.17.102.post1)", "mypy-boto3-cognito-identity (==1.17.102.post1)", "mypy-boto3-cognito-idp (==1.17.102.post1)", "mypy-boto3-cognito-sync (==1.17.102.post1)", "mypy-boto3-comprehend (==1.17.102.post1)", "mypy-boto3-comprehendmedical (==1.17.102.post1)", "mypy-boto3-compute-optimizer (==1.17.102.post1)", "mypy-boto3-config (==1.17.102.post1)", "mypy-boto3-connect (==1.17.102.post1)", "mypy-boto3-connect-contact-lens (==1.17.102.post1)", "mypy-boto3-connectparticipant (==1.17.102.post1)", "mypy-boto3-cur (==1.17.102.post1)", "mypy-boto3-customer-profiles (==1.17.102.post1)", "mypy-boto3-databrew (==1.17.102.post1)", "mypy-boto3-dataexchange (==1.17.102.post1)", "mypy-boto3-datapipeline (==1.17.102.post1)", "mypy-boto3-datasync (==1.17.102.post1)", "mypy-boto3-dax (==1.17.102.post1)", "mypy-boto3-detective (==1.17.102.post1)", "mypy-boto3-devicefarm (==1.17.102.post1)", "mypy-boto3-devops-guru (==1.17.102.post1)", "mypy-boto3-directconnect (==1.17.102.post1)", "mypy-boto3-discovery (==1.17.102.post1)", "mypy-boto3-dlm (==1.17.102.post1)", "mypy-boto3-dms (==1.17.102.post1)", "mypy-boto3-docdb (==1.17.102.post1)", "mypy-boto3-ds (==1.17.102.post1)", "mypy-boto3-dynamodb (==1.17.102.post1)", "mypy-boto3-dynamodbstreams (==1.17.102.post1)", "mypy-boto3-ebs (==1.17.102.post1)", "mypy-boto3-ec2 (==1.17.102.post1)", "mypy-boto3-ec2-instance-connect (==1.17.102.post1)", "mypy-boto3-ecr (==1.17.102.post1)", "mypy-boto3-ecr-public (==1.17.102.post1)", "mypy-boto3-ecs (==1.17.102.post1)", "mypy-boto3-efs (==1.17.102.post1)", "mypy-boto3-eks (==1.17.102.post1)", "mypy-boto3-elastic-inference (==1.17.102.post1)", "mypy-boto3-elasticache (==1.17.102.post1)", "mypy-boto3-elasticbeanstalk (==1.17.102.post1)", "mypy-boto3-elastictranscoder (==1.17.102.post1)", "mypy-boto3-elb (==1.17.102.post1)", "mypy-boto3-elbv2 (==1.17.102.post1)", "mypy-boto3-emr (==1.17.102.post1)", "mypy-boto3-emr-containers (==1.17.102.post1)", "mypy-boto3-es (==1.17.102.post1)", "mypy-boto3-events (==1.17.102.post1)", "mypy-boto3-finspace (==1.17.102.post1)", "mypy-boto3-finspace-data (==1.17.102.post1)", "mypy-boto3-firehose (==1.17.102.post1)", "mypy-boto3-fis (==1.17.102.post1)", "mypy-boto3-fms (==1.17.102.post1)", "mypy-boto3-forecast (==1.17.102.post1)", "mypy-boto3-forecastquery (==1.17.102.post1)", "mypy-boto3-frauddetector (==1.17.102.post1)", "mypy-boto3-fsx (==1.17.102.post1)", "mypy-boto3-gamelift (==1.17.102.post1)", "mypy-boto3-glacier (==1.17.102.post1)", "mypy-boto3-globalaccelerator (==1.17.102.post1)", "mypy-boto3-glue (==1.17.102.post1)", "mypy-boto3-greengrass (==1.17.102.post1)", "mypy-boto3-greengrassv2 (==1.17.102.post1)", "mypy-boto3-groundstation (==1.17.102.post1)", "mypy-boto3-guardduty (==1.17.102.post1)", "mypy-boto3-health (==1.17.102.post1)", "mypy-boto3-healthlake (==1.17.102.post1)", "mypy-boto3-honeycode (==1.17.102.post1)", "mypy-boto3-iam (==1.17.102.post1)", "mypy-boto3-identitystore (==1.17.102.post1)", "mypy-boto3-imagebuilder (==1.17.102.post1)", "mypy-boto3-importexport (==1.17.102.post1)", "mypy-boto3-inspector (==1.17.102.post1)", "mypy-boto3-iot (==1.17.102.post1)", "mypy-boto3-iot-data (==1.17.102.post1)", "mypy-boto3-iot-jobs-data (==1.17.102.post1)", "mypy-boto3-iot1click-devices (==1.17.102.post1)", "mypy-boto3-iot1click-projects (==1.17.102.post1)", "mypy-boto3-iotanalytics (==1.17.102.post1)", "mypy-boto3-iotdeviceadvisor (==1.17.102.post1)", "mypy-boto3-iotevents (==1.17.102.post1)", "mypy-boto3-iotevents-data (==1.17.102.post1)", "mypy-boto3-iotfleethub (==1.17.102.post1)", "mypy-boto3-iotsecuretunneling (==1.17.102.post1)", "mypy-boto3-iotsitewise (==1.17.102.post1)", "mypy-boto3-iotthingsgraph (==1.17.102.post1)", "mypy-boto3-iotwireless (==1.17.102.post1)", "mypy-boto3-ivs (==1.17.102.post1)", "mypy-boto3-kafka (==1.17.102.post1)", "mypy-boto3-kendra (==1.17.102.post1)", "mypy-boto3-kinesis (==1.17.102.post1)", "mypy-boto3-kinesis-video-archived-media (==1.17.102.post1)", "mypy-boto3-kinesis-video-media (==1.17.102.post1)", "mypy-boto3-kinesis-video-signaling (==1.17.102.post1)", "mypy-boto3-kinesisanalytics (==1.17.102.post1)", "mypy-boto3-kinesisanalyticsv2 (==1.17.102.post1)", "mypy-boto3-kinesisvideo (==1.17.102.post1)", "mypy-boto3-kms (==1.17.102.post1)", "mypy-boto3-lakeformation (==1.17.102.post1)", "mypy-boto3-lambda (==1.17.102.post1)", "mypy-boto3-lex-models (==1.17.102.post1)", "mypy-boto3-lex-runtime (==1.17.102.post1)", "mypy-boto3-lexv2-models (==1.17.102.post1)", "mypy-boto3-lexv2-runtime (==1.17.102.post1)", "mypy-boto3-license-manager (==1.17.102.post1)", "mypy-boto3-lightsail (==1.17.102.post1)", "mypy-boto3-location (==1.17.102.post1)", "mypy-boto3-logs (==1.17.102.post1)", "mypy-boto3-lookoutequipment (==1.17.102.post1)", "mypy-boto3-lookoutmetrics (==1.17.102.post1)", "mypy-boto3-lookoutvision (==1.17.102.post1)", "mypy-boto3-machinelearning (==1.17.102.post1)", "mypy-boto3-macie (==1.17.102.post1)", "mypy-boto3-macie2 (==1.17.102.post1)", "mypy-boto3-managedblockchain (==1.17.102.post1)", "mypy-boto3-marketplace-catalog (==1.17.102.post1)", "mypy-boto3-marketplace-entitlement (==1.17.102.post1)", "mypy-boto3-marketplacecommerceanalytics (==1.17.102.post1)", "mypy-boto3-mediaconnect (==1.17.102.post1)", "mypy-boto3-mediaconvert (==1.17.102.post1)", "mypy-boto3-medialive (==1.17.102.post1)", "mypy-boto3-mediapackage (==1.17.102.post1)", "mypy-boto3-mediapackage-vod (==1.17.102.post1)", "mypy-boto3-mediastore (==1.17.102.post1)", "mypy-boto3-mediastore-data (==1.17.102.post1)", "mypy-boto3-mediatailor (==1.17.102.post1)", "mypy-boto3-meteringmarketplace (==1.17.102.post1)", "mypy-boto3-mgh (==1.17.102.post1)", "mypy-boto3-mgn (==1.17.102.post1)", "mypy-boto3-migrationhub-config (==1.17.102.post1)", "mypy-boto3-mobile (==1.17.102.post1)", "mypy-boto3-mq (==1.17.102.post1)", "mypy-boto3-mturk (==1.17.102.post1)", "mypy-boto3-mwaa (==1.17.102.post1)", "mypy-boto3-neptune (==1.17.102.post1)", "mypy-boto3-network-firewall (==1.17.102.post1)", "mypy-boto3-networkmanager (==1.17.102.post1)", "mypy-boto3-nimble (==1.17.102.post1)", "mypy-boto3-opsworks (==1.17.102.post1)", "mypy-boto3-opsworkscm (==1.17.102.post1)", "mypy-boto3-organizations (==1.17.102.post1)", "mypy-boto3-outposts (==1.17.102.post1)", "mypy-boto3-personalize (==1.17.102.post1)", "mypy-boto3-personalize-events (==1.17.102.post1)", "mypy-boto3-personalize-runtime (==1.17.102.post1)", "mypy-boto3-pi (==1.17.102.post1)", "mypy-boto3-pinpoint (==1.17.102.post1)", "mypy-boto3-pinpoint-email (==1.17.102.post1)", "mypy-boto3-pinpoint-sms-voice (==1.17.102.post1)", "mypy-boto3-polly (==1.17.102.post1)", "mypy-boto3-pricing (==1.17.102.post1)", "mypy-boto3-proton (==1.17.102.post1)", "mypy-boto3-qldb (==1.17.102.post1)", "mypy-boto3-qldb-session (==1.17.102.post1)", "mypy-boto3-quicksight (==1.17.102.post1)", "mypy-boto3-ram (==1.17.102.post1)", "mypy-boto3-rds (==1.17.102.post1)", "mypy-boto3-rds-data (==1.17.102.post1)", "mypy-boto3-redshift (==1.17.102.post1)", "mypy-boto3-redshift-data (==1.17.102.post1)", "mypy-boto3-rekognition (==1.17.102.post1)", "mypy-boto3-resource-groups (==1.17.102.post1)", "mypy-boto3-resourcegroupstaggingapi (==1.17.102.post1)", "mypy-boto3-robomaker (==1.17.102.post1)", "mypy-boto3-route53 (==1.17.102.post1)", "mypy-boto3-route53domains (==1.17.102.post1)", "mypy-boto3-route53resolver (==1.17.102.post1)", "mypy-boto3-s3 (==1.17.102.post1)", "mypy-boto3-s3control (==1.17.102.post1)", "mypy-boto3-s3outposts (==1.17.102.post1)", "mypy-boto3-sagemaker (==1.17.102.post1)", "mypy-boto3-sagemaker-a2i-runtime (==1.17.102.post1)", "mypy-boto3-sagemaker-edge (==1.17.102.post1)", "mypy-boto3-sagemaker-featurestore-runtime (==1.17.102.post1)", "mypy-boto3-sagemaker-runtime (==1.17.102.post1)", "mypy-boto3-savingsplans (==1.17.102.post1)", "mypy-boto3-schemas (==1.17.102.post1)", "mypy-boto3-sdb (==1.17.102.post1)", "mypy-boto3-secretsmanager (==1.17.102.post1)", "mypy-boto3-securityhub (==1.17.102.post1)", "mypy-boto3-serverlessrepo (==1.17.102.post1)", "mypy-boto3-service-quotas (==1.17.102.post1)", "mypy-boto3-servicecatalog (==1.17.102.post1)", "mypy-boto3-servicecatalog-appregistry (==1.17.102.post1)", "mypy-boto3-servicediscovery (==1.17.102.post1)", "mypy-boto3-ses (==1.17.102.post1)", "mypy-boto3-sesv2 (==1.17.102.post1)", "mypy-boto3-shield (==1.17.102.post1)", "mypy-boto3-signer (==1.17.102.post1)", "mypy-boto3-sms (==1.17.102.post1)", "mypy-boto3-sms-voice (==1.17.102.post1)", "mypy-boto3-snowball (==1.17.102.post1)", "mypy-boto3-sns (==1.17.102.post1)", "mypy-boto3-sqs (==1.17.102.post1)", "mypy-boto3-ssm (==1.17.102.post1)", "mypy-boto3-ssm-contacts (==1.17.102.post1)", "mypy-boto3-ssm-incidents (==1.17.102.post1)", "mypy-boto3-sso (==1.17.102.post1)", "mypy-boto3-sso-admin (==1.17.102.post1)", "mypy-boto3-sso-oidc (==1.17.102.post1)", "mypy-boto3-stepfunctions (==1.17.102.post1)", "mypy-boto3-storagegateway (==1.17.102.post1)", "mypy-boto3-sts (==1.17.102.post1)", "mypy-boto3-support (==1.17.102.post1)", "mypy-boto3-swf (==1.17.102.post1)", "mypy-boto3-synthetics (==1.17.102.post1)", "mypy-boto3-textract (==1.17.102.post1)", "mypy-boto3-timestream-query (==1.17.102.post1)", "mypy-boto3-timestream-write (==1.17.102.post1)", "mypy-boto3-transcribe (==1.17.102.post1)", "mypy-boto3-transfer (==1.17.102.post1)", "mypy-boto3-translate (==1.17.102.post1)", "mypy-boto3-waf (==1.17.102.post1)", "mypy-boto3-waf-regional (==1.17.102.post1)", "mypy-boto3-wafv2 (==1.17.102.post1)", "mypy-boto3-wellarchitected (==1.17.102.post1)", "mypy-boto3-workdocs (==1.17.102.post1)", "mypy-boto3-worklink (==1.17.102.post1)", "mypy-boto3-workmail (==1.17.102.post1)", "mypy-boto3-workmailmessageflow (==1.17.102.post1)", "mypy-boto3-workspaces (==1.17.102.post1)", "mypy-boto3-xray (==1.17.102.post1)"] +amp = ["mypy-boto3-amp (==1.17.102.post1)"] +amplify = ["mypy-boto3-amplify (==1.17.102.post1)"] +amplifybackend = ["mypy-boto3-amplifybackend (==1.17.102.post1)"] +apigateway = ["mypy-boto3-apigateway (==1.17.102.post1)"] +apigatewaymanagementapi = ["mypy-boto3-apigatewaymanagementapi (==1.17.102.post1)"] +apigatewayv2 = ["mypy-boto3-apigatewayv2 (==1.17.102.post1)"] +appconfig = ["mypy-boto3-appconfig (==1.17.102.post1)"] +appflow = ["mypy-boto3-appflow (==1.17.102.post1)"] +appintegrations = ["mypy-boto3-appintegrations (==1.17.102.post1)"] +application-autoscaling = ["mypy-boto3-application-autoscaling (==1.17.102.post1)"] +application-insights = ["mypy-boto3-application-insights (==1.17.102.post1)"] +applicationcostprofiler = ["mypy-boto3-applicationcostprofiler (==1.17.102.post1)"] +appmesh = ["mypy-boto3-appmesh (==1.17.102.post1)"] +apprunner = ["mypy-boto3-apprunner (==1.17.102.post1)"] +appstream = ["mypy-boto3-appstream (==1.17.102.post1)"] +appsync = ["mypy-boto3-appsync (==1.17.102.post1)"] +athena = ["mypy-boto3-athena (==1.17.102.post1)"] +auditmanager = ["mypy-boto3-auditmanager (==1.17.102.post1)"] +autoscaling = ["mypy-boto3-autoscaling (==1.17.102.post1)"] +autoscaling-plans = ["mypy-boto3-autoscaling-plans (==1.17.102.post1)"] +backup = ["mypy-boto3-backup (==1.17.102.post1)"] +batch = ["mypy-boto3-batch (==1.17.102.post1)"] +braket = ["mypy-boto3-braket (==1.17.102.post1)"] +budgets = ["mypy-boto3-budgets (==1.17.102.post1)"] +ce = ["mypy-boto3-ce (==1.17.102.post1)"] +chime = ["mypy-boto3-chime (==1.17.102.post1)"] +cloud9 = ["mypy-boto3-cloud9 (==1.17.102.post1)"] +clouddirectory = ["mypy-boto3-clouddirectory (==1.17.102.post1)"] +cloudformation = ["mypy-boto3-cloudformation (==1.17.102.post1)"] +cloudfront = ["mypy-boto3-cloudfront (==1.17.102.post1)"] +cloudhsm = ["mypy-boto3-cloudhsm (==1.17.102.post1)"] +cloudhsmv2 = ["mypy-boto3-cloudhsmv2 (==1.17.102.post1)"] +cloudsearch = ["mypy-boto3-cloudsearch (==1.17.102.post1)"] +cloudsearchdomain = ["mypy-boto3-cloudsearchdomain (==1.17.102.post1)"] +cloudtrail = ["mypy-boto3-cloudtrail (==1.17.102.post1)"] +cloudwatch = ["mypy-boto3-cloudwatch (==1.17.102.post1)"] +codeartifact = ["mypy-boto3-codeartifact (==1.17.102.post1)"] +codebuild = ["mypy-boto3-codebuild (==1.17.102.post1)"] +codecommit = ["mypy-boto3-codecommit (==1.17.102.post1)"] +codedeploy = ["mypy-boto3-codedeploy (==1.17.102.post1)"] +codeguru-reviewer = ["mypy-boto3-codeguru-reviewer (==1.17.102.post1)"] +codeguruprofiler = ["mypy-boto3-codeguruprofiler (==1.17.102.post1)"] +codepipeline = ["mypy-boto3-codepipeline (==1.17.102.post1)"] +codestar = ["mypy-boto3-codestar (==1.17.102.post1)"] +codestar-connections = ["mypy-boto3-codestar-connections (==1.17.102.post1)"] +codestar-notifications = ["mypy-boto3-codestar-notifications (==1.17.102.post1)"] +cognito-identity = ["mypy-boto3-cognito-identity (==1.17.102.post1)"] +cognito-idp = ["mypy-boto3-cognito-idp (==1.17.102.post1)"] +cognito-sync = ["mypy-boto3-cognito-sync (==1.17.102.post1)"] +comprehend = ["mypy-boto3-comprehend (==1.17.102.post1)"] +comprehendmedical = ["mypy-boto3-comprehendmedical (==1.17.102.post1)"] +compute-optimizer = ["mypy-boto3-compute-optimizer (==1.17.102.post1)"] +config = ["mypy-boto3-config (==1.17.102.post1)"] +connect = ["mypy-boto3-connect (==1.17.102.post1)"] +connect-contact-lens = ["mypy-boto3-connect-contact-lens (==1.17.102.post1)"] +connectparticipant = ["mypy-boto3-connectparticipant (==1.17.102.post1)"] +cur = ["mypy-boto3-cur (==1.17.102.post1)"] +customer-profiles = ["mypy-boto3-customer-profiles (==1.17.102.post1)"] +databrew = ["mypy-boto3-databrew (==1.17.102.post1)"] +dataexchange = ["mypy-boto3-dataexchange (==1.17.102.post1)"] +datapipeline = ["mypy-boto3-datapipeline (==1.17.102.post1)"] +datasync = ["mypy-boto3-datasync (==1.17.102.post1)"] +dax = ["mypy-boto3-dax (==1.17.102.post1)"] +detective = ["mypy-boto3-detective (==1.17.102.post1)"] +devicefarm = ["mypy-boto3-devicefarm (==1.17.102.post1)"] +devops-guru = ["mypy-boto3-devops-guru (==1.17.102.post1)"] +directconnect = ["mypy-boto3-directconnect (==1.17.102.post1)"] +discovery = ["mypy-boto3-discovery (==1.17.102.post1)"] +dlm = ["mypy-boto3-dlm (==1.17.102.post1)"] +dms = ["mypy-boto3-dms (==1.17.102.post1)"] +docdb = ["mypy-boto3-docdb (==1.17.102.post1)"] +ds = ["mypy-boto3-ds (==1.17.102.post1)"] +dynamodb = ["mypy-boto3-dynamodb (==1.17.102.post1)"] +dynamodbstreams = ["mypy-boto3-dynamodbstreams (==1.17.102.post1)"] +ebs = ["mypy-boto3-ebs (==1.17.102.post1)"] +ec2 = ["mypy-boto3-ec2 (==1.17.102.post1)"] +ec2-instance-connect = ["mypy-boto3-ec2-instance-connect (==1.17.102.post1)"] +ecr = ["mypy-boto3-ecr (==1.17.102.post1)"] +ecr-public = ["mypy-boto3-ecr-public (==1.17.102.post1)"] +ecs = ["mypy-boto3-ecs (==1.17.102.post1)"] +efs = ["mypy-boto3-efs (==1.17.102.post1)"] +eks = ["mypy-boto3-eks (==1.17.102.post1)"] +elastic-inference = ["mypy-boto3-elastic-inference (==1.17.102.post1)"] +elasticache = ["mypy-boto3-elasticache (==1.17.102.post1)"] +elasticbeanstalk = ["mypy-boto3-elasticbeanstalk (==1.17.102.post1)"] +elastictranscoder = ["mypy-boto3-elastictranscoder (==1.17.102.post1)"] +elb = ["mypy-boto3-elb (==1.17.102.post1)"] +elbv2 = ["mypy-boto3-elbv2 (==1.17.102.post1)"] +emr = ["mypy-boto3-emr (==1.17.102.post1)"] +emr-containers = ["mypy-boto3-emr-containers (==1.17.102.post1)"] +es = ["mypy-boto3-es (==1.17.102.post1)"] +essential = ["mypy-boto3-cloudformation (==1.17.102.post1)", "mypy-boto3-dynamodb (==1.17.102.post1)", "mypy-boto3-ec2 (==1.17.102.post1)", "mypy-boto3-lambda (==1.17.102.post1)", "mypy-boto3-rds (==1.17.102.post1)", "mypy-boto3-s3 (==1.17.102.post1)", "mypy-boto3-sqs (==1.17.102.post1)"] +events = ["mypy-boto3-events (==1.17.102.post1)"] +finspace = ["mypy-boto3-finspace (==1.17.102.post1)"] +finspace-data = ["mypy-boto3-finspace-data (==1.17.102.post1)"] +firehose = ["mypy-boto3-firehose (==1.17.102.post1)"] +fis = ["mypy-boto3-fis (==1.17.102.post1)"] +fms = ["mypy-boto3-fms (==1.17.102.post1)"] +forecast = ["mypy-boto3-forecast (==1.17.102.post1)"] +forecastquery = ["mypy-boto3-forecastquery (==1.17.102.post1)"] +frauddetector = ["mypy-boto3-frauddetector (==1.17.102.post1)"] +fsx = ["mypy-boto3-fsx (==1.17.102.post1)"] +gamelift = ["mypy-boto3-gamelift (==1.17.102.post1)"] +glacier = ["mypy-boto3-glacier (==1.17.102.post1)"] +globalaccelerator = ["mypy-boto3-globalaccelerator (==1.17.102.post1)"] +glue = ["mypy-boto3-glue (==1.17.102.post1)"] +greengrass = ["mypy-boto3-greengrass (==1.17.102.post1)"] +greengrassv2 = ["mypy-boto3-greengrassv2 (==1.17.102.post1)"] +groundstation = ["mypy-boto3-groundstation (==1.17.102.post1)"] +guardduty = ["mypy-boto3-guardduty (==1.17.102.post1)"] +health = ["mypy-boto3-health (==1.17.102.post1)"] +healthlake = ["mypy-boto3-healthlake (==1.17.102.post1)"] +honeycode = ["mypy-boto3-honeycode (==1.17.102.post1)"] +iam = ["mypy-boto3-iam (==1.17.102.post1)"] +identitystore = ["mypy-boto3-identitystore (==1.17.102.post1)"] +imagebuilder = ["mypy-boto3-imagebuilder (==1.17.102.post1)"] +importexport = ["mypy-boto3-importexport (==1.17.102.post1)"] +inspector = ["mypy-boto3-inspector (==1.17.102.post1)"] +iot = ["mypy-boto3-iot (==1.17.102.post1)"] +iot-data = ["mypy-boto3-iot-data (==1.17.102.post1)"] +iot-jobs-data = ["mypy-boto3-iot-jobs-data (==1.17.102.post1)"] +iot1click-devices = ["mypy-boto3-iot1click-devices (==1.17.102.post1)"] +iot1click-projects = ["mypy-boto3-iot1click-projects (==1.17.102.post1)"] +iotanalytics = ["mypy-boto3-iotanalytics (==1.17.102.post1)"] +iotdeviceadvisor = ["mypy-boto3-iotdeviceadvisor (==1.17.102.post1)"] +iotevents = ["mypy-boto3-iotevents (==1.17.102.post1)"] +iotevents-data = ["mypy-boto3-iotevents-data (==1.17.102.post1)"] +iotfleethub = ["mypy-boto3-iotfleethub (==1.17.102.post1)"] +iotsecuretunneling = ["mypy-boto3-iotsecuretunneling (==1.17.102.post1)"] +iotsitewise = ["mypy-boto3-iotsitewise (==1.17.102.post1)"] +iotthingsgraph = ["mypy-boto3-iotthingsgraph (==1.17.102.post1)"] +iotwireless = ["mypy-boto3-iotwireless (==1.17.102.post1)"] +ivs = ["mypy-boto3-ivs (==1.17.102.post1)"] +kafka = ["mypy-boto3-kafka (==1.17.102.post1)"] +kendra = ["mypy-boto3-kendra (==1.17.102.post1)"] +kinesis = ["mypy-boto3-kinesis (==1.17.102.post1)"] +kinesis-video-archived-media = ["mypy-boto3-kinesis-video-archived-media (==1.17.102.post1)"] +kinesis-video-media = ["mypy-boto3-kinesis-video-media (==1.17.102.post1)"] +kinesis-video-signaling = ["mypy-boto3-kinesis-video-signaling (==1.17.102.post1)"] +kinesisanalytics = ["mypy-boto3-kinesisanalytics (==1.17.102.post1)"] +kinesisanalyticsv2 = ["mypy-boto3-kinesisanalyticsv2 (==1.17.102.post1)"] +kinesisvideo = ["mypy-boto3-kinesisvideo (==1.17.102.post1)"] +kms = ["mypy-boto3-kms (==1.17.102.post1)"] +lakeformation = ["mypy-boto3-lakeformation (==1.17.102.post1)"] +lambda = ["mypy-boto3-lambda (==1.17.102.post1)"] +lex-models = ["mypy-boto3-lex-models (==1.17.102.post1)"] +lex-runtime = ["mypy-boto3-lex-runtime (==1.17.102.post1)"] +lexv2-models = ["mypy-boto3-lexv2-models (==1.17.102.post1)"] +lexv2-runtime = ["mypy-boto3-lexv2-runtime (==1.17.102.post1)"] +license-manager = ["mypy-boto3-license-manager (==1.17.102.post1)"] +lightsail = ["mypy-boto3-lightsail (==1.17.102.post1)"] +location = ["mypy-boto3-location (==1.17.102.post1)"] +logs = ["mypy-boto3-logs (==1.17.102.post1)"] +lookoutequipment = ["mypy-boto3-lookoutequipment (==1.17.102.post1)"] +lookoutmetrics = ["mypy-boto3-lookoutmetrics (==1.17.102.post1)"] +lookoutvision = ["mypy-boto3-lookoutvision (==1.17.102.post1)"] +machinelearning = ["mypy-boto3-machinelearning (==1.17.102.post1)"] +macie = ["mypy-boto3-macie (==1.17.102.post1)"] +macie2 = ["mypy-boto3-macie2 (==1.17.102.post1)"] +managedblockchain = ["mypy-boto3-managedblockchain (==1.17.102.post1)"] +marketplace-catalog = ["mypy-boto3-marketplace-catalog (==1.17.102.post1)"] +marketplace-entitlement = ["mypy-boto3-marketplace-entitlement (==1.17.102.post1)"] +marketplacecommerceanalytics = ["mypy-boto3-marketplacecommerceanalytics (==1.17.102.post1)"] +mediaconnect = ["mypy-boto3-mediaconnect (==1.17.102.post1)"] +mediaconvert = ["mypy-boto3-mediaconvert (==1.17.102.post1)"] +medialive = ["mypy-boto3-medialive (==1.17.102.post1)"] +mediapackage = ["mypy-boto3-mediapackage (==1.17.102.post1)"] +mediapackage-vod = ["mypy-boto3-mediapackage-vod (==1.17.102.post1)"] +mediastore = ["mypy-boto3-mediastore (==1.17.102.post1)"] +mediastore-data = ["mypy-boto3-mediastore-data (==1.17.102.post1)"] +mediatailor = ["mypy-boto3-mediatailor (==1.17.102.post1)"] +meteringmarketplace = ["mypy-boto3-meteringmarketplace (==1.17.102.post1)"] +mgh = ["mypy-boto3-mgh (==1.17.102.post1)"] +mgn = ["mypy-boto3-mgn (==1.17.102.post1)"] +migrationhub-config = ["mypy-boto3-migrationhub-config (==1.17.102.post1)"] +mobile = ["mypy-boto3-mobile (==1.17.102.post1)"] +mq = ["mypy-boto3-mq (==1.17.102.post1)"] +mturk = ["mypy-boto3-mturk (==1.17.102.post1)"] +mwaa = ["mypy-boto3-mwaa (==1.17.102.post1)"] +neptune = ["mypy-boto3-neptune (==1.17.102.post1)"] +network-firewall = ["mypy-boto3-network-firewall (==1.17.102.post1)"] +networkmanager = ["mypy-boto3-networkmanager (==1.17.102.post1)"] +nimble = ["mypy-boto3-nimble (==1.17.102.post1)"] +opsworks = ["mypy-boto3-opsworks (==1.17.102.post1)"] +opsworkscm = ["mypy-boto3-opsworkscm (==1.17.102.post1)"] +organizations = ["mypy-boto3-organizations (==1.17.102.post1)"] +outposts = ["mypy-boto3-outposts (==1.17.102.post1)"] +personalize = ["mypy-boto3-personalize (==1.17.102.post1)"] +personalize-events = ["mypy-boto3-personalize-events (==1.17.102.post1)"] +personalize-runtime = ["mypy-boto3-personalize-runtime (==1.17.102.post1)"] +pi = ["mypy-boto3-pi (==1.17.102.post1)"] +pinpoint = ["mypy-boto3-pinpoint (==1.17.102.post1)"] +pinpoint-email = ["mypy-boto3-pinpoint-email (==1.17.102.post1)"] +pinpoint-sms-voice = ["mypy-boto3-pinpoint-sms-voice (==1.17.102.post1)"] +polly = ["mypy-boto3-polly (==1.17.102.post1)"] +pricing = ["mypy-boto3-pricing (==1.17.102.post1)"] +proton = ["mypy-boto3-proton (==1.17.102.post1)"] +qldb = ["mypy-boto3-qldb (==1.17.102.post1)"] +qldb-session = ["mypy-boto3-qldb-session (==1.17.102.post1)"] +quicksight = ["mypy-boto3-quicksight (==1.17.102.post1)"] +ram = ["mypy-boto3-ram (==1.17.102.post1)"] +rds = ["mypy-boto3-rds (==1.17.102.post1)"] +rds-data = ["mypy-boto3-rds-data (==1.17.102.post1)"] +redshift = ["mypy-boto3-redshift (==1.17.102.post1)"] +redshift-data = ["mypy-boto3-redshift-data (==1.17.102.post1)"] +rekognition = ["mypy-boto3-rekognition (==1.17.102.post1)"] +resource-groups = ["mypy-boto3-resource-groups (==1.17.102.post1)"] +resourcegroupstaggingapi = ["mypy-boto3-resourcegroupstaggingapi (==1.17.102.post1)"] +robomaker = ["mypy-boto3-robomaker (==1.17.102.post1)"] +route53 = ["mypy-boto3-route53 (==1.17.102.post1)"] +route53domains = ["mypy-boto3-route53domains (==1.17.102.post1)"] +route53resolver = ["mypy-boto3-route53resolver (==1.17.102.post1)"] +s3 = ["mypy-boto3-s3 (==1.17.102.post1)"] +s3control = ["mypy-boto3-s3control (==1.17.102.post1)"] +s3outposts = ["mypy-boto3-s3outposts (==1.17.102.post1)"] +sagemaker = ["mypy-boto3-sagemaker (==1.17.102.post1)"] +sagemaker-a2i-runtime = ["mypy-boto3-sagemaker-a2i-runtime (==1.17.102.post1)"] +sagemaker-edge = ["mypy-boto3-sagemaker-edge (==1.17.102.post1)"] +sagemaker-featurestore-runtime = ["mypy-boto3-sagemaker-featurestore-runtime (==1.17.102.post1)"] +sagemaker-runtime = ["mypy-boto3-sagemaker-runtime (==1.17.102.post1)"] +savingsplans = ["mypy-boto3-savingsplans (==1.17.102.post1)"] +schemas = ["mypy-boto3-schemas (==1.17.102.post1)"] +sdb = ["mypy-boto3-sdb (==1.17.102.post1)"] +secretsmanager = ["mypy-boto3-secretsmanager (==1.17.102.post1)"] +securityhub = ["mypy-boto3-securityhub (==1.17.102.post1)"] +serverlessrepo = ["mypy-boto3-serverlessrepo (==1.17.102.post1)"] +service-quotas = ["mypy-boto3-service-quotas (==1.17.102.post1)"] +servicecatalog = ["mypy-boto3-servicecatalog (==1.17.102.post1)"] +servicecatalog-appregistry = ["mypy-boto3-servicecatalog-appregistry (==1.17.102.post1)"] +servicediscovery = ["mypy-boto3-servicediscovery (==1.17.102.post1)"] +ses = ["mypy-boto3-ses (==1.17.102.post1)"] +sesv2 = ["mypy-boto3-sesv2 (==1.17.102.post1)"] +shield = ["mypy-boto3-shield (==1.17.102.post1)"] +signer = ["mypy-boto3-signer (==1.17.102.post1)"] +sms = ["mypy-boto3-sms (==1.17.102.post1)"] +sms-voice = ["mypy-boto3-sms-voice (==1.17.102.post1)"] +snowball = ["mypy-boto3-snowball (==1.17.102.post1)"] +sns = ["mypy-boto3-sns (==1.17.102.post1)"] +sqs = ["mypy-boto3-sqs (==1.17.102.post1)"] +ssm = ["mypy-boto3-ssm (==1.17.102.post1)"] +ssm-contacts = ["mypy-boto3-ssm-contacts (==1.17.102.post1)"] +ssm-incidents = ["mypy-boto3-ssm-incidents (==1.17.102.post1)"] +sso = ["mypy-boto3-sso (==1.17.102.post1)"] +sso-admin = ["mypy-boto3-sso-admin (==1.17.102.post1)"] +sso-oidc = ["mypy-boto3-sso-oidc (==1.17.102.post1)"] +stepfunctions = ["mypy-boto3-stepfunctions (==1.17.102.post1)"] +storagegateway = ["mypy-boto3-storagegateway (==1.17.102.post1)"] +sts = ["mypy-boto3-sts (==1.17.102.post1)"] +support = ["mypy-boto3-support (==1.17.102.post1)"] +swf = ["mypy-boto3-swf (==1.17.102.post1)"] +synthetics = ["mypy-boto3-synthetics (==1.17.102.post1)"] +textract = ["mypy-boto3-textract (==1.17.102.post1)"] +timestream-query = ["mypy-boto3-timestream-query (==1.17.102.post1)"] +timestream-write = ["mypy-boto3-timestream-write (==1.17.102.post1)"] +transcribe = ["mypy-boto3-transcribe (==1.17.102.post1)"] +transfer = ["mypy-boto3-transfer (==1.17.102.post1)"] +translate = ["mypy-boto3-translate (==1.17.102.post1)"] +waf = ["mypy-boto3-waf (==1.17.102.post1)"] +waf-regional = ["mypy-boto3-waf-regional (==1.17.102.post1)"] +wafv2 = ["mypy-boto3-wafv2 (==1.17.102.post1)"] +wellarchitected = ["mypy-boto3-wellarchitected (==1.17.102.post1)"] +workdocs = ["mypy-boto3-workdocs (==1.17.102.post1)"] +worklink = ["mypy-boto3-worklink (==1.17.102.post1)"] +workmail = ["mypy-boto3-workmail (==1.17.102.post1)"] +workmailmessageflow = ["mypy-boto3-workmailmessageflow (==1.17.102.post1)"] +workspaces = ["mypy-boto3-workspaces (==1.17.102.post1)"] +xray = ["mypy-boto3-xray (==1.17.102.post1)"] [[package]] name = "botocore" -version = "1.20.99" +version = "1.20.102" description = "Low-level, data-driven core of boto 3." category = "main" optional = false @@ -408,8 +408,8 @@ crt = ["awscrt (==0.11.24)"] [[package]] name = "botocore-stubs" -version = "1.20.99" -description = "Type annotations for botocore 1.20.99, generated by mypy-boto3-buider 4.18.4" +version = "1.20.102.post1" +description = "Type annotations for botocore 1.20.102, generated by mypy-boto3-buider 4.21.0" category = "dev" optional = false python-versions = ">=3.6" @@ -550,7 +550,7 @@ python-versions = "*" [[package]] name = "dcicsnovault" -version = "4.8.0.0b2" +version = "4.8.1b0" description = "Storage support for 4DN Data Portals." category = "main" optional = false @@ -866,7 +866,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "importlib-metadata" -version = "4.5.0" +version = "4.6.0" description = "Read metadata from Python packages" category = "dev" optional = false @@ -878,11 +878,12 @@ zipp = ">=0.5" [package.extras] docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] +perf = ["ipython"] +testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] [[package]] name = "importlib-resources" -version = "5.1.4" +version = "5.2.0" description = "Read resources from Python packages" category = "main" optional = false @@ -1920,7 +1921,7 @@ python-versions = "*" [[package]] name = "urllib3" -version = "1.26.5" +version = "1.26.6" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false @@ -2108,7 +2109,7 @@ test = ["zope.testing"] [metadata] lock-version = "1.1" python-versions = ">=3.6.1,<3.7" -content-hash = "8eccb570d4ce85e64a03653a48c06b436fb4aab207da54f7304ae399bdfe533d" +content-hash = "3678ab66d10291bab7f178d357aa02e8de3864e66daa9501c33953ebc98e72de" [metadata.files] apipkg = [ @@ -2152,16 +2153,16 @@ boto3 = [ {file = "boto3-1.17.53.tar.gz", hash = "sha256:1d26f6e7ae3c940cb07119077ac42485dcf99164350da0ab50d0f5ad345800cd"}, ] boto3-stubs = [ - {file = "boto3-stubs-1.17.99.tar.gz", hash = "sha256:17f32726e1f1b318b3a9ba6683a4dbd5873b6e143eea632c66042d7586b5911f"}, - {file = "boto3_stubs-1.17.99-py3-none-any.whl", hash = "sha256:f69706063ad49fade04d63d4e2e8b1b750fb1066de2281a7b994cd0003a5680c"}, + {file = "boto3-stubs-1.17.102.post1.tar.gz", hash = "sha256:938936682238359d419c4936d81a696ed225ac42d46f564bd5f213aabb106af3"}, + {file = "boto3_stubs-1.17.102.post1-py3-none-any.whl", hash = "sha256:27c8836a06d6c20d9e992f7e6dbc50b340fa5b80849b67de484226b0de0ca69d"}, ] botocore = [ - {file = "botocore-1.20.99-py2.py3-none-any.whl", hash = "sha256:a236bb890e2b25f0db1b9bb4dd49e2d825b051ba953830c7cd7be7200f5aecbf"}, - {file = "botocore-1.20.99.tar.gz", hash = "sha256:683c7cc7d01c94a6e593694d1d7bcdd3ea5f59c00421fa7e34500458175b9346"}, + {file = "botocore-1.20.102-py2.py3-none-any.whl", hash = "sha256:bdf08a4f7f01ead00d386848f089c08270499711447569c18d0db60023619c06"}, + {file = "botocore-1.20.102.tar.gz", hash = "sha256:2f57f7ceed1598d96cc497aeb45317db5d3b21a5aafea4732d0e561d0fc2a8fa"}, ] botocore-stubs = [ - {file = "botocore-stubs-1.20.99.tar.gz", hash = "sha256:cce3a22449ada550d621c7350636c5b6730033857f250c14f633e52ea5e6e50e"}, - {file = "botocore_stubs-1.20.99-py3-none-any.whl", hash = "sha256:f9a6e1977ab0b6a651cdb956594192fd7d465757825823bba2109b14292223d6"}, + {file = "botocore-stubs-1.20.102.post1.tar.gz", hash = "sha256:aa0ee90660ff4dae63402fd3924ca9bfff5f8ab30601db60058e671216e30705"}, + {file = "botocore_stubs-1.20.102.post1-py3-none-any.whl", hash = "sha256:9fbb89f5053d20ffafb48d5c29365b5097a6f63a5f72ca8bea47ab020b15c637"}, ] cached-property = [ {file = "cached-property-1.5.2.tar.gz", hash = "sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130"}, @@ -2317,8 +2318,8 @@ dcicpyvcf = [ {file = "dcicpyvcf-1.0.0.tar.gz", hash = "sha256:c5bf8d585002ab3b95d13a47803376b456b931865e4189c38a18cca47b108449"}, ] dcicsnovault = [ - {file = "dcicsnovault-4.8.0.0b2-py3-none-any.whl", hash = "sha256:5bf9d1000a53b250d2218ed31dfb19f0d58cb5d2de81c6baa517f03dc2803805"}, - {file = "dcicsnovault-4.8.0.0b2.tar.gz", hash = "sha256:4dcde76ab8cb82469ca3a2a4eae9476ac04396faf370e124a3114f63378cf48a"}, + {file = "dcicsnovault-4.8.1b0-py3-none-any.whl", hash = "sha256:2d7707f40e69e9b1684b65ee6b1889d61f0beae8771bc1facc56b3be68a38d21"}, + {file = "dcicsnovault-4.8.1b0.tar.gz", hash = "sha256:3e071b4f7cf6df579db8f3d1d5a7d9c5cd10d5bf0630cd6cefd65b823a495fb1"}, ] dcicutils = [ {file = "dcicutils-1.18.1.2b2-py3-none-any.whl", hash = "sha256:906c126d5d8a205e8c5ee6fc3915e888f1e0d7ea5f36677366507f3841f18680"}, @@ -2415,12 +2416,12 @@ idna = [ {file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"}, ] importlib-metadata = [ - {file = "importlib_metadata-4.5.0-py3-none-any.whl", hash = "sha256:833b26fb89d5de469b24a390e9df088d4e52e4ba33b01dc5e0e4f41b81a16c00"}, - {file = "importlib_metadata-4.5.0.tar.gz", hash = "sha256:b142cc1dd1342f31ff04bb7d022492b09920cb64fed867cd3ea6f80fe3ebd139"}, + {file = "importlib_metadata-4.6.0-py3-none-any.whl", hash = "sha256:c6513572926a96458f8c8f725bf0e00108fba0c9583ade9bd15b869c9d726e33"}, + {file = "importlib_metadata-4.6.0.tar.gz", hash = "sha256:4a5611fea3768d3d967c447ab4e93f567d95db92225b43b7b238dbfb855d70bb"}, ] importlib-resources = [ - {file = "importlib_resources-5.1.4-py3-none-any.whl", hash = "sha256:e962bff7440364183203d179d7ae9ad90cb1f2b74dcb84300e88ecc42dca3351"}, - {file = "importlib_resources-5.1.4.tar.gz", hash = "sha256:54161657e8ffc76596c4ede7080ca68cb02962a2e074a2586b695a93a925d36e"}, + {file = "importlib_resources-5.2.0-py3-none-any.whl", hash = "sha256:a0143290bef3cbc99de9e40176e4987780939a955b8632f02ce6c935f42e9bfc"}, + {file = "importlib_resources-5.2.0.tar.gz", hash = "sha256:22a2c42d8c6a1d30aa8a0e1f57293725bfd5c013d562585e46aff469e0ff78b3"}, ] isodate = [ {file = "isodate-0.5.4.tar.gz", hash = "sha256:42105c41d037246dc1987e36d96f3752ffd5c0c24834dd12e4fdbe1e79544e31"}, @@ -3041,8 +3042,8 @@ uptime = [ {file = "uptime-3.0.1.tar.gz", hash = "sha256:7c300254775b807ce46e3dcbcda30aa3b9a204b9c57a7ac1e79ee6dbe3942973"}, ] urllib3 = [ - {file = "urllib3-1.26.5-py2.py3-none-any.whl", hash = "sha256:753a0374df26658f99d826cfe40394a686d05985786d946fbe4165b5148f5a7c"}, - {file = "urllib3-1.26.5.tar.gz", hash = "sha256:a7acd0977125325f516bda9735fa7142b909a8d01e8b2e4c8108d0984e6e0098"}, + {file = "urllib3-1.26.6-py2.py3-none-any.whl", hash = "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4"}, + {file = "urllib3-1.26.6.tar.gz", hash = "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f"}, ] venusian = [ {file = "venusian-1.2.0-py2.py3-none-any.whl", hash = "sha256:2f2d077a1eedc3fda40425f65687c8c494da7e83d7c23bc2c4d1a40eb3ca5b6d"}, diff --git a/pyproject.toml b/pyproject.toml index 586cec5c87..9711b0423d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,12 +37,12 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.6.1,<3.7" boto3 = "^1.17.32" -botocore = "^1.20.32" +botocore = "^1.20.86" certifi = ">=2020.11.8" chardet = "3.0.4" colorama = "0.3.3" dcicpyvcf = "1.0.0" -dcicsnovault = "4.8.0.0b2" +dcicsnovault = "4.8.1b0" dcicutils = "1.18.1.2b2" elasticsearch = "6.8.1" execnet = "1.4.1" From c1f7220803be1b6a29027503b57f5002d3a8d448 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Tue, 29 Jun 2021 15:50:52 -0400 Subject: [PATCH 117/120] use special key for federation token --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- src/encoded/types/file.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/poetry.lock b/poetry.lock index 3ce70693db..0788e37a40 100644 --- a/poetry.lock +++ b/poetry.lock @@ -605,7 +605,7 @@ xlrd = ">=1.0.0,<2.0.0" [[package]] name = "dcicutils" -version = "1.18.1.2b2" +version = "1.19.0" description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources" category = "main" optional = false @@ -2109,7 +2109,7 @@ test = ["zope.testing"] [metadata] lock-version = "1.1" python-versions = ">=3.6.1,<3.7" -content-hash = "3678ab66d10291bab7f178d357aa02e8de3864e66daa9501c33953ebc98e72de" +content-hash = "daeb56da7c345facf5873af124b87356700a8f3641fbcab72913afabb3b93b23" [metadata.files] apipkg = [ @@ -2322,8 +2322,8 @@ dcicsnovault = [ {file = "dcicsnovault-4.8.1b0.tar.gz", hash = "sha256:3e071b4f7cf6df579db8f3d1d5a7d9c5cd10d5bf0630cd6cefd65b823a495fb1"}, ] dcicutils = [ - {file = "dcicutils-1.18.1.2b2-py3-none-any.whl", hash = "sha256:906c126d5d8a205e8c5ee6fc3915e888f1e0d7ea5f36677366507f3841f18680"}, - {file = "dcicutils-1.18.1.2b2.tar.gz", hash = "sha256:d557c6a121daed98973f9ac43d927bf487edd6c52525fa4a362285e5e8abc964"}, + {file = "dcicutils-1.19.0-py3-none-any.whl", hash = "sha256:af6e2b13ea92857140fe98a18a06bfe41e7fdd1c4c065e0f74368aed5b2983c4"}, + {file = "dcicutils-1.19.0.tar.gz", hash = "sha256:883bbaa442a6c410fe1278cc39d2b0b0d808be2e14cb55e9adc1751e3bd232f1"}, ] docker = [ {file = "docker-4.4.4-py2.py3-none-any.whl", hash = "sha256:f3607d5695be025fa405a12aca2e5df702a57db63790c73b927eb6a94aac60af"}, diff --git a/pyproject.toml b/pyproject.toml index 9711b0423d..a943754c20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,7 @@ chardet = "3.0.4" colorama = "0.3.3" dcicpyvcf = "1.0.0" dcicsnovault = "4.8.1b0" -dcicutils = "1.18.1.2b2" +dcicutils = "^1.19.0" elasticsearch = "6.8.1" execnet = "1.4.1" future = "^0.15.2" diff --git a/src/encoded/types/file.py b/src/encoded/types/file.py index e75212677d..748688ca36 100644 --- a/src/encoded/types/file.py +++ b/src/encoded/types/file.py @@ -111,8 +111,8 @@ def external_creds(bucket, key, name=None, profile_name=None): ] } # boto.set_stream_logger('boto3') - conn = boto3.client('sts', aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID"), - aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY")) + conn = boto3.client('sts', aws_access_key_id=os.environ.get("S3_AWS_ACCESS_KEY_ID"), + aws_secret_access_key=os.environ.get("S3_AWS_SECRET_ACCESS_KEY")) token = conn.get_federation_token(Name=name, Policy=json.dumps(policy)) # 'access_key' 'secret_key' 'expiration' 'session_token' credentials = token.get('Credentials') From 63bfdbc21c534440bc9d37a8139ca018367ace7d Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Wed, 30 Jun 2021 12:59:51 -0400 Subject: [PATCH 118/120] grab S3 access keys from global application configuration --- deploy/docker/production/assume_identity.py | 1 - src/encoded/types/file.py | 16 ++++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/deploy/docker/production/assume_identity.py b/deploy/docker/production/assume_identity.py index 239175b3a1..e43ebd992e 100644 --- a/deploy/docker/production/assume_identity.py +++ b/deploy/docker/production/assume_identity.py @@ -6,7 +6,6 @@ import os import logging -import subprocess from dcicutils.qa_utils import override_environ from dcicutils.deployment_utils import IniFileManager from dcicutils.secrets_utils import assume_identity diff --git a/src/encoded/types/file.py b/src/encoded/types/file.py index 748688ca36..6ac0a45497 100644 --- a/src/encoded/types/file.py +++ b/src/encoded/types/file.py @@ -19,7 +19,8 @@ from pyramid.threadlocal import get_current_request from pyramid.traversal import resource_path from pyramid.view import view_config -from dcicutils.env_utils import CGAP_ENV_WEBPROD, CGAP_ENV_WOLF +from dcicutils.secrets_utils import assume_identity +from dcicutils.qa_utils import override_environ from snovault import ( AfterModified, BeforeModified, @@ -110,9 +111,16 @@ def external_creds(bucket, key, name=None, profile_name=None): } ] } - # boto.set_stream_logger('boto3') - conn = boto3.client('sts', aws_access_key_id=os.environ.get("S3_AWS_ACCESS_KEY_ID"), - aws_secret_access_key=os.environ.get("S3_AWS_SECRET_ACCESS_KEY")) + # In the new environment, extract S3 Keys from global application configuration + if 'IDENTITY' in os.environ: + identity = assume_identity() + with override_environ(**identity): + conn = boto3.client('sts', aws_access_key_id=os.environ.get('S3_AWS_ACCESS_KEY_ID'), + aws_secret_access_key=os.environ.get('S3_AWS_SECRET_ACCESS_KEY')) + # In the old account, we are always passing IAM User creds so these will just work + else: + conn = boto3.client('sts', aws_access_key_id=os.environ.get('AWS_ACCESS_KEY_ID'), + aws_secret_access_key=os.environ.get('AWS_SECRET_ACCESS_KEY')) token = conn.get_federation_token(Name=name, Policy=json.dumps(policy)) # 'access_key' 'secret_key' 'expiration' 'session_token' credentials = token.get('Credentials') From e5156aef31ad09b8cfc2b940967c824491100e05 Mon Sep 17 00:00:00 2001 From: William Ronchetti Date: Thu, 1 Jul 2021 13:16:23 -0400 Subject: [PATCH 119/120] bring in snovault --- poetry.lock | 574 ++++++++++++++++++++++++------------------------- pyproject.toml | 2 +- 2 files changed, 288 insertions(+), 288 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0788e37a40..023d6345c8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -106,8 +106,8 @@ s3transfer = ">=0.3.0,<0.4.0" [[package]] name = "boto3-stubs" -version = "1.17.102.post1" -description = "Type annotations for boto3 1.17.102, generated by mypy-boto3-buider 4.21.0" +version = "1.17.103.post1" +description = "Type annotations for boto3 1.17.103, generated by mypy-boto3-buider 4.22.1" category = "dev" optional = false python-versions = ">=3.6" @@ -117,282 +117,282 @@ botocore-stubs = "*" typing-extensions = {version = "*", markers = "python_version < \"3.8\""} [package.extras] -accessanalyzer = ["mypy-boto3-accessanalyzer (==1.17.102.post1)"] -acm = ["mypy-boto3-acm (==1.17.102.post1)"] -acm-pca = ["mypy-boto3-acm-pca (==1.17.102.post1)"] -alexaforbusiness = ["mypy-boto3-alexaforbusiness (==1.17.102.post1)"] -all = ["mypy-boto3-accessanalyzer (==1.17.102.post1)", "mypy-boto3-acm (==1.17.102.post1)", "mypy-boto3-acm-pca (==1.17.102.post1)", "mypy-boto3-alexaforbusiness (==1.17.102.post1)", "mypy-boto3-amp (==1.17.102.post1)", "mypy-boto3-amplify (==1.17.102.post1)", "mypy-boto3-amplifybackend (==1.17.102.post1)", "mypy-boto3-apigateway (==1.17.102.post1)", "mypy-boto3-apigatewaymanagementapi (==1.17.102.post1)", "mypy-boto3-apigatewayv2 (==1.17.102.post1)", "mypy-boto3-appconfig (==1.17.102.post1)", "mypy-boto3-appflow (==1.17.102.post1)", "mypy-boto3-appintegrations (==1.17.102.post1)", "mypy-boto3-application-autoscaling (==1.17.102.post1)", "mypy-boto3-application-insights (==1.17.102.post1)", "mypy-boto3-applicationcostprofiler (==1.17.102.post1)", "mypy-boto3-appmesh (==1.17.102.post1)", "mypy-boto3-apprunner (==1.17.102.post1)", "mypy-boto3-appstream (==1.17.102.post1)", "mypy-boto3-appsync (==1.17.102.post1)", "mypy-boto3-athena (==1.17.102.post1)", "mypy-boto3-auditmanager (==1.17.102.post1)", "mypy-boto3-autoscaling (==1.17.102.post1)", "mypy-boto3-autoscaling-plans (==1.17.102.post1)", "mypy-boto3-backup (==1.17.102.post1)", "mypy-boto3-batch (==1.17.102.post1)", "mypy-boto3-braket (==1.17.102.post1)", "mypy-boto3-budgets (==1.17.102.post1)", "mypy-boto3-ce (==1.17.102.post1)", "mypy-boto3-chime (==1.17.102.post1)", "mypy-boto3-cloud9 (==1.17.102.post1)", "mypy-boto3-clouddirectory (==1.17.102.post1)", "mypy-boto3-cloudformation (==1.17.102.post1)", "mypy-boto3-cloudfront (==1.17.102.post1)", "mypy-boto3-cloudhsm (==1.17.102.post1)", "mypy-boto3-cloudhsmv2 (==1.17.102.post1)", "mypy-boto3-cloudsearch (==1.17.102.post1)", "mypy-boto3-cloudsearchdomain (==1.17.102.post1)", "mypy-boto3-cloudtrail (==1.17.102.post1)", "mypy-boto3-cloudwatch (==1.17.102.post1)", "mypy-boto3-codeartifact (==1.17.102.post1)", "mypy-boto3-codebuild (==1.17.102.post1)", "mypy-boto3-codecommit (==1.17.102.post1)", "mypy-boto3-codedeploy (==1.17.102.post1)", "mypy-boto3-codeguru-reviewer (==1.17.102.post1)", "mypy-boto3-codeguruprofiler (==1.17.102.post1)", "mypy-boto3-codepipeline (==1.17.102.post1)", "mypy-boto3-codestar (==1.17.102.post1)", "mypy-boto3-codestar-connections (==1.17.102.post1)", "mypy-boto3-codestar-notifications (==1.17.102.post1)", "mypy-boto3-cognito-identity (==1.17.102.post1)", "mypy-boto3-cognito-idp (==1.17.102.post1)", "mypy-boto3-cognito-sync (==1.17.102.post1)", "mypy-boto3-comprehend (==1.17.102.post1)", "mypy-boto3-comprehendmedical (==1.17.102.post1)", "mypy-boto3-compute-optimizer (==1.17.102.post1)", "mypy-boto3-config (==1.17.102.post1)", "mypy-boto3-connect (==1.17.102.post1)", "mypy-boto3-connect-contact-lens (==1.17.102.post1)", "mypy-boto3-connectparticipant (==1.17.102.post1)", "mypy-boto3-cur (==1.17.102.post1)", "mypy-boto3-customer-profiles (==1.17.102.post1)", "mypy-boto3-databrew (==1.17.102.post1)", "mypy-boto3-dataexchange (==1.17.102.post1)", "mypy-boto3-datapipeline (==1.17.102.post1)", "mypy-boto3-datasync (==1.17.102.post1)", "mypy-boto3-dax (==1.17.102.post1)", "mypy-boto3-detective (==1.17.102.post1)", "mypy-boto3-devicefarm (==1.17.102.post1)", "mypy-boto3-devops-guru (==1.17.102.post1)", "mypy-boto3-directconnect (==1.17.102.post1)", "mypy-boto3-discovery (==1.17.102.post1)", "mypy-boto3-dlm (==1.17.102.post1)", "mypy-boto3-dms (==1.17.102.post1)", "mypy-boto3-docdb (==1.17.102.post1)", "mypy-boto3-ds (==1.17.102.post1)", "mypy-boto3-dynamodb (==1.17.102.post1)", "mypy-boto3-dynamodbstreams (==1.17.102.post1)", "mypy-boto3-ebs (==1.17.102.post1)", "mypy-boto3-ec2 (==1.17.102.post1)", "mypy-boto3-ec2-instance-connect (==1.17.102.post1)", "mypy-boto3-ecr (==1.17.102.post1)", "mypy-boto3-ecr-public (==1.17.102.post1)", "mypy-boto3-ecs (==1.17.102.post1)", "mypy-boto3-efs (==1.17.102.post1)", "mypy-boto3-eks (==1.17.102.post1)", "mypy-boto3-elastic-inference (==1.17.102.post1)", "mypy-boto3-elasticache (==1.17.102.post1)", "mypy-boto3-elasticbeanstalk (==1.17.102.post1)", "mypy-boto3-elastictranscoder (==1.17.102.post1)", "mypy-boto3-elb (==1.17.102.post1)", "mypy-boto3-elbv2 (==1.17.102.post1)", "mypy-boto3-emr (==1.17.102.post1)", "mypy-boto3-emr-containers (==1.17.102.post1)", "mypy-boto3-es (==1.17.102.post1)", "mypy-boto3-events (==1.17.102.post1)", "mypy-boto3-finspace (==1.17.102.post1)", "mypy-boto3-finspace-data (==1.17.102.post1)", "mypy-boto3-firehose (==1.17.102.post1)", "mypy-boto3-fis (==1.17.102.post1)", "mypy-boto3-fms (==1.17.102.post1)", "mypy-boto3-forecast (==1.17.102.post1)", "mypy-boto3-forecastquery (==1.17.102.post1)", "mypy-boto3-frauddetector (==1.17.102.post1)", "mypy-boto3-fsx (==1.17.102.post1)", "mypy-boto3-gamelift (==1.17.102.post1)", "mypy-boto3-glacier (==1.17.102.post1)", "mypy-boto3-globalaccelerator (==1.17.102.post1)", "mypy-boto3-glue (==1.17.102.post1)", "mypy-boto3-greengrass (==1.17.102.post1)", "mypy-boto3-greengrassv2 (==1.17.102.post1)", "mypy-boto3-groundstation (==1.17.102.post1)", "mypy-boto3-guardduty (==1.17.102.post1)", "mypy-boto3-health (==1.17.102.post1)", "mypy-boto3-healthlake (==1.17.102.post1)", "mypy-boto3-honeycode (==1.17.102.post1)", "mypy-boto3-iam (==1.17.102.post1)", "mypy-boto3-identitystore (==1.17.102.post1)", "mypy-boto3-imagebuilder (==1.17.102.post1)", "mypy-boto3-importexport (==1.17.102.post1)", "mypy-boto3-inspector (==1.17.102.post1)", "mypy-boto3-iot (==1.17.102.post1)", "mypy-boto3-iot-data (==1.17.102.post1)", "mypy-boto3-iot-jobs-data (==1.17.102.post1)", "mypy-boto3-iot1click-devices (==1.17.102.post1)", "mypy-boto3-iot1click-projects (==1.17.102.post1)", "mypy-boto3-iotanalytics (==1.17.102.post1)", "mypy-boto3-iotdeviceadvisor (==1.17.102.post1)", "mypy-boto3-iotevents (==1.17.102.post1)", "mypy-boto3-iotevents-data (==1.17.102.post1)", "mypy-boto3-iotfleethub (==1.17.102.post1)", "mypy-boto3-iotsecuretunneling (==1.17.102.post1)", "mypy-boto3-iotsitewise (==1.17.102.post1)", "mypy-boto3-iotthingsgraph (==1.17.102.post1)", "mypy-boto3-iotwireless (==1.17.102.post1)", "mypy-boto3-ivs (==1.17.102.post1)", "mypy-boto3-kafka (==1.17.102.post1)", "mypy-boto3-kendra (==1.17.102.post1)", "mypy-boto3-kinesis (==1.17.102.post1)", "mypy-boto3-kinesis-video-archived-media (==1.17.102.post1)", "mypy-boto3-kinesis-video-media (==1.17.102.post1)", "mypy-boto3-kinesis-video-signaling (==1.17.102.post1)", "mypy-boto3-kinesisanalytics (==1.17.102.post1)", "mypy-boto3-kinesisanalyticsv2 (==1.17.102.post1)", "mypy-boto3-kinesisvideo (==1.17.102.post1)", "mypy-boto3-kms (==1.17.102.post1)", "mypy-boto3-lakeformation (==1.17.102.post1)", "mypy-boto3-lambda (==1.17.102.post1)", "mypy-boto3-lex-models (==1.17.102.post1)", "mypy-boto3-lex-runtime (==1.17.102.post1)", "mypy-boto3-lexv2-models (==1.17.102.post1)", "mypy-boto3-lexv2-runtime (==1.17.102.post1)", "mypy-boto3-license-manager (==1.17.102.post1)", "mypy-boto3-lightsail (==1.17.102.post1)", "mypy-boto3-location (==1.17.102.post1)", "mypy-boto3-logs (==1.17.102.post1)", "mypy-boto3-lookoutequipment (==1.17.102.post1)", "mypy-boto3-lookoutmetrics (==1.17.102.post1)", "mypy-boto3-lookoutvision (==1.17.102.post1)", "mypy-boto3-machinelearning (==1.17.102.post1)", "mypy-boto3-macie (==1.17.102.post1)", "mypy-boto3-macie2 (==1.17.102.post1)", "mypy-boto3-managedblockchain (==1.17.102.post1)", "mypy-boto3-marketplace-catalog (==1.17.102.post1)", "mypy-boto3-marketplace-entitlement (==1.17.102.post1)", "mypy-boto3-marketplacecommerceanalytics (==1.17.102.post1)", "mypy-boto3-mediaconnect (==1.17.102.post1)", "mypy-boto3-mediaconvert (==1.17.102.post1)", "mypy-boto3-medialive (==1.17.102.post1)", "mypy-boto3-mediapackage (==1.17.102.post1)", "mypy-boto3-mediapackage-vod (==1.17.102.post1)", "mypy-boto3-mediastore (==1.17.102.post1)", "mypy-boto3-mediastore-data (==1.17.102.post1)", "mypy-boto3-mediatailor (==1.17.102.post1)", "mypy-boto3-meteringmarketplace (==1.17.102.post1)", "mypy-boto3-mgh (==1.17.102.post1)", "mypy-boto3-mgn (==1.17.102.post1)", "mypy-boto3-migrationhub-config (==1.17.102.post1)", "mypy-boto3-mobile (==1.17.102.post1)", "mypy-boto3-mq (==1.17.102.post1)", "mypy-boto3-mturk (==1.17.102.post1)", "mypy-boto3-mwaa (==1.17.102.post1)", "mypy-boto3-neptune (==1.17.102.post1)", "mypy-boto3-network-firewall (==1.17.102.post1)", "mypy-boto3-networkmanager (==1.17.102.post1)", "mypy-boto3-nimble (==1.17.102.post1)", "mypy-boto3-opsworks (==1.17.102.post1)", "mypy-boto3-opsworkscm (==1.17.102.post1)", "mypy-boto3-organizations (==1.17.102.post1)", "mypy-boto3-outposts (==1.17.102.post1)", "mypy-boto3-personalize (==1.17.102.post1)", "mypy-boto3-personalize-events (==1.17.102.post1)", "mypy-boto3-personalize-runtime (==1.17.102.post1)", "mypy-boto3-pi (==1.17.102.post1)", "mypy-boto3-pinpoint (==1.17.102.post1)", "mypy-boto3-pinpoint-email (==1.17.102.post1)", "mypy-boto3-pinpoint-sms-voice (==1.17.102.post1)", "mypy-boto3-polly (==1.17.102.post1)", "mypy-boto3-pricing (==1.17.102.post1)", "mypy-boto3-proton (==1.17.102.post1)", "mypy-boto3-qldb (==1.17.102.post1)", "mypy-boto3-qldb-session (==1.17.102.post1)", "mypy-boto3-quicksight (==1.17.102.post1)", "mypy-boto3-ram (==1.17.102.post1)", "mypy-boto3-rds (==1.17.102.post1)", "mypy-boto3-rds-data (==1.17.102.post1)", "mypy-boto3-redshift (==1.17.102.post1)", "mypy-boto3-redshift-data (==1.17.102.post1)", "mypy-boto3-rekognition (==1.17.102.post1)", "mypy-boto3-resource-groups (==1.17.102.post1)", "mypy-boto3-resourcegroupstaggingapi (==1.17.102.post1)", "mypy-boto3-robomaker (==1.17.102.post1)", "mypy-boto3-route53 (==1.17.102.post1)", "mypy-boto3-route53domains (==1.17.102.post1)", "mypy-boto3-route53resolver (==1.17.102.post1)", "mypy-boto3-s3 (==1.17.102.post1)", "mypy-boto3-s3control (==1.17.102.post1)", "mypy-boto3-s3outposts (==1.17.102.post1)", "mypy-boto3-sagemaker (==1.17.102.post1)", "mypy-boto3-sagemaker-a2i-runtime (==1.17.102.post1)", "mypy-boto3-sagemaker-edge (==1.17.102.post1)", "mypy-boto3-sagemaker-featurestore-runtime (==1.17.102.post1)", "mypy-boto3-sagemaker-runtime (==1.17.102.post1)", "mypy-boto3-savingsplans (==1.17.102.post1)", "mypy-boto3-schemas (==1.17.102.post1)", "mypy-boto3-sdb (==1.17.102.post1)", "mypy-boto3-secretsmanager (==1.17.102.post1)", "mypy-boto3-securityhub (==1.17.102.post1)", "mypy-boto3-serverlessrepo (==1.17.102.post1)", "mypy-boto3-service-quotas (==1.17.102.post1)", "mypy-boto3-servicecatalog (==1.17.102.post1)", "mypy-boto3-servicecatalog-appregistry (==1.17.102.post1)", "mypy-boto3-servicediscovery (==1.17.102.post1)", "mypy-boto3-ses (==1.17.102.post1)", "mypy-boto3-sesv2 (==1.17.102.post1)", "mypy-boto3-shield (==1.17.102.post1)", "mypy-boto3-signer (==1.17.102.post1)", "mypy-boto3-sms (==1.17.102.post1)", "mypy-boto3-sms-voice (==1.17.102.post1)", "mypy-boto3-snowball (==1.17.102.post1)", "mypy-boto3-sns (==1.17.102.post1)", "mypy-boto3-sqs (==1.17.102.post1)", "mypy-boto3-ssm (==1.17.102.post1)", "mypy-boto3-ssm-contacts (==1.17.102.post1)", "mypy-boto3-ssm-incidents (==1.17.102.post1)", "mypy-boto3-sso (==1.17.102.post1)", "mypy-boto3-sso-admin (==1.17.102.post1)", "mypy-boto3-sso-oidc (==1.17.102.post1)", "mypy-boto3-stepfunctions (==1.17.102.post1)", "mypy-boto3-storagegateway (==1.17.102.post1)", "mypy-boto3-sts (==1.17.102.post1)", "mypy-boto3-support (==1.17.102.post1)", "mypy-boto3-swf (==1.17.102.post1)", "mypy-boto3-synthetics (==1.17.102.post1)", "mypy-boto3-textract (==1.17.102.post1)", "mypy-boto3-timestream-query (==1.17.102.post1)", "mypy-boto3-timestream-write (==1.17.102.post1)", "mypy-boto3-transcribe (==1.17.102.post1)", "mypy-boto3-transfer (==1.17.102.post1)", "mypy-boto3-translate (==1.17.102.post1)", "mypy-boto3-waf (==1.17.102.post1)", "mypy-boto3-waf-regional (==1.17.102.post1)", "mypy-boto3-wafv2 (==1.17.102.post1)", "mypy-boto3-wellarchitected (==1.17.102.post1)", "mypy-boto3-workdocs (==1.17.102.post1)", "mypy-boto3-worklink (==1.17.102.post1)", "mypy-boto3-workmail (==1.17.102.post1)", "mypy-boto3-workmailmessageflow (==1.17.102.post1)", "mypy-boto3-workspaces (==1.17.102.post1)", "mypy-boto3-xray (==1.17.102.post1)"] -amp = ["mypy-boto3-amp (==1.17.102.post1)"] -amplify = ["mypy-boto3-amplify (==1.17.102.post1)"] -amplifybackend = ["mypy-boto3-amplifybackend (==1.17.102.post1)"] -apigateway = ["mypy-boto3-apigateway (==1.17.102.post1)"] -apigatewaymanagementapi = ["mypy-boto3-apigatewaymanagementapi (==1.17.102.post1)"] -apigatewayv2 = ["mypy-boto3-apigatewayv2 (==1.17.102.post1)"] -appconfig = ["mypy-boto3-appconfig (==1.17.102.post1)"] -appflow = ["mypy-boto3-appflow (==1.17.102.post1)"] -appintegrations = ["mypy-boto3-appintegrations (==1.17.102.post1)"] -application-autoscaling = ["mypy-boto3-application-autoscaling (==1.17.102.post1)"] -application-insights = ["mypy-boto3-application-insights (==1.17.102.post1)"] -applicationcostprofiler = ["mypy-boto3-applicationcostprofiler (==1.17.102.post1)"] -appmesh = ["mypy-boto3-appmesh (==1.17.102.post1)"] -apprunner = ["mypy-boto3-apprunner (==1.17.102.post1)"] -appstream = ["mypy-boto3-appstream (==1.17.102.post1)"] -appsync = ["mypy-boto3-appsync (==1.17.102.post1)"] -athena = ["mypy-boto3-athena (==1.17.102.post1)"] -auditmanager = ["mypy-boto3-auditmanager (==1.17.102.post1)"] -autoscaling = ["mypy-boto3-autoscaling (==1.17.102.post1)"] -autoscaling-plans = ["mypy-boto3-autoscaling-plans (==1.17.102.post1)"] -backup = ["mypy-boto3-backup (==1.17.102.post1)"] -batch = ["mypy-boto3-batch (==1.17.102.post1)"] -braket = ["mypy-boto3-braket (==1.17.102.post1)"] -budgets = ["mypy-boto3-budgets (==1.17.102.post1)"] -ce = ["mypy-boto3-ce (==1.17.102.post1)"] -chime = ["mypy-boto3-chime (==1.17.102.post1)"] -cloud9 = ["mypy-boto3-cloud9 (==1.17.102.post1)"] -clouddirectory = ["mypy-boto3-clouddirectory (==1.17.102.post1)"] -cloudformation = ["mypy-boto3-cloudformation (==1.17.102.post1)"] -cloudfront = ["mypy-boto3-cloudfront (==1.17.102.post1)"] -cloudhsm = ["mypy-boto3-cloudhsm (==1.17.102.post1)"] -cloudhsmv2 = ["mypy-boto3-cloudhsmv2 (==1.17.102.post1)"] -cloudsearch = ["mypy-boto3-cloudsearch (==1.17.102.post1)"] -cloudsearchdomain = ["mypy-boto3-cloudsearchdomain (==1.17.102.post1)"] -cloudtrail = ["mypy-boto3-cloudtrail (==1.17.102.post1)"] -cloudwatch = ["mypy-boto3-cloudwatch (==1.17.102.post1)"] -codeartifact = ["mypy-boto3-codeartifact (==1.17.102.post1)"] -codebuild = ["mypy-boto3-codebuild (==1.17.102.post1)"] -codecommit = ["mypy-boto3-codecommit (==1.17.102.post1)"] -codedeploy = ["mypy-boto3-codedeploy (==1.17.102.post1)"] -codeguru-reviewer = ["mypy-boto3-codeguru-reviewer (==1.17.102.post1)"] -codeguruprofiler = ["mypy-boto3-codeguruprofiler (==1.17.102.post1)"] -codepipeline = ["mypy-boto3-codepipeline (==1.17.102.post1)"] -codestar = ["mypy-boto3-codestar (==1.17.102.post1)"] -codestar-connections = ["mypy-boto3-codestar-connections (==1.17.102.post1)"] -codestar-notifications = ["mypy-boto3-codestar-notifications (==1.17.102.post1)"] -cognito-identity = ["mypy-boto3-cognito-identity (==1.17.102.post1)"] -cognito-idp = ["mypy-boto3-cognito-idp (==1.17.102.post1)"] -cognito-sync = ["mypy-boto3-cognito-sync (==1.17.102.post1)"] -comprehend = ["mypy-boto3-comprehend (==1.17.102.post1)"] -comprehendmedical = ["mypy-boto3-comprehendmedical (==1.17.102.post1)"] -compute-optimizer = ["mypy-boto3-compute-optimizer (==1.17.102.post1)"] -config = ["mypy-boto3-config (==1.17.102.post1)"] -connect = ["mypy-boto3-connect (==1.17.102.post1)"] -connect-contact-lens = ["mypy-boto3-connect-contact-lens (==1.17.102.post1)"] -connectparticipant = ["mypy-boto3-connectparticipant (==1.17.102.post1)"] -cur = ["mypy-boto3-cur (==1.17.102.post1)"] -customer-profiles = ["mypy-boto3-customer-profiles (==1.17.102.post1)"] -databrew = ["mypy-boto3-databrew (==1.17.102.post1)"] -dataexchange = ["mypy-boto3-dataexchange (==1.17.102.post1)"] -datapipeline = ["mypy-boto3-datapipeline (==1.17.102.post1)"] -datasync = ["mypy-boto3-datasync (==1.17.102.post1)"] -dax = ["mypy-boto3-dax (==1.17.102.post1)"] -detective = ["mypy-boto3-detective (==1.17.102.post1)"] -devicefarm = ["mypy-boto3-devicefarm (==1.17.102.post1)"] -devops-guru = ["mypy-boto3-devops-guru (==1.17.102.post1)"] -directconnect = ["mypy-boto3-directconnect (==1.17.102.post1)"] -discovery = ["mypy-boto3-discovery (==1.17.102.post1)"] -dlm = ["mypy-boto3-dlm (==1.17.102.post1)"] -dms = ["mypy-boto3-dms (==1.17.102.post1)"] -docdb = ["mypy-boto3-docdb (==1.17.102.post1)"] -ds = ["mypy-boto3-ds (==1.17.102.post1)"] -dynamodb = ["mypy-boto3-dynamodb (==1.17.102.post1)"] -dynamodbstreams = ["mypy-boto3-dynamodbstreams (==1.17.102.post1)"] -ebs = ["mypy-boto3-ebs (==1.17.102.post1)"] -ec2 = ["mypy-boto3-ec2 (==1.17.102.post1)"] -ec2-instance-connect = ["mypy-boto3-ec2-instance-connect (==1.17.102.post1)"] -ecr = ["mypy-boto3-ecr (==1.17.102.post1)"] -ecr-public = ["mypy-boto3-ecr-public (==1.17.102.post1)"] -ecs = ["mypy-boto3-ecs (==1.17.102.post1)"] -efs = ["mypy-boto3-efs (==1.17.102.post1)"] -eks = ["mypy-boto3-eks (==1.17.102.post1)"] -elastic-inference = ["mypy-boto3-elastic-inference (==1.17.102.post1)"] -elasticache = ["mypy-boto3-elasticache (==1.17.102.post1)"] -elasticbeanstalk = ["mypy-boto3-elasticbeanstalk (==1.17.102.post1)"] -elastictranscoder = ["mypy-boto3-elastictranscoder (==1.17.102.post1)"] -elb = ["mypy-boto3-elb (==1.17.102.post1)"] -elbv2 = ["mypy-boto3-elbv2 (==1.17.102.post1)"] -emr = ["mypy-boto3-emr (==1.17.102.post1)"] -emr-containers = ["mypy-boto3-emr-containers (==1.17.102.post1)"] -es = ["mypy-boto3-es (==1.17.102.post1)"] -essential = ["mypy-boto3-cloudformation (==1.17.102.post1)", "mypy-boto3-dynamodb (==1.17.102.post1)", "mypy-boto3-ec2 (==1.17.102.post1)", "mypy-boto3-lambda (==1.17.102.post1)", "mypy-boto3-rds (==1.17.102.post1)", "mypy-boto3-s3 (==1.17.102.post1)", "mypy-boto3-sqs (==1.17.102.post1)"] -events = ["mypy-boto3-events (==1.17.102.post1)"] -finspace = ["mypy-boto3-finspace (==1.17.102.post1)"] -finspace-data = ["mypy-boto3-finspace-data (==1.17.102.post1)"] -firehose = ["mypy-boto3-firehose (==1.17.102.post1)"] -fis = ["mypy-boto3-fis (==1.17.102.post1)"] -fms = ["mypy-boto3-fms (==1.17.102.post1)"] -forecast = ["mypy-boto3-forecast (==1.17.102.post1)"] -forecastquery = ["mypy-boto3-forecastquery (==1.17.102.post1)"] -frauddetector = ["mypy-boto3-frauddetector (==1.17.102.post1)"] -fsx = ["mypy-boto3-fsx (==1.17.102.post1)"] -gamelift = ["mypy-boto3-gamelift (==1.17.102.post1)"] -glacier = ["mypy-boto3-glacier (==1.17.102.post1)"] -globalaccelerator = ["mypy-boto3-globalaccelerator (==1.17.102.post1)"] -glue = ["mypy-boto3-glue (==1.17.102.post1)"] -greengrass = ["mypy-boto3-greengrass (==1.17.102.post1)"] -greengrassv2 = ["mypy-boto3-greengrassv2 (==1.17.102.post1)"] -groundstation = ["mypy-boto3-groundstation (==1.17.102.post1)"] -guardduty = ["mypy-boto3-guardduty (==1.17.102.post1)"] -health = ["mypy-boto3-health (==1.17.102.post1)"] -healthlake = ["mypy-boto3-healthlake (==1.17.102.post1)"] -honeycode = ["mypy-boto3-honeycode (==1.17.102.post1)"] -iam = ["mypy-boto3-iam (==1.17.102.post1)"] -identitystore = ["mypy-boto3-identitystore (==1.17.102.post1)"] -imagebuilder = ["mypy-boto3-imagebuilder (==1.17.102.post1)"] -importexport = ["mypy-boto3-importexport (==1.17.102.post1)"] -inspector = ["mypy-boto3-inspector (==1.17.102.post1)"] -iot = ["mypy-boto3-iot (==1.17.102.post1)"] -iot-data = ["mypy-boto3-iot-data (==1.17.102.post1)"] -iot-jobs-data = ["mypy-boto3-iot-jobs-data (==1.17.102.post1)"] -iot1click-devices = ["mypy-boto3-iot1click-devices (==1.17.102.post1)"] -iot1click-projects = ["mypy-boto3-iot1click-projects (==1.17.102.post1)"] -iotanalytics = ["mypy-boto3-iotanalytics (==1.17.102.post1)"] -iotdeviceadvisor = ["mypy-boto3-iotdeviceadvisor (==1.17.102.post1)"] -iotevents = ["mypy-boto3-iotevents (==1.17.102.post1)"] -iotevents-data = ["mypy-boto3-iotevents-data (==1.17.102.post1)"] -iotfleethub = ["mypy-boto3-iotfleethub (==1.17.102.post1)"] -iotsecuretunneling = ["mypy-boto3-iotsecuretunneling (==1.17.102.post1)"] -iotsitewise = ["mypy-boto3-iotsitewise (==1.17.102.post1)"] -iotthingsgraph = ["mypy-boto3-iotthingsgraph (==1.17.102.post1)"] -iotwireless = ["mypy-boto3-iotwireless (==1.17.102.post1)"] -ivs = ["mypy-boto3-ivs (==1.17.102.post1)"] -kafka = ["mypy-boto3-kafka (==1.17.102.post1)"] -kendra = ["mypy-boto3-kendra (==1.17.102.post1)"] -kinesis = ["mypy-boto3-kinesis (==1.17.102.post1)"] -kinesis-video-archived-media = ["mypy-boto3-kinesis-video-archived-media (==1.17.102.post1)"] -kinesis-video-media = ["mypy-boto3-kinesis-video-media (==1.17.102.post1)"] -kinesis-video-signaling = ["mypy-boto3-kinesis-video-signaling (==1.17.102.post1)"] -kinesisanalytics = ["mypy-boto3-kinesisanalytics (==1.17.102.post1)"] -kinesisanalyticsv2 = ["mypy-boto3-kinesisanalyticsv2 (==1.17.102.post1)"] -kinesisvideo = ["mypy-boto3-kinesisvideo (==1.17.102.post1)"] -kms = ["mypy-boto3-kms (==1.17.102.post1)"] -lakeformation = ["mypy-boto3-lakeformation (==1.17.102.post1)"] -lambda = ["mypy-boto3-lambda (==1.17.102.post1)"] -lex-models = ["mypy-boto3-lex-models (==1.17.102.post1)"] -lex-runtime = ["mypy-boto3-lex-runtime (==1.17.102.post1)"] -lexv2-models = ["mypy-boto3-lexv2-models (==1.17.102.post1)"] -lexv2-runtime = ["mypy-boto3-lexv2-runtime (==1.17.102.post1)"] -license-manager = ["mypy-boto3-license-manager (==1.17.102.post1)"] -lightsail = ["mypy-boto3-lightsail (==1.17.102.post1)"] -location = ["mypy-boto3-location (==1.17.102.post1)"] -logs = ["mypy-boto3-logs (==1.17.102.post1)"] -lookoutequipment = ["mypy-boto3-lookoutequipment (==1.17.102.post1)"] -lookoutmetrics = ["mypy-boto3-lookoutmetrics (==1.17.102.post1)"] -lookoutvision = ["mypy-boto3-lookoutvision (==1.17.102.post1)"] -machinelearning = ["mypy-boto3-machinelearning (==1.17.102.post1)"] -macie = ["mypy-boto3-macie (==1.17.102.post1)"] -macie2 = ["mypy-boto3-macie2 (==1.17.102.post1)"] -managedblockchain = ["mypy-boto3-managedblockchain (==1.17.102.post1)"] -marketplace-catalog = ["mypy-boto3-marketplace-catalog (==1.17.102.post1)"] -marketplace-entitlement = ["mypy-boto3-marketplace-entitlement (==1.17.102.post1)"] -marketplacecommerceanalytics = ["mypy-boto3-marketplacecommerceanalytics (==1.17.102.post1)"] -mediaconnect = ["mypy-boto3-mediaconnect (==1.17.102.post1)"] -mediaconvert = ["mypy-boto3-mediaconvert (==1.17.102.post1)"] -medialive = ["mypy-boto3-medialive (==1.17.102.post1)"] -mediapackage = ["mypy-boto3-mediapackage (==1.17.102.post1)"] -mediapackage-vod = ["mypy-boto3-mediapackage-vod (==1.17.102.post1)"] -mediastore = ["mypy-boto3-mediastore (==1.17.102.post1)"] -mediastore-data = ["mypy-boto3-mediastore-data (==1.17.102.post1)"] -mediatailor = ["mypy-boto3-mediatailor (==1.17.102.post1)"] -meteringmarketplace = ["mypy-boto3-meteringmarketplace (==1.17.102.post1)"] -mgh = ["mypy-boto3-mgh (==1.17.102.post1)"] -mgn = ["mypy-boto3-mgn (==1.17.102.post1)"] -migrationhub-config = ["mypy-boto3-migrationhub-config (==1.17.102.post1)"] -mobile = ["mypy-boto3-mobile (==1.17.102.post1)"] -mq = ["mypy-boto3-mq (==1.17.102.post1)"] -mturk = ["mypy-boto3-mturk (==1.17.102.post1)"] -mwaa = ["mypy-boto3-mwaa (==1.17.102.post1)"] -neptune = ["mypy-boto3-neptune (==1.17.102.post1)"] -network-firewall = ["mypy-boto3-network-firewall (==1.17.102.post1)"] -networkmanager = ["mypy-boto3-networkmanager (==1.17.102.post1)"] -nimble = ["mypy-boto3-nimble (==1.17.102.post1)"] -opsworks = ["mypy-boto3-opsworks (==1.17.102.post1)"] -opsworkscm = ["mypy-boto3-opsworkscm (==1.17.102.post1)"] -organizations = ["mypy-boto3-organizations (==1.17.102.post1)"] -outposts = ["mypy-boto3-outposts (==1.17.102.post1)"] -personalize = ["mypy-boto3-personalize (==1.17.102.post1)"] -personalize-events = ["mypy-boto3-personalize-events (==1.17.102.post1)"] -personalize-runtime = ["mypy-boto3-personalize-runtime (==1.17.102.post1)"] -pi = ["mypy-boto3-pi (==1.17.102.post1)"] -pinpoint = ["mypy-boto3-pinpoint (==1.17.102.post1)"] -pinpoint-email = ["mypy-boto3-pinpoint-email (==1.17.102.post1)"] -pinpoint-sms-voice = ["mypy-boto3-pinpoint-sms-voice (==1.17.102.post1)"] -polly = ["mypy-boto3-polly (==1.17.102.post1)"] -pricing = ["mypy-boto3-pricing (==1.17.102.post1)"] -proton = ["mypy-boto3-proton (==1.17.102.post1)"] -qldb = ["mypy-boto3-qldb (==1.17.102.post1)"] -qldb-session = ["mypy-boto3-qldb-session (==1.17.102.post1)"] -quicksight = ["mypy-boto3-quicksight (==1.17.102.post1)"] -ram = ["mypy-boto3-ram (==1.17.102.post1)"] -rds = ["mypy-boto3-rds (==1.17.102.post1)"] -rds-data = ["mypy-boto3-rds-data (==1.17.102.post1)"] -redshift = ["mypy-boto3-redshift (==1.17.102.post1)"] -redshift-data = ["mypy-boto3-redshift-data (==1.17.102.post1)"] -rekognition = ["mypy-boto3-rekognition (==1.17.102.post1)"] -resource-groups = ["mypy-boto3-resource-groups (==1.17.102.post1)"] -resourcegroupstaggingapi = ["mypy-boto3-resourcegroupstaggingapi (==1.17.102.post1)"] -robomaker = ["mypy-boto3-robomaker (==1.17.102.post1)"] -route53 = ["mypy-boto3-route53 (==1.17.102.post1)"] -route53domains = ["mypy-boto3-route53domains (==1.17.102.post1)"] -route53resolver = ["mypy-boto3-route53resolver (==1.17.102.post1)"] -s3 = ["mypy-boto3-s3 (==1.17.102.post1)"] -s3control = ["mypy-boto3-s3control (==1.17.102.post1)"] -s3outposts = ["mypy-boto3-s3outposts (==1.17.102.post1)"] -sagemaker = ["mypy-boto3-sagemaker (==1.17.102.post1)"] -sagemaker-a2i-runtime = ["mypy-boto3-sagemaker-a2i-runtime (==1.17.102.post1)"] -sagemaker-edge = ["mypy-boto3-sagemaker-edge (==1.17.102.post1)"] -sagemaker-featurestore-runtime = ["mypy-boto3-sagemaker-featurestore-runtime (==1.17.102.post1)"] -sagemaker-runtime = ["mypy-boto3-sagemaker-runtime (==1.17.102.post1)"] -savingsplans = ["mypy-boto3-savingsplans (==1.17.102.post1)"] -schemas = ["mypy-boto3-schemas (==1.17.102.post1)"] -sdb = ["mypy-boto3-sdb (==1.17.102.post1)"] -secretsmanager = ["mypy-boto3-secretsmanager (==1.17.102.post1)"] -securityhub = ["mypy-boto3-securityhub (==1.17.102.post1)"] -serverlessrepo = ["mypy-boto3-serverlessrepo (==1.17.102.post1)"] -service-quotas = ["mypy-boto3-service-quotas (==1.17.102.post1)"] -servicecatalog = ["mypy-boto3-servicecatalog (==1.17.102.post1)"] -servicecatalog-appregistry = ["mypy-boto3-servicecatalog-appregistry (==1.17.102.post1)"] -servicediscovery = ["mypy-boto3-servicediscovery (==1.17.102.post1)"] -ses = ["mypy-boto3-ses (==1.17.102.post1)"] -sesv2 = ["mypy-boto3-sesv2 (==1.17.102.post1)"] -shield = ["mypy-boto3-shield (==1.17.102.post1)"] -signer = ["mypy-boto3-signer (==1.17.102.post1)"] -sms = ["mypy-boto3-sms (==1.17.102.post1)"] -sms-voice = ["mypy-boto3-sms-voice (==1.17.102.post1)"] -snowball = ["mypy-boto3-snowball (==1.17.102.post1)"] -sns = ["mypy-boto3-sns (==1.17.102.post1)"] -sqs = ["mypy-boto3-sqs (==1.17.102.post1)"] -ssm = ["mypy-boto3-ssm (==1.17.102.post1)"] -ssm-contacts = ["mypy-boto3-ssm-contacts (==1.17.102.post1)"] -ssm-incidents = ["mypy-boto3-ssm-incidents (==1.17.102.post1)"] -sso = ["mypy-boto3-sso (==1.17.102.post1)"] -sso-admin = ["mypy-boto3-sso-admin (==1.17.102.post1)"] -sso-oidc = ["mypy-boto3-sso-oidc (==1.17.102.post1)"] -stepfunctions = ["mypy-boto3-stepfunctions (==1.17.102.post1)"] -storagegateway = ["mypy-boto3-storagegateway (==1.17.102.post1)"] -sts = ["mypy-boto3-sts (==1.17.102.post1)"] -support = ["mypy-boto3-support (==1.17.102.post1)"] -swf = ["mypy-boto3-swf (==1.17.102.post1)"] -synthetics = ["mypy-boto3-synthetics (==1.17.102.post1)"] -textract = ["mypy-boto3-textract (==1.17.102.post1)"] -timestream-query = ["mypy-boto3-timestream-query (==1.17.102.post1)"] -timestream-write = ["mypy-boto3-timestream-write (==1.17.102.post1)"] -transcribe = ["mypy-boto3-transcribe (==1.17.102.post1)"] -transfer = ["mypy-boto3-transfer (==1.17.102.post1)"] -translate = ["mypy-boto3-translate (==1.17.102.post1)"] -waf = ["mypy-boto3-waf (==1.17.102.post1)"] -waf-regional = ["mypy-boto3-waf-regional (==1.17.102.post1)"] -wafv2 = ["mypy-boto3-wafv2 (==1.17.102.post1)"] -wellarchitected = ["mypy-boto3-wellarchitected (==1.17.102.post1)"] -workdocs = ["mypy-boto3-workdocs (==1.17.102.post1)"] -worklink = ["mypy-boto3-worklink (==1.17.102.post1)"] -workmail = ["mypy-boto3-workmail (==1.17.102.post1)"] -workmailmessageflow = ["mypy-boto3-workmailmessageflow (==1.17.102.post1)"] -workspaces = ["mypy-boto3-workspaces (==1.17.102.post1)"] -xray = ["mypy-boto3-xray (==1.17.102.post1)"] +accessanalyzer = ["mypy-boto3-accessanalyzer (==1.17.103.post1)"] +acm = ["mypy-boto3-acm (==1.17.103.post1)"] +acm-pca = ["mypy-boto3-acm-pca (==1.17.103.post1)"] +alexaforbusiness = ["mypy-boto3-alexaforbusiness (==1.17.103.post1)"] +all = ["mypy-boto3-accessanalyzer (==1.17.103.post1)", "mypy-boto3-acm (==1.17.103.post1)", "mypy-boto3-acm-pca (==1.17.103.post1)", "mypy-boto3-alexaforbusiness (==1.17.103.post1)", "mypy-boto3-amp (==1.17.103.post1)", "mypy-boto3-amplify (==1.17.103.post1)", "mypy-boto3-amplifybackend (==1.17.103.post1)", "mypy-boto3-apigateway (==1.17.103.post1)", "mypy-boto3-apigatewaymanagementapi (==1.17.103.post1)", "mypy-boto3-apigatewayv2 (==1.17.103.post1)", "mypy-boto3-appconfig (==1.17.103.post1)", "mypy-boto3-appflow (==1.17.103.post1)", "mypy-boto3-appintegrations (==1.17.103.post1)", "mypy-boto3-application-autoscaling (==1.17.103.post1)", "mypy-boto3-application-insights (==1.17.103.post1)", "mypy-boto3-applicationcostprofiler (==1.17.103.post1)", "mypy-boto3-appmesh (==1.17.103.post1)", "mypy-boto3-apprunner (==1.17.103.post1)", "mypy-boto3-appstream (==1.17.103.post1)", "mypy-boto3-appsync (==1.17.103.post1)", "mypy-boto3-athena (==1.17.103.post1)", "mypy-boto3-auditmanager (==1.17.103.post1)", "mypy-boto3-autoscaling (==1.17.103.post1)", "mypy-boto3-autoscaling-plans (==1.17.103.post1)", "mypy-boto3-backup (==1.17.103.post1)", "mypy-boto3-batch (==1.17.103.post1)", "mypy-boto3-braket (==1.17.103.post1)", "mypy-boto3-budgets (==1.17.103.post1)", "mypy-boto3-ce (==1.17.103.post1)", "mypy-boto3-chime (==1.17.103.post1)", "mypy-boto3-cloud9 (==1.17.103.post1)", "mypy-boto3-clouddirectory (==1.17.103.post1)", "mypy-boto3-cloudformation (==1.17.103.post1)", "mypy-boto3-cloudfront (==1.17.103.post1)", "mypy-boto3-cloudhsm (==1.17.103.post1)", "mypy-boto3-cloudhsmv2 (==1.17.103.post1)", "mypy-boto3-cloudsearch (==1.17.103.post1)", "mypy-boto3-cloudsearchdomain (==1.17.103.post1)", "mypy-boto3-cloudtrail (==1.17.103.post1)", "mypy-boto3-cloudwatch (==1.17.103.post1)", "mypy-boto3-codeartifact (==1.17.103.post1)", "mypy-boto3-codebuild (==1.17.103.post1)", "mypy-boto3-codecommit (==1.17.103.post1)", "mypy-boto3-codedeploy (==1.17.103.post1)", "mypy-boto3-codeguru-reviewer (==1.17.103.post1)", "mypy-boto3-codeguruprofiler (==1.17.103.post1)", "mypy-boto3-codepipeline (==1.17.103.post1)", "mypy-boto3-codestar (==1.17.103.post1)", "mypy-boto3-codestar-connections (==1.17.103.post1)", "mypy-boto3-codestar-notifications (==1.17.103.post1)", "mypy-boto3-cognito-identity (==1.17.103.post1)", "mypy-boto3-cognito-idp (==1.17.103.post1)", "mypy-boto3-cognito-sync (==1.17.103.post1)", "mypy-boto3-comprehend (==1.17.103.post1)", "mypy-boto3-comprehendmedical (==1.17.103.post1)", "mypy-boto3-compute-optimizer (==1.17.103.post1)", "mypy-boto3-config (==1.17.103.post1)", "mypy-boto3-connect (==1.17.103.post1)", "mypy-boto3-connect-contact-lens (==1.17.103.post1)", "mypy-boto3-connectparticipant (==1.17.103.post1)", "mypy-boto3-cur (==1.17.103.post1)", "mypy-boto3-customer-profiles (==1.17.103.post1)", "mypy-boto3-databrew (==1.17.103.post1)", "mypy-boto3-dataexchange (==1.17.103.post1)", "mypy-boto3-datapipeline (==1.17.103.post1)", "mypy-boto3-datasync (==1.17.103.post1)", "mypy-boto3-dax (==1.17.103.post1)", "mypy-boto3-detective (==1.17.103.post1)", "mypy-boto3-devicefarm (==1.17.103.post1)", "mypy-boto3-devops-guru (==1.17.103.post1)", "mypy-boto3-directconnect (==1.17.103.post1)", "mypy-boto3-discovery (==1.17.103.post1)", "mypy-boto3-dlm (==1.17.103.post1)", "mypy-boto3-dms (==1.17.103.post1)", "mypy-boto3-docdb (==1.17.103.post1)", "mypy-boto3-ds (==1.17.103.post1)", "mypy-boto3-dynamodb (==1.17.103.post1)", "mypy-boto3-dynamodbstreams (==1.17.103.post1)", "mypy-boto3-ebs (==1.17.103.post1)", "mypy-boto3-ec2 (==1.17.103.post1)", "mypy-boto3-ec2-instance-connect (==1.17.103.post1)", "mypy-boto3-ecr (==1.17.103.post1)", "mypy-boto3-ecr-public (==1.17.103.post1)", "mypy-boto3-ecs (==1.17.103.post1)", "mypy-boto3-efs (==1.17.103.post1)", "mypy-boto3-eks (==1.17.103.post1)", "mypy-boto3-elastic-inference (==1.17.103.post1)", "mypy-boto3-elasticache (==1.17.103.post1)", "mypy-boto3-elasticbeanstalk (==1.17.103.post1)", "mypy-boto3-elastictranscoder (==1.17.103.post1)", "mypy-boto3-elb (==1.17.103.post1)", "mypy-boto3-elbv2 (==1.17.103.post1)", "mypy-boto3-emr (==1.17.103.post1)", "mypy-boto3-emr-containers (==1.17.103.post1)", "mypy-boto3-es (==1.17.103.post1)", "mypy-boto3-events (==1.17.103.post1)", "mypy-boto3-finspace (==1.17.103.post1)", "mypy-boto3-finspace-data (==1.17.103.post1)", "mypy-boto3-firehose (==1.17.103.post1)", "mypy-boto3-fis (==1.17.103.post1)", "mypy-boto3-fms (==1.17.103.post1)", "mypy-boto3-forecast (==1.17.103.post1)", "mypy-boto3-forecastquery (==1.17.103.post1)", "mypy-boto3-frauddetector (==1.17.103.post1)", "mypy-boto3-fsx (==1.17.103.post1)", "mypy-boto3-gamelift (==1.17.103.post1)", "mypy-boto3-glacier (==1.17.103.post1)", "mypy-boto3-globalaccelerator (==1.17.103.post1)", "mypy-boto3-glue (==1.17.103.post1)", "mypy-boto3-greengrass (==1.17.103.post1)", "mypy-boto3-greengrassv2 (==1.17.103.post1)", "mypy-boto3-groundstation (==1.17.103.post1)", "mypy-boto3-guardduty (==1.17.103.post1)", "mypy-boto3-health (==1.17.103.post1)", "mypy-boto3-healthlake (==1.17.103.post1)", "mypy-boto3-honeycode (==1.17.103.post1)", "mypy-boto3-iam (==1.17.103.post1)", "mypy-boto3-identitystore (==1.17.103.post1)", "mypy-boto3-imagebuilder (==1.17.103.post1)", "mypy-boto3-importexport (==1.17.103.post1)", "mypy-boto3-inspector (==1.17.103.post1)", "mypy-boto3-iot (==1.17.103.post1)", "mypy-boto3-iot-data (==1.17.103.post1)", "mypy-boto3-iot-jobs-data (==1.17.103.post1)", "mypy-boto3-iot1click-devices (==1.17.103.post1)", "mypy-boto3-iot1click-projects (==1.17.103.post1)", "mypy-boto3-iotanalytics (==1.17.103.post1)", "mypy-boto3-iotdeviceadvisor (==1.17.103.post1)", "mypy-boto3-iotevents (==1.17.103.post1)", "mypy-boto3-iotevents-data (==1.17.103.post1)", "mypy-boto3-iotfleethub (==1.17.103.post1)", "mypy-boto3-iotsecuretunneling (==1.17.103.post1)", "mypy-boto3-iotsitewise (==1.17.103.post1)", "mypy-boto3-iotthingsgraph (==1.17.103.post1)", "mypy-boto3-iotwireless (==1.17.103.post1)", "mypy-boto3-ivs (==1.17.103.post1)", "mypy-boto3-kafka (==1.17.103.post1)", "mypy-boto3-kendra (==1.17.103.post1)", "mypy-boto3-kinesis (==1.17.103.post1)", "mypy-boto3-kinesis-video-archived-media (==1.17.103.post1)", "mypy-boto3-kinesis-video-media (==1.17.103.post1)", "mypy-boto3-kinesis-video-signaling (==1.17.103.post1)", "mypy-boto3-kinesisanalytics (==1.17.103.post1)", "mypy-boto3-kinesisanalyticsv2 (==1.17.103.post1)", "mypy-boto3-kinesisvideo (==1.17.103.post1)", "mypy-boto3-kms (==1.17.103.post1)", "mypy-boto3-lakeformation (==1.17.103.post1)", "mypy-boto3-lambda (==1.17.103.post1)", "mypy-boto3-lex-models (==1.17.103.post1)", "mypy-boto3-lex-runtime (==1.17.103.post1)", "mypy-boto3-lexv2-models (==1.17.103.post1)", "mypy-boto3-lexv2-runtime (==1.17.103.post1)", "mypy-boto3-license-manager (==1.17.103.post1)", "mypy-boto3-lightsail (==1.17.103.post1)", "mypy-boto3-location (==1.17.103.post1)", "mypy-boto3-logs (==1.17.103.post1)", "mypy-boto3-lookoutequipment (==1.17.103.post1)", "mypy-boto3-lookoutmetrics (==1.17.103.post1)", "mypy-boto3-lookoutvision (==1.17.103.post1)", "mypy-boto3-machinelearning (==1.17.103.post1)", "mypy-boto3-macie (==1.17.103.post1)", "mypy-boto3-macie2 (==1.17.103.post1)", "mypy-boto3-managedblockchain (==1.17.103.post1)", "mypy-boto3-marketplace-catalog (==1.17.103.post1)", "mypy-boto3-marketplace-entitlement (==1.17.103.post1)", "mypy-boto3-marketplacecommerceanalytics (==1.17.103.post1)", "mypy-boto3-mediaconnect (==1.17.103.post1)", "mypy-boto3-mediaconvert (==1.17.103.post1)", "mypy-boto3-medialive (==1.17.103.post1)", "mypy-boto3-mediapackage (==1.17.103.post1)", "mypy-boto3-mediapackage-vod (==1.17.103.post1)", "mypy-boto3-mediastore (==1.17.103.post1)", "mypy-boto3-mediastore-data (==1.17.103.post1)", "mypy-boto3-mediatailor (==1.17.103.post1)", "mypy-boto3-meteringmarketplace (==1.17.103.post1)", "mypy-boto3-mgh (==1.17.103.post1)", "mypy-boto3-mgn (==1.17.103.post1)", "mypy-boto3-migrationhub-config (==1.17.103.post1)", "mypy-boto3-mobile (==1.17.103.post1)", "mypy-boto3-mq (==1.17.103.post1)", "mypy-boto3-mturk (==1.17.103.post1)", "mypy-boto3-mwaa (==1.17.103.post1)", "mypy-boto3-neptune (==1.17.103.post1)", "mypy-boto3-network-firewall (==1.17.103.post1)", "mypy-boto3-networkmanager (==1.17.103.post1)", "mypy-boto3-nimble (==1.17.103.post1)", "mypy-boto3-opsworks (==1.17.103.post1)", "mypy-boto3-opsworkscm (==1.17.103.post1)", "mypy-boto3-organizations (==1.17.103.post1)", "mypy-boto3-outposts (==1.17.103.post1)", "mypy-boto3-personalize (==1.17.103.post1)", "mypy-boto3-personalize-events (==1.17.103.post1)", "mypy-boto3-personalize-runtime (==1.17.103.post1)", "mypy-boto3-pi (==1.17.103.post1)", "mypy-boto3-pinpoint (==1.17.103.post1)", "mypy-boto3-pinpoint-email (==1.17.103.post1)", "mypy-boto3-pinpoint-sms-voice (==1.17.103.post1)", "mypy-boto3-polly (==1.17.103.post1)", "mypy-boto3-pricing (==1.17.103.post1)", "mypy-boto3-proton (==1.17.103.post1)", "mypy-boto3-qldb (==1.17.103.post1)", "mypy-boto3-qldb-session (==1.17.103.post1)", "mypy-boto3-quicksight (==1.17.103.post1)", "mypy-boto3-ram (==1.17.103.post1)", "mypy-boto3-rds (==1.17.103.post1)", "mypy-boto3-rds-data (==1.17.103.post1)", "mypy-boto3-redshift (==1.17.103.post1)", "mypy-boto3-redshift-data (==1.17.103.post1)", "mypy-boto3-rekognition (==1.17.103.post1)", "mypy-boto3-resource-groups (==1.17.103.post1)", "mypy-boto3-resourcegroupstaggingapi (==1.17.103.post1)", "mypy-boto3-robomaker (==1.17.103.post1)", "mypy-boto3-route53 (==1.17.103.post1)", "mypy-boto3-route53domains (==1.17.103.post1)", "mypy-boto3-route53resolver (==1.17.103.post1)", "mypy-boto3-s3 (==1.17.103.post1)", "mypy-boto3-s3control (==1.17.103.post1)", "mypy-boto3-s3outposts (==1.17.103.post1)", "mypy-boto3-sagemaker (==1.17.103.post1)", "mypy-boto3-sagemaker-a2i-runtime (==1.17.103.post1)", "mypy-boto3-sagemaker-edge (==1.17.103.post1)", "mypy-boto3-sagemaker-featurestore-runtime (==1.17.103.post1)", "mypy-boto3-sagemaker-runtime (==1.17.103.post1)", "mypy-boto3-savingsplans (==1.17.103.post1)", "mypy-boto3-schemas (==1.17.103.post1)", "mypy-boto3-sdb (==1.17.103.post1)", "mypy-boto3-secretsmanager (==1.17.103.post1)", "mypy-boto3-securityhub (==1.17.103.post1)", "mypy-boto3-serverlessrepo (==1.17.103.post1)", "mypy-boto3-service-quotas (==1.17.103.post1)", "mypy-boto3-servicecatalog (==1.17.103.post1)", "mypy-boto3-servicecatalog-appregistry (==1.17.103.post1)", "mypy-boto3-servicediscovery (==1.17.103.post1)", "mypy-boto3-ses (==1.17.103.post1)", "mypy-boto3-sesv2 (==1.17.103.post1)", "mypy-boto3-shield (==1.17.103.post1)", "mypy-boto3-signer (==1.17.103.post1)", "mypy-boto3-sms (==1.17.103.post1)", "mypy-boto3-sms-voice (==1.17.103.post1)", "mypy-boto3-snowball (==1.17.103.post1)", "mypy-boto3-sns (==1.17.103.post1)", "mypy-boto3-sqs (==1.17.103.post1)", "mypy-boto3-ssm (==1.17.103.post1)", "mypy-boto3-ssm-contacts (==1.17.103.post1)", "mypy-boto3-ssm-incidents (==1.17.103.post1)", "mypy-boto3-sso (==1.17.103.post1)", "mypy-boto3-sso-admin (==1.17.103.post1)", "mypy-boto3-sso-oidc (==1.17.103.post1)", "mypy-boto3-stepfunctions (==1.17.103.post1)", "mypy-boto3-storagegateway (==1.17.103.post1)", "mypy-boto3-sts (==1.17.103.post1)", "mypy-boto3-support (==1.17.103.post1)", "mypy-boto3-swf (==1.17.103.post1)", "mypy-boto3-synthetics (==1.17.103.post1)", "mypy-boto3-textract (==1.17.103.post1)", "mypy-boto3-timestream-query (==1.17.103.post1)", "mypy-boto3-timestream-write (==1.17.103.post1)", "mypy-boto3-transcribe (==1.17.103.post1)", "mypy-boto3-transfer (==1.17.103.post1)", "mypy-boto3-translate (==1.17.103.post1)", "mypy-boto3-waf (==1.17.103.post1)", "mypy-boto3-waf-regional (==1.17.103.post1)", "mypy-boto3-wafv2 (==1.17.103.post1)", "mypy-boto3-wellarchitected (==1.17.103.post1)", "mypy-boto3-workdocs (==1.17.103.post1)", "mypy-boto3-worklink (==1.17.103.post1)", "mypy-boto3-workmail (==1.17.103.post1)", "mypy-boto3-workmailmessageflow (==1.17.103.post1)", "mypy-boto3-workspaces (==1.17.103.post1)", "mypy-boto3-xray (==1.17.103.post1)"] +amp = ["mypy-boto3-amp (==1.17.103.post1)"] +amplify = ["mypy-boto3-amplify (==1.17.103.post1)"] +amplifybackend = ["mypy-boto3-amplifybackend (==1.17.103.post1)"] +apigateway = ["mypy-boto3-apigateway (==1.17.103.post1)"] +apigatewaymanagementapi = ["mypy-boto3-apigatewaymanagementapi (==1.17.103.post1)"] +apigatewayv2 = ["mypy-boto3-apigatewayv2 (==1.17.103.post1)"] +appconfig = ["mypy-boto3-appconfig (==1.17.103.post1)"] +appflow = ["mypy-boto3-appflow (==1.17.103.post1)"] +appintegrations = ["mypy-boto3-appintegrations (==1.17.103.post1)"] +application-autoscaling = ["mypy-boto3-application-autoscaling (==1.17.103.post1)"] +application-insights = ["mypy-boto3-application-insights (==1.17.103.post1)"] +applicationcostprofiler = ["mypy-boto3-applicationcostprofiler (==1.17.103.post1)"] +appmesh = ["mypy-boto3-appmesh (==1.17.103.post1)"] +apprunner = ["mypy-boto3-apprunner (==1.17.103.post1)"] +appstream = ["mypy-boto3-appstream (==1.17.103.post1)"] +appsync = ["mypy-boto3-appsync (==1.17.103.post1)"] +athena = ["mypy-boto3-athena (==1.17.103.post1)"] +auditmanager = ["mypy-boto3-auditmanager (==1.17.103.post1)"] +autoscaling = ["mypy-boto3-autoscaling (==1.17.103.post1)"] +autoscaling-plans = ["mypy-boto3-autoscaling-plans (==1.17.103.post1)"] +backup = ["mypy-boto3-backup (==1.17.103.post1)"] +batch = ["mypy-boto3-batch (==1.17.103.post1)"] +braket = ["mypy-boto3-braket (==1.17.103.post1)"] +budgets = ["mypy-boto3-budgets (==1.17.103.post1)"] +ce = ["mypy-boto3-ce (==1.17.103.post1)"] +chime = ["mypy-boto3-chime (==1.17.103.post1)"] +cloud9 = ["mypy-boto3-cloud9 (==1.17.103.post1)"] +clouddirectory = ["mypy-boto3-clouddirectory (==1.17.103.post1)"] +cloudformation = ["mypy-boto3-cloudformation (==1.17.103.post1)"] +cloudfront = ["mypy-boto3-cloudfront (==1.17.103.post1)"] +cloudhsm = ["mypy-boto3-cloudhsm (==1.17.103.post1)"] +cloudhsmv2 = ["mypy-boto3-cloudhsmv2 (==1.17.103.post1)"] +cloudsearch = ["mypy-boto3-cloudsearch (==1.17.103.post1)"] +cloudsearchdomain = ["mypy-boto3-cloudsearchdomain (==1.17.103.post1)"] +cloudtrail = ["mypy-boto3-cloudtrail (==1.17.103.post1)"] +cloudwatch = ["mypy-boto3-cloudwatch (==1.17.103.post1)"] +codeartifact = ["mypy-boto3-codeartifact (==1.17.103.post1)"] +codebuild = ["mypy-boto3-codebuild (==1.17.103.post1)"] +codecommit = ["mypy-boto3-codecommit (==1.17.103.post1)"] +codedeploy = ["mypy-boto3-codedeploy (==1.17.103.post1)"] +codeguru-reviewer = ["mypy-boto3-codeguru-reviewer (==1.17.103.post1)"] +codeguruprofiler = ["mypy-boto3-codeguruprofiler (==1.17.103.post1)"] +codepipeline = ["mypy-boto3-codepipeline (==1.17.103.post1)"] +codestar = ["mypy-boto3-codestar (==1.17.103.post1)"] +codestar-connections = ["mypy-boto3-codestar-connections (==1.17.103.post1)"] +codestar-notifications = ["mypy-boto3-codestar-notifications (==1.17.103.post1)"] +cognito-identity = ["mypy-boto3-cognito-identity (==1.17.103.post1)"] +cognito-idp = ["mypy-boto3-cognito-idp (==1.17.103.post1)"] +cognito-sync = ["mypy-boto3-cognito-sync (==1.17.103.post1)"] +comprehend = ["mypy-boto3-comprehend (==1.17.103.post1)"] +comprehendmedical = ["mypy-boto3-comprehendmedical (==1.17.103.post1)"] +compute-optimizer = ["mypy-boto3-compute-optimizer (==1.17.103.post1)"] +config = ["mypy-boto3-config (==1.17.103.post1)"] +connect = ["mypy-boto3-connect (==1.17.103.post1)"] +connect-contact-lens = ["mypy-boto3-connect-contact-lens (==1.17.103.post1)"] +connectparticipant = ["mypy-boto3-connectparticipant (==1.17.103.post1)"] +cur = ["mypy-boto3-cur (==1.17.103.post1)"] +customer-profiles = ["mypy-boto3-customer-profiles (==1.17.103.post1)"] +databrew = ["mypy-boto3-databrew (==1.17.103.post1)"] +dataexchange = ["mypy-boto3-dataexchange (==1.17.103.post1)"] +datapipeline = ["mypy-boto3-datapipeline (==1.17.103.post1)"] +datasync = ["mypy-boto3-datasync (==1.17.103.post1)"] +dax = ["mypy-boto3-dax (==1.17.103.post1)"] +detective = ["mypy-boto3-detective (==1.17.103.post1)"] +devicefarm = ["mypy-boto3-devicefarm (==1.17.103.post1)"] +devops-guru = ["mypy-boto3-devops-guru (==1.17.103.post1)"] +directconnect = ["mypy-boto3-directconnect (==1.17.103.post1)"] +discovery = ["mypy-boto3-discovery (==1.17.103.post1)"] +dlm = ["mypy-boto3-dlm (==1.17.103.post1)"] +dms = ["mypy-boto3-dms (==1.17.103.post1)"] +docdb = ["mypy-boto3-docdb (==1.17.103.post1)"] +ds = ["mypy-boto3-ds (==1.17.103.post1)"] +dynamodb = ["mypy-boto3-dynamodb (==1.17.103.post1)"] +dynamodbstreams = ["mypy-boto3-dynamodbstreams (==1.17.103.post1)"] +ebs = ["mypy-boto3-ebs (==1.17.103.post1)"] +ec2 = ["mypy-boto3-ec2 (==1.17.103.post1)"] +ec2-instance-connect = ["mypy-boto3-ec2-instance-connect (==1.17.103.post1)"] +ecr = ["mypy-boto3-ecr (==1.17.103.post1)"] +ecr-public = ["mypy-boto3-ecr-public (==1.17.103.post1)"] +ecs = ["mypy-boto3-ecs (==1.17.103.post1)"] +efs = ["mypy-boto3-efs (==1.17.103.post1)"] +eks = ["mypy-boto3-eks (==1.17.103.post1)"] +elastic-inference = ["mypy-boto3-elastic-inference (==1.17.103.post1)"] +elasticache = ["mypy-boto3-elasticache (==1.17.103.post1)"] +elasticbeanstalk = ["mypy-boto3-elasticbeanstalk (==1.17.103.post1)"] +elastictranscoder = ["mypy-boto3-elastictranscoder (==1.17.103.post1)"] +elb = ["mypy-boto3-elb (==1.17.103.post1)"] +elbv2 = ["mypy-boto3-elbv2 (==1.17.103.post1)"] +emr = ["mypy-boto3-emr (==1.17.103.post1)"] +emr-containers = ["mypy-boto3-emr-containers (==1.17.103.post1)"] +es = ["mypy-boto3-es (==1.17.103.post1)"] +essential = ["mypy-boto3-cloudformation (==1.17.103.post1)", "mypy-boto3-dynamodb (==1.17.103.post1)", "mypy-boto3-ec2 (==1.17.103.post1)", "mypy-boto3-lambda (==1.17.103.post1)", "mypy-boto3-rds (==1.17.103.post1)", "mypy-boto3-s3 (==1.17.103.post1)", "mypy-boto3-sqs (==1.17.103.post1)"] +events = ["mypy-boto3-events (==1.17.103.post1)"] +finspace = ["mypy-boto3-finspace (==1.17.103.post1)"] +finspace-data = ["mypy-boto3-finspace-data (==1.17.103.post1)"] +firehose = ["mypy-boto3-firehose (==1.17.103.post1)"] +fis = ["mypy-boto3-fis (==1.17.103.post1)"] +fms = ["mypy-boto3-fms (==1.17.103.post1)"] +forecast = ["mypy-boto3-forecast (==1.17.103.post1)"] +forecastquery = ["mypy-boto3-forecastquery (==1.17.103.post1)"] +frauddetector = ["mypy-boto3-frauddetector (==1.17.103.post1)"] +fsx = ["mypy-boto3-fsx (==1.17.103.post1)"] +gamelift = ["mypy-boto3-gamelift (==1.17.103.post1)"] +glacier = ["mypy-boto3-glacier (==1.17.103.post1)"] +globalaccelerator = ["mypy-boto3-globalaccelerator (==1.17.103.post1)"] +glue = ["mypy-boto3-glue (==1.17.103.post1)"] +greengrass = ["mypy-boto3-greengrass (==1.17.103.post1)"] +greengrassv2 = ["mypy-boto3-greengrassv2 (==1.17.103.post1)"] +groundstation = ["mypy-boto3-groundstation (==1.17.103.post1)"] +guardduty = ["mypy-boto3-guardduty (==1.17.103.post1)"] +health = ["mypy-boto3-health (==1.17.103.post1)"] +healthlake = ["mypy-boto3-healthlake (==1.17.103.post1)"] +honeycode = ["mypy-boto3-honeycode (==1.17.103.post1)"] +iam = ["mypy-boto3-iam (==1.17.103.post1)"] +identitystore = ["mypy-boto3-identitystore (==1.17.103.post1)"] +imagebuilder = ["mypy-boto3-imagebuilder (==1.17.103.post1)"] +importexport = ["mypy-boto3-importexport (==1.17.103.post1)"] +inspector = ["mypy-boto3-inspector (==1.17.103.post1)"] +iot = ["mypy-boto3-iot (==1.17.103.post1)"] +iot-data = ["mypy-boto3-iot-data (==1.17.103.post1)"] +iot-jobs-data = ["mypy-boto3-iot-jobs-data (==1.17.103.post1)"] +iot1click-devices = ["mypy-boto3-iot1click-devices (==1.17.103.post1)"] +iot1click-projects = ["mypy-boto3-iot1click-projects (==1.17.103.post1)"] +iotanalytics = ["mypy-boto3-iotanalytics (==1.17.103.post1)"] +iotdeviceadvisor = ["mypy-boto3-iotdeviceadvisor (==1.17.103.post1)"] +iotevents = ["mypy-boto3-iotevents (==1.17.103.post1)"] +iotevents-data = ["mypy-boto3-iotevents-data (==1.17.103.post1)"] +iotfleethub = ["mypy-boto3-iotfleethub (==1.17.103.post1)"] +iotsecuretunneling = ["mypy-boto3-iotsecuretunneling (==1.17.103.post1)"] +iotsitewise = ["mypy-boto3-iotsitewise (==1.17.103.post1)"] +iotthingsgraph = ["mypy-boto3-iotthingsgraph (==1.17.103.post1)"] +iotwireless = ["mypy-boto3-iotwireless (==1.17.103.post1)"] +ivs = ["mypy-boto3-ivs (==1.17.103.post1)"] +kafka = ["mypy-boto3-kafka (==1.17.103.post1)"] +kendra = ["mypy-boto3-kendra (==1.17.103.post1)"] +kinesis = ["mypy-boto3-kinesis (==1.17.103.post1)"] +kinesis-video-archived-media = ["mypy-boto3-kinesis-video-archived-media (==1.17.103.post1)"] +kinesis-video-media = ["mypy-boto3-kinesis-video-media (==1.17.103.post1)"] +kinesis-video-signaling = ["mypy-boto3-kinesis-video-signaling (==1.17.103.post1)"] +kinesisanalytics = ["mypy-boto3-kinesisanalytics (==1.17.103.post1)"] +kinesisanalyticsv2 = ["mypy-boto3-kinesisanalyticsv2 (==1.17.103.post1)"] +kinesisvideo = ["mypy-boto3-kinesisvideo (==1.17.103.post1)"] +kms = ["mypy-boto3-kms (==1.17.103.post1)"] +lakeformation = ["mypy-boto3-lakeformation (==1.17.103.post1)"] +lambda = ["mypy-boto3-lambda (==1.17.103.post1)"] +lex-models = ["mypy-boto3-lex-models (==1.17.103.post1)"] +lex-runtime = ["mypy-boto3-lex-runtime (==1.17.103.post1)"] +lexv2-models = ["mypy-boto3-lexv2-models (==1.17.103.post1)"] +lexv2-runtime = ["mypy-boto3-lexv2-runtime (==1.17.103.post1)"] +license-manager = ["mypy-boto3-license-manager (==1.17.103.post1)"] +lightsail = ["mypy-boto3-lightsail (==1.17.103.post1)"] +location = ["mypy-boto3-location (==1.17.103.post1)"] +logs = ["mypy-boto3-logs (==1.17.103.post1)"] +lookoutequipment = ["mypy-boto3-lookoutequipment (==1.17.103.post1)"] +lookoutmetrics = ["mypy-boto3-lookoutmetrics (==1.17.103.post1)"] +lookoutvision = ["mypy-boto3-lookoutvision (==1.17.103.post1)"] +machinelearning = ["mypy-boto3-machinelearning (==1.17.103.post1)"] +macie = ["mypy-boto3-macie (==1.17.103.post1)"] +macie2 = ["mypy-boto3-macie2 (==1.17.103.post1)"] +managedblockchain = ["mypy-boto3-managedblockchain (==1.17.103.post1)"] +marketplace-catalog = ["mypy-boto3-marketplace-catalog (==1.17.103.post1)"] +marketplace-entitlement = ["mypy-boto3-marketplace-entitlement (==1.17.103.post1)"] +marketplacecommerceanalytics = ["mypy-boto3-marketplacecommerceanalytics (==1.17.103.post1)"] +mediaconnect = ["mypy-boto3-mediaconnect (==1.17.103.post1)"] +mediaconvert = ["mypy-boto3-mediaconvert (==1.17.103.post1)"] +medialive = ["mypy-boto3-medialive (==1.17.103.post1)"] +mediapackage = ["mypy-boto3-mediapackage (==1.17.103.post1)"] +mediapackage-vod = ["mypy-boto3-mediapackage-vod (==1.17.103.post1)"] +mediastore = ["mypy-boto3-mediastore (==1.17.103.post1)"] +mediastore-data = ["mypy-boto3-mediastore-data (==1.17.103.post1)"] +mediatailor = ["mypy-boto3-mediatailor (==1.17.103.post1)"] +meteringmarketplace = ["mypy-boto3-meteringmarketplace (==1.17.103.post1)"] +mgh = ["mypy-boto3-mgh (==1.17.103.post1)"] +mgn = ["mypy-boto3-mgn (==1.17.103.post1)"] +migrationhub-config = ["mypy-boto3-migrationhub-config (==1.17.103.post1)"] +mobile = ["mypy-boto3-mobile (==1.17.103.post1)"] +mq = ["mypy-boto3-mq (==1.17.103.post1)"] +mturk = ["mypy-boto3-mturk (==1.17.103.post1)"] +mwaa = ["mypy-boto3-mwaa (==1.17.103.post1)"] +neptune = ["mypy-boto3-neptune (==1.17.103.post1)"] +network-firewall = ["mypy-boto3-network-firewall (==1.17.103.post1)"] +networkmanager = ["mypy-boto3-networkmanager (==1.17.103.post1)"] +nimble = ["mypy-boto3-nimble (==1.17.103.post1)"] +opsworks = ["mypy-boto3-opsworks (==1.17.103.post1)"] +opsworkscm = ["mypy-boto3-opsworkscm (==1.17.103.post1)"] +organizations = ["mypy-boto3-organizations (==1.17.103.post1)"] +outposts = ["mypy-boto3-outposts (==1.17.103.post1)"] +personalize = ["mypy-boto3-personalize (==1.17.103.post1)"] +personalize-events = ["mypy-boto3-personalize-events (==1.17.103.post1)"] +personalize-runtime = ["mypy-boto3-personalize-runtime (==1.17.103.post1)"] +pi = ["mypy-boto3-pi (==1.17.103.post1)"] +pinpoint = ["mypy-boto3-pinpoint (==1.17.103.post1)"] +pinpoint-email = ["mypy-boto3-pinpoint-email (==1.17.103.post1)"] +pinpoint-sms-voice = ["mypy-boto3-pinpoint-sms-voice (==1.17.103.post1)"] +polly = ["mypy-boto3-polly (==1.17.103.post1)"] +pricing = ["mypy-boto3-pricing (==1.17.103.post1)"] +proton = ["mypy-boto3-proton (==1.17.103.post1)"] +qldb = ["mypy-boto3-qldb (==1.17.103.post1)"] +qldb-session = ["mypy-boto3-qldb-session (==1.17.103.post1)"] +quicksight = ["mypy-boto3-quicksight (==1.17.103.post1)"] +ram = ["mypy-boto3-ram (==1.17.103.post1)"] +rds = ["mypy-boto3-rds (==1.17.103.post1)"] +rds-data = ["mypy-boto3-rds-data (==1.17.103.post1)"] +redshift = ["mypy-boto3-redshift (==1.17.103.post1)"] +redshift-data = ["mypy-boto3-redshift-data (==1.17.103.post1)"] +rekognition = ["mypy-boto3-rekognition (==1.17.103.post1)"] +resource-groups = ["mypy-boto3-resource-groups (==1.17.103.post1)"] +resourcegroupstaggingapi = ["mypy-boto3-resourcegroupstaggingapi (==1.17.103.post1)"] +robomaker = ["mypy-boto3-robomaker (==1.17.103.post1)"] +route53 = ["mypy-boto3-route53 (==1.17.103.post1)"] +route53domains = ["mypy-boto3-route53domains (==1.17.103.post1)"] +route53resolver = ["mypy-boto3-route53resolver (==1.17.103.post1)"] +s3 = ["mypy-boto3-s3 (==1.17.103.post1)"] +s3control = ["mypy-boto3-s3control (==1.17.103.post1)"] +s3outposts = ["mypy-boto3-s3outposts (==1.17.103.post1)"] +sagemaker = ["mypy-boto3-sagemaker (==1.17.103.post1)"] +sagemaker-a2i-runtime = ["mypy-boto3-sagemaker-a2i-runtime (==1.17.103.post1)"] +sagemaker-edge = ["mypy-boto3-sagemaker-edge (==1.17.103.post1)"] +sagemaker-featurestore-runtime = ["mypy-boto3-sagemaker-featurestore-runtime (==1.17.103.post1)"] +sagemaker-runtime = ["mypy-boto3-sagemaker-runtime (==1.17.103.post1)"] +savingsplans = ["mypy-boto3-savingsplans (==1.17.103.post1)"] +schemas = ["mypy-boto3-schemas (==1.17.103.post1)"] +sdb = ["mypy-boto3-sdb (==1.17.103.post1)"] +secretsmanager = ["mypy-boto3-secretsmanager (==1.17.103.post1)"] +securityhub = ["mypy-boto3-securityhub (==1.17.103.post1)"] +serverlessrepo = ["mypy-boto3-serverlessrepo (==1.17.103.post1)"] +service-quotas = ["mypy-boto3-service-quotas (==1.17.103.post1)"] +servicecatalog = ["mypy-boto3-servicecatalog (==1.17.103.post1)"] +servicecatalog-appregistry = ["mypy-boto3-servicecatalog-appregistry (==1.17.103.post1)"] +servicediscovery = ["mypy-boto3-servicediscovery (==1.17.103.post1)"] +ses = ["mypy-boto3-ses (==1.17.103.post1)"] +sesv2 = ["mypy-boto3-sesv2 (==1.17.103.post1)"] +shield = ["mypy-boto3-shield (==1.17.103.post1)"] +signer = ["mypy-boto3-signer (==1.17.103.post1)"] +sms = ["mypy-boto3-sms (==1.17.103.post1)"] +sms-voice = ["mypy-boto3-sms-voice (==1.17.103.post1)"] +snowball = ["mypy-boto3-snowball (==1.17.103.post1)"] +sns = ["mypy-boto3-sns (==1.17.103.post1)"] +sqs = ["mypy-boto3-sqs (==1.17.103.post1)"] +ssm = ["mypy-boto3-ssm (==1.17.103.post1)"] +ssm-contacts = ["mypy-boto3-ssm-contacts (==1.17.103.post1)"] +ssm-incidents = ["mypy-boto3-ssm-incidents (==1.17.103.post1)"] +sso = ["mypy-boto3-sso (==1.17.103.post1)"] +sso-admin = ["mypy-boto3-sso-admin (==1.17.103.post1)"] +sso-oidc = ["mypy-boto3-sso-oidc (==1.17.103.post1)"] +stepfunctions = ["mypy-boto3-stepfunctions (==1.17.103.post1)"] +storagegateway = ["mypy-boto3-storagegateway (==1.17.103.post1)"] +sts = ["mypy-boto3-sts (==1.17.103.post1)"] +support = ["mypy-boto3-support (==1.17.103.post1)"] +swf = ["mypy-boto3-swf (==1.17.103.post1)"] +synthetics = ["mypy-boto3-synthetics (==1.17.103.post1)"] +textract = ["mypy-boto3-textract (==1.17.103.post1)"] +timestream-query = ["mypy-boto3-timestream-query (==1.17.103.post1)"] +timestream-write = ["mypy-boto3-timestream-write (==1.17.103.post1)"] +transcribe = ["mypy-boto3-transcribe (==1.17.103.post1)"] +transfer = ["mypy-boto3-transfer (==1.17.103.post1)"] +translate = ["mypy-boto3-translate (==1.17.103.post1)"] +waf = ["mypy-boto3-waf (==1.17.103.post1)"] +waf-regional = ["mypy-boto3-waf-regional (==1.17.103.post1)"] +wafv2 = ["mypy-boto3-wafv2 (==1.17.103.post1)"] +wellarchitected = ["mypy-boto3-wellarchitected (==1.17.103.post1)"] +workdocs = ["mypy-boto3-workdocs (==1.17.103.post1)"] +worklink = ["mypy-boto3-worklink (==1.17.103.post1)"] +workmail = ["mypy-boto3-workmail (==1.17.103.post1)"] +workmailmessageflow = ["mypy-boto3-workmailmessageflow (==1.17.103.post1)"] +workspaces = ["mypy-boto3-workspaces (==1.17.103.post1)"] +xray = ["mypy-boto3-xray (==1.17.103.post1)"] [[package]] name = "botocore" -version = "1.20.102" +version = "1.20.103" description = "Low-level, data-driven core of boto 3." category = "main" optional = false @@ -408,8 +408,8 @@ crt = ["awscrt (==0.11.24)"] [[package]] name = "botocore-stubs" -version = "1.20.102.post1" -description = "Type annotations for botocore 1.20.102, generated by mypy-boto3-buider 4.21.0" +version = "1.20.103.post1" +description = "Type annotations for botocore 1.20.103, generated by mypy-boto3-buider 4.22.1" category = "dev" optional = false python-versions = ">=3.6" @@ -550,7 +550,7 @@ python-versions = "*" [[package]] name = "dcicsnovault" -version = "4.8.1b0" +version = "4.8.1" description = "Storage support for 4DN Data Portals." category = "main" optional = false @@ -2109,7 +2109,7 @@ test = ["zope.testing"] [metadata] lock-version = "1.1" python-versions = ">=3.6.1,<3.7" -content-hash = "daeb56da7c345facf5873af124b87356700a8f3641fbcab72913afabb3b93b23" +content-hash = "736c54125a13491741dc3ee99dc0714f83d8050443ae55a8c9ef2aceac1a311c" [metadata.files] apipkg = [ @@ -2153,16 +2153,16 @@ boto3 = [ {file = "boto3-1.17.53.tar.gz", hash = "sha256:1d26f6e7ae3c940cb07119077ac42485dcf99164350da0ab50d0f5ad345800cd"}, ] boto3-stubs = [ - {file = "boto3-stubs-1.17.102.post1.tar.gz", hash = "sha256:938936682238359d419c4936d81a696ed225ac42d46f564bd5f213aabb106af3"}, - {file = "boto3_stubs-1.17.102.post1-py3-none-any.whl", hash = "sha256:27c8836a06d6c20d9e992f7e6dbc50b340fa5b80849b67de484226b0de0ca69d"}, + {file = "boto3-stubs-1.17.103.post1.tar.gz", hash = "sha256:715ce7c1e5e1468d90d6633e2a8847ceea160c628fdcf0f9566e61596fd701d8"}, + {file = "boto3_stubs-1.17.103.post1-py3-none-any.whl", hash = "sha256:9dfc04e7a35ecad684e40e0f3357d485c5db2958fc09e4765fa3e556b3734b1d"}, ] botocore = [ - {file = "botocore-1.20.102-py2.py3-none-any.whl", hash = "sha256:bdf08a4f7f01ead00d386848f089c08270499711447569c18d0db60023619c06"}, - {file = "botocore-1.20.102.tar.gz", hash = "sha256:2f57f7ceed1598d96cc497aeb45317db5d3b21a5aafea4732d0e561d0fc2a8fa"}, + {file = "botocore-1.20.103-py2.py3-none-any.whl", hash = "sha256:5b39773056a94f85e884a658a5126bb4fee957e31d98b69c255b137eb9f11d6b"}, + {file = "botocore-1.20.103.tar.gz", hash = "sha256:afbfe10fcd580224016d652330db21e7d89099181a437c9ec588b5b7cb3ea644"}, ] botocore-stubs = [ - {file = "botocore-stubs-1.20.102.post1.tar.gz", hash = "sha256:aa0ee90660ff4dae63402fd3924ca9bfff5f8ab30601db60058e671216e30705"}, - {file = "botocore_stubs-1.20.102.post1-py3-none-any.whl", hash = "sha256:9fbb89f5053d20ffafb48d5c29365b5097a6f63a5f72ca8bea47ab020b15c637"}, + {file = "botocore-stubs-1.20.103.post1.tar.gz", hash = "sha256:d66554becc1e44572e197f769883214382d79cf24d5fa3e6670fddf13f3506a4"}, + {file = "botocore_stubs-1.20.103.post1-py3-none-any.whl", hash = "sha256:52b605546ba546c9c72572a11c0a307a25a65d33c42e03fa20dc97bde8d8c71b"}, ] cached-property = [ {file = "cached-property-1.5.2.tar.gz", hash = "sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130"}, @@ -2318,8 +2318,8 @@ dcicpyvcf = [ {file = "dcicpyvcf-1.0.0.tar.gz", hash = "sha256:c5bf8d585002ab3b95d13a47803376b456b931865e4189c38a18cca47b108449"}, ] dcicsnovault = [ - {file = "dcicsnovault-4.8.1b0-py3-none-any.whl", hash = "sha256:2d7707f40e69e9b1684b65ee6b1889d61f0beae8771bc1facc56b3be68a38d21"}, - {file = "dcicsnovault-4.8.1b0.tar.gz", hash = "sha256:3e071b4f7cf6df579db8f3d1d5a7d9c5cd10d5bf0630cd6cefd65b823a495fb1"}, + {file = "dcicsnovault-4.8.1-py3-none-any.whl", hash = "sha256:109b3d7214add70a048b5f284d7ce4c51c02a3b56fdb2a4b51f6485b9f907d9a"}, + {file = "dcicsnovault-4.8.1.tar.gz", hash = "sha256:e229f54f09eb68a40d90172cefc0fecc580890867b6b32e63817e3ca59525a91"}, ] dcicutils = [ {file = "dcicutils-1.19.0-py3-none-any.whl", hash = "sha256:af6e2b13ea92857140fe98a18a06bfe41e7fdd1c4c065e0f74368aed5b2983c4"}, diff --git a/pyproject.toml b/pyproject.toml index a943754c20..5c2c403714 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ certifi = ">=2020.11.8" chardet = "3.0.4" colorama = "0.3.3" dcicpyvcf = "1.0.0" -dcicsnovault = "4.8.1b0" +dcicsnovault = "4.8.1" dcicutils = "^1.19.0" elasticsearch = "6.8.1" execnet = "1.4.1" From c218354bb81f3a3d73e10da7b82ce6c47bd1355b Mon Sep 17 00:00:00 2001 From: Kent Pitman Date: Wed, 7 Jul 2021 07:56:36 -0400 Subject: [PATCH 120/120] Support for INITIAL_DEPLOY environment variable in deploy/docker/production/entrypoint_deployment.sh. Add scripts/build-docker-test script in support of a build-docker-test make target. --- Makefile | 11 +++- .../production/entrypoint_deployment.sh | 6 +- scripts/build-docker-test | 61 +++++++++++++++++++ 3 files changed, 75 insertions(+), 3 deletions(-) create mode 100755 scripts/build-docker-test diff --git a/Makefile b/Makefile index 7ab6856671..8c7e77c13f 100644 --- a/Makefile +++ b/Makefile @@ -196,10 +196,19 @@ rebuild-docker-production: docker build -t ${ENV_NAME}:latest . --no-cache make tag-and-push-docker-production +build-docker-test: + # This will do the equivalent of + # make ecr-login AWS_ACCOUNT= + # make build-docker-production AWS_ACCOUNT= ENV_NAME= + # but it has to do the login inside the script, we can't do it separately here + # because it has to infer the correct AWS_ACCOUNT and ENV_NAME by nosing into + # ~/.aws_test/test_creds.sh looking for ACCOUNT_NUMBER (note: not AWS_ACCOUNT) and ENV_NAME. + scripts/build-docker-test --login # The login must be done inside the script, after inferring account number + build-docker-production: @echo "Making build-docker-production AWS_ACCOUNT=${AWS_ACCOUNT} ENV_NAME=${ENV_NAME} ..." docker build -t ${ENV_NAME}:latest . - make tag-and-push-docker-production + make tag-and-push-docker-production ENV_NAME=${ENV_NAME} AWS_ACCOUNT=${AWS_ACCOUNT} tag-and-push-docker-production: @echo "Making tag-and-push-docker-production AWS_ACCOUNT=${AWS_ACCOUNT} ENV_NAME=${ENV_NAME} ..." diff --git a/deploy/docker/production/entrypoint_deployment.sh b/deploy/docker/production/entrypoint_deployment.sh index a9acec7a25..74aad6465b 100644 --- a/deploy/docker/production/entrypoint_deployment.sh +++ b/deploy/docker/production/entrypoint_deployment.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash echo "Running a CGAP deployment on the given environment" @@ -15,7 +15,9 @@ poetry run create-mapping-on-deploy production.ini --app-name app # Load Data (based on development.ini, for now just master-inserts) # Not necessary after first deploy -# poetry run load-data production.ini --app-name app --prod +if [ -n "${INITIAL_DEPLOYMENT}" ]; then + poetry run load-data production.ini --app-name app --prod +fi # Load access keys # Note that the secret name must match that which was created for this environment diff --git a/scripts/build-docker-test b/scripts/build-docker-test new file mode 100755 index 0000000000..3b58bc9550 --- /dev/null +++ b/scripts/build-docker-test @@ -0,0 +1,61 @@ +#!/bin/bash -f + +aws_account= +env_name= +do_login= +do_help= +creds_file=$HOME/.aws_test/test_creds.sh + +if [ -f "${creds_file}" ]; then + # Default the values of these variables by peeking in the test_creds.sh script + aws_account=`grep 'export ACCOUNT_NUMBER=' ${creds_file} | sed -E 's|^export ACCOUNT_NUMBER=(.*)$|\1|'` + env_name=`grep 'export ENV_NAME=' ${creds_file} | sed -E 's|^export ENV_NAME=(.*)$|\1|'` +fi + +while [ $# -gt 0 ]; do + if [ "$1" = "--aws_account" ]; then + aws_account=$2 + shift 2 + elif [ "$1" = "--env_name" ]; then + env_name=$2 + shift 2 + elif [ "$1" = "--login" ]; then + do_login=TRUE + shift 1 + elif [ "$1" = "--help" ]; then + do_help=TRUE + shift 1 + else + do_help=TRUE + break + fi +done + +if [ -n "${do_help}" ]; then + echo "Syntax: $0 { --aws_account | --env_name | --login | --help }" + echo "" + echo " This will execute 'make build-docker-production AWS_ACCOUNT= ENV_NAME='." + echo " If --login is given, 'make ecr-login AWS_ACCOUNT=' will be done first." + echo " If unspecified, defaults to '${aws_account}' (from 'export ACCOUNT_NUMBER=...' in ${creds_file})." + echo " If unspecified, defaults to '${env_name}' (from 'export ENV_NAME=...' in ${creds_file}.)" + if [ ! -f "${creds_file}" ]; then + echo " NOTE: The file ${creds_file} does not exist." + fi + echo "" + exit 1 +fi + +if [ -z "${aws_account}" ]; then + echo "--aws_account was not given to $0 and could not be found in ~/.aws_test/test_creds.sh" + exit 1 +fi + +if [ -z "${env_name}" ]; then + echo "--env_name was not given to $0 and could not be found in ~/.aws_test/test_creds.sh." + exit 1 +fi + +if [ -n "${do_login}" ]; then + make ecr-login AWS_ACCOUNT="${aws_account}" +fi +make build-docker-production AWS_ACCOUNT="${aws_account}" ENV_NAME="${env_name}"