Merge pull request #142 from dbmi-bgm/kmp_pytest_4-5-0_prep

Another step in preparing for a possible shift to pytest 4.5.0. (Does not yet include top level imports fixes.)

Makefile

@@ -47,7 +47,7 @@ build-dev:  # same as build, but sets up locust as well
 
 macbuild-dev:  # same as macbuild, but sets up locust as well
 	make macbuild
-	pip install locust
+	make build-locust
 
 build-locust:  # just pip installs locust - may cause instability
 	pip install locust
@@ -102,7 +102,7 @@ help:
 	@make info
 
 info:
-	@: $(info Printing some info on how to use make)
+	@: $(info Here are some 'make' options:)
 	   $(info - Use 'make aws-ip-ranges' to download latest ip range information. Invoked automatically when needed.)
 	   $(info - Use 'make build' (or 'make macbuild' on OSX Catalina) to build only application dependencies.)
 	   $(info - Use 'make build-dev' (or 'make macbuild-dev' on OSX Catalina) to build all dependencies, even locust.)

bin/test

@@ -32,9 +32,12 @@ else
     echo "      You may need '--setup-moto' if moto server doesn't start."
 fi
 
+echo "TRAVIS_JOB_ID = ${TRAVIS_JOB_ID}"
 if [ "${TRAVIS_JOB_ID}" = "" -o "$(echo "$TRAVIS_JOB_ID" | cut -c '1-4')" = "4dn-" ]; then
     export TRAVIS_JOB_ID=4dn-`date "+%m%d%H%M%Y%S"`
-    echo "NOTE: Simulating TRAVIS_JOB_ID=$TRAVIS_JOB_ID"
+    echo "NOTE: Simulating fresh TRAVIS_JOB_ID=${TRAVIS_JOB_ID}"
+else
+    echo "NOTE: Using existing TRAVIS_JOB_ID."
 fi
 
 python $(dirname $0)/test.py "$@"

conftest.py

@@ -1,4 +1,6 @@
 import pytest
+import tempfile
+
 
 def pytest_addoption(parser):
     parser.addoption("--es", action="store", default="", dest='es',
@@ -15,3 +17,10 @@ def remote_es(request):
 @pytest.fixture(scope='session')
 def aws_auth(request):
     return request.config.getoption("--aws-auth")
+
+
+def pytest_configure():
+    # This adjustment is important to set the default choice of temporary filenames to a nice short name
+    # because without it some of the filenames we generate end up being too long, and critical functionality
+    # ends up failing. Some socket-related filenames, for example, seem to have length limits. -kmp 5-Jun-2020
+    tempfile.tempdir = '/tmp'

deploy/travis_after_all.py

@@ -121,7 +121,7 @@ def get_token():
         log.error("Others Failed")
         os.environ[BUILD_AGGREGATE_STATUS] = "others_failed"
     else:
-        log.warn("Others Unknown")
+        log.warning("Others Unknown")
         os.environ[BUILD_AGGREGATE_STATUS] = "unknown"
     # since python is subprocess, env variables are exported back via file
     with open(".to_export_back", "w") as export_var:

development.ini

@@ -11,7 +11,6 @@ load_test_only = true
 create_tables = true
 testing = true
 postgresql.statement_timeout = 20
-# indexer.processes =
 mpindexer = true
 indexer = true
 elasticsearch.aws_auth = false
@@ -24,6 +23,8 @@ pyramid.default_locale_name = en
 # most deployments use: "load_test_data = encoded.loadxl:load_test_data"
 load_test_data = encoded.loadxl:load_local_data
 encoded_version = 100.200.300
+snovault_version = 200.300.400
+utils_version = 300.400.500
 eb_app_version = app-v-development-simulation
 
 [pipeline:debug]

pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 # Note: Various modules refer to this system as "encoded", not "cgap-portal".
 name = "encoded"
-version = "2.1.10"
+version = "2.1.11"
 description = "Clinical Genomics Analysis Platform"
 authors = ["4DN-DCIC Team <[email protected]>"]
 license = "MIT"
@@ -42,7 +42,7 @@ certifi = ">=2020.4.5.2"
 chardet = "3.0.4"
 colorama = "0.3.3"
 dcicpyvcf = "1.0.0"
-dcicsnovault = ">=3.1.1,<4"
+dcicsnovault = ">=3.1.4,<4"
 dcicutils = ">=0.31.1,<1"
 docutils = "0.12"
 elasticsearch = "5.5.3"
@@ -91,7 +91,7 @@ rutter = ">=0.2,<1"
 s3transfer = "^0.2.0"
 simplejson = "^3.17.0"
 SPARQLWrapper = "1.7.6"
-SQLAlchemy = "1.2.16"
+SQLAlchemy = "1.3.16"
 structlog = ">=18.1.0,<20"
 submit4dn = "0.9.7"
 subprocess-middleware = ">=0.3,<1"
@@ -103,7 +103,7 @@ translationstring = "1.3"
 uptime = ">=3.0.1,<4"
 urllib3 = "^1.24.3"
 venusian = "^1.2.0"
-waitress = "1.2.0"
+waitress = "1.2.0"  # 1.4.3 had lots of problems, so pin this -kmp 18-May-2020
 WebOb = "1.8.5"
 WebTest = "^2.0.21"
 wheel = "0.29.0"
@@ -112,13 +112,13 @@ xlrd = "^1.0.0"
 xlwt = "1.2.0"
 "zope.deprecation" = "4.4.0"
 "zope.interface" = "4.6.0"
-"zope.sqlalchemy" = "^1.2"
+"zope.sqlalchemy" = "1.3"
 
 [tool.poetry.dev-dependencies]
 coverage = ">=5.2"
 codacy-coverage = ">=1.3.11"
 coveralls = ">=2.1.1"
-# flake8 = "3.7.8"
+flake8 = "^3.7.8"
 flaky = "3.6.1"
 # flask only for moto[server]
 flask = ">=1.1.1"

pytest.ini

@@ -9,6 +9,7 @@ markers =
     indexing: mark a test as an indexing test (deselect with '-m "not indexing"')
     ingestion: mark a test as an ingestion test (deselect with '-m "not ingestion"')
     performance: mark a test as a performance test (deselect with '-m "not performance"')
+    schema: mark a test as a schema-related test (deselect with '-m "not schema"')
     slow: mark a test as slow (deselect with '-m "not slow"')
     storage: mark a test as about storage (deselect with '-m "not storage"')
     working: mark a test as working (deselect with '-m "not working"')

setup_eb.py

@@ -10,7 +10,7 @@
 PYPROJECT_TOML = toml.decoder.load(os.path.join(ROOT_DIR, 'pyproject.toml'))
 POETRY_DATA = PYPROJECT_TOML['tool']['poetry']
 
-_CARET_MATCH = re.compile(r"[\^]([0-9]+)([.].*)$")
+_CARET_MATCH = re.compile(r"[\^]([0-9]+)([.].*)?$")
 
 
 def fix_requirement(requirement):

src/encoded/authentication.py

@@ -46,6 +46,20 @@
 CRYPT_CONTEXT = __name__ + ':crypt_context'
 
 
+JWT_ENCODING_ALGORITHM = 'HS256'
+
+# Might need to keep a list of previously used algorithms here, not just the one we use now.
+# Decryption algorithm used to default to a long list,
+# but more recent versions of jwt library say we should stop assuming that.
+#
+# In case it goes away, as far as I can tell, the default for decoding from their
+# default_algorithms() method used to be: ['ES512', 'RS384', 'HS512', 'ES256', 'none',
+# 'RS256', 'PS512', 'ES384', 'HS384', 'ES521', 'PS384', 'HS256', 'PS256', 'RS512']
+# -kmp 15-May-2020
+
+# TODO: JWT_DECODING_ALGORITHMS = [JWT_ENCODING_ALGORITHM]
+
+
 def includeme(config):
     config.include('.edw_hash')
     setting_prefix = 'passlib.'
@@ -222,6 +236,7 @@ def get_token_info(self, token, request):
             if auth0_client and auth0_secret:
                 # leeway accounts for clock drift between us and auth0
                 payload = jwt.decode(token, b64decode(auth0_secret, '-_'),
+                                     # algorithms=JWT_DECODING_ALGORITHMS
                                      audience=auth0_client, leeway=30)
                 if 'email' in payload and self.email_is_partners_or_hms(payload):
                     request.set_property(lambda r: False, 'auth0_expired')
@@ -427,7 +442,9 @@ def impersonate_user(context, request):
         'aud': auth0_client,
     }
 
-    id_token = jwt.encode(jwt_contents, b64decode(auth0_secret, '-_'), algorithm='HS256')
+    id_token = jwt.encode(jwt_contents, b64decode(auth0_secret, '-_'),
+                          algorithm=JWT_ENCODING_ALGORITHM
+						  )
     user_properties['id_token'] = id_token.decode('utf-8')
 
     return user_properties

src/encoded/commands/add_date_created.py

@@ -12,11 +12,17 @@
     %(prog)s development.ini --app-name app
 
 """
-from future.utils import iteritems
-from pyramid.traversal import resource_path
+
+import argparse
 import logging
 import pytz
 
+from future.utils import iteritems
+from pyramid import paster
+from pyramid.traversal import resource_path
+from webtest import TestApp
+
+
 pacific = pytz.timezone('US/Pacific')
 
 EPILOG = __doc__
@@ -25,8 +31,6 @@
 
 
 def internal_app(configfile, app_name=None, username=None):
-    from webtest import TestApp
-    from pyramid import paster
     app = paster.get_app(configfile, app_name)
     if not username:
         username = 'IMPORT'
@@ -86,8 +90,7 @@ def run(testapp, collections=None, exclude=None, dry_run=False):
 
 
 def main():
-    import argparse
-    parser = argparse.ArgumentParser(  # noqa - PyCharm wrongly thinks the formatter_class is specified wrong here.
+    parser = argparse.ArgumentParser(  # noqa - PyCharm wrongly thinks the formatter_class is invalid
         description="Fix date_created", epilog=EPILOG,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )