requirements merge conflict

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.8
+FROM python:3.8.2
 ENV PYTHONUNBUFFERED=1
 COPY ./requirements-dev.txt /usr/src/apps/StoryMapJS/
 COPY ./.localstack/KnightLabRootCA.crt /usr/src/apps/StoryMapJS/

celery.Dockerfile → Dockerfile.huey

@@ -1,4 +1,4 @@
-FROM python:3.8
+FROM python:3.8.2
 ENV PYTHONUNBUFFERED=1
 COPY ./requirements-dev.txt /usr/src/apps/StoryMapJS/
 #COPY ./.localstack/KnightLabRootCA.crt /usr/src/apps/StoryMapJS/

docker-compose.yml

@@ -1,4 +1,10 @@
-version: '2.1'
+version: '3.9'
+
+
+volumes:
+  localstack-data:
+    external: true
+
 
 services:
 
@@ -9,7 +15,7 @@ services:
       - "443:5000"
     links:
       - pg:pg
-      - localstack:localstack
+      #- localstack:localstack
     volumes:
       - ./storymap:/usr/src/apps/StoryMapJS/storymap
       - ./dist:/usr/src/apps/StoryMapJS/dist
@@ -19,30 +25,36 @@ services:
       DEBUG: 'true'
       PYTHONPATH: .
       AWS_ACCESS_KEY_ID: xyz
-      AWS_DEFAULT_REGION: xyz
+      AWS_DEFAULT_REGION: us-east-1
       AWS_SECRET_ACCESS_KEY: xyz
       REDIS_HOST: redis
       RABBITMQ_HOST: rabbitmq
     env_file: .env
     command: gunicorn -b :5000 --workers 1 --reload --log-level debug storymap.core.wsgi:application --keyfile KnightLabRootCA.key --certfile KnightLabRootCA.crt
     depends_on:
       - pg
-      - localstack
+      #- localstack
 
   localstack:
     container_name: "${LOCALSTACK_DOCKER_NAME-localstack_main}"
-    image: localstack/localstack-light
+    image: localstack/localstack
     ports:
       - "4566:4566"
       - "4571:4571"
     environment:
       - LOCALSTACK_SERVICES=s3
-      - LOCALSTACK_DATA_DIR=/tmp/localstack/data
+      - LOCALSTACK_LEGACY_PERSISTENCE=1
+      - LOCALSTACK_PERSISTENCE=1
+      # - LOCALSTACK_DATA_DIR=/localstack/data
     volumes:
-      - ./.localstack/:/tmp/localstack
+      #- ./.localstack/:/var/lib/localstack
+      - localstack-data:/var/lib/localstack
+      #- ./.localstack/:/localstack
+      #- "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack"
+
 
   pg:
-    image: postgres:11.10
+    image: postgres:12.17
     restart: always
     environment:
       POSTGRES_USER: storymap
@@ -56,7 +68,7 @@ services:
   huey:
     build:
       context: .
-      dockerfile: celery.Dockerfile
+      dockerfile: Dockerfile.huey
     restart: always
     working_dir: /usr/src/apps/StoryMapJS
     volumes:
@@ -65,37 +77,8 @@ services:
     environment:
       PYTHONPATH: .
       AWS_ACCESS_KEY_ID: xyz
-      AWS_DEFAULT_REGION: xyz
+      AWS_DEFAULT_REGION: us-east-1
       AWS_SECRET_ACCESS_KEY: xyz
     command: huey_consumer.py --logfile=storymap/huey.log storymap.tasks.huey
-    depends_on:
-      - localstack
-
-  #celery:
-  #  build:
-  #    context: .
-  #    dockerfile: celery.Dockerfile
-  #  restart: always
-  #  working_dir: /usr/src/apps/StoryMapJS
-  #  volumes:
-  #    - ./storymap:/usr/src/apps/StoryMapJS/storymap
-  #  env_file: .env
-  #  environment:
-  #    PYTHONPATH: .
-  #    AWS_ACCESS_KEY_ID: xyz
-  #    AWS_DEFAULT_REGION: xyz
-  #    AWS_SECRET_ACCESS_KEY: xyz
-  #    REDIS_HOST: redis
-  #    RABBITMQ_HOST: rabbitmq
-  #  command: celery -A storymap.tasks worker -l debug
-  #  depends_on:
-  #    - localstack
-  #    - rabbitmq
-
-
-  #redis:
-  #  image: redis
-
-  #rabbitmq:
-  #  image: rabbitmq
-  #  command: rabbitmq-server
+    #depends_on:
+    #  - localstack

requirements-dev.txt

@@ -1,7 +1,7 @@
 asn1crypto==0.24.0
 bcrypt==3.1.6
 boto3==1.14.10
-celery==5.2.6
+#boto3
 cffi==1.14.1
 chardet==3.0.4
 Click==8.0.3
@@ -23,10 +23,7 @@ ipaddress==1.0.17
 ipython==7.16.3
 itsdangerous==1.1.0
 Jinja2==2.11.3
-livereload==1.0.0
 MarkupSafe==1.1.1
-mock==2.0.0
-moto==0.4.25
 oauth2client==4.1.3
 paramiko==2.4.2
 pbr==1.10.0
@@ -38,8 +35,6 @@ pycparser==2.14
 PyNaCl==1.3.0
 pyOpenSSL==20.0.1
 python-slugify==1.1.2
-redis==4.2.2
-redislite==6.0.674960
 requests==2.24.0
 robotframework==3.2.1
 robotframework-selenium2library==3.0.0

requirements.txt

@@ -1,58 +1,10 @@
-asn1crypto==0.24.0
-bcrypt==3.1.6
-boto3==1.14.10
-celery==5.2.6
-cffi==1.14.1
-chardet==3.0.4
-Click==8.0.3
-decorator==4.0.10
-docopt==0.6.1
-enum34==1.1.6
-fabric==2.5.0
-Flask==1.1.2
-flask-cors
-funcsigs==1.0.2
-google-api-python-client==1.9.3
-gunicorn==20.0.4
-httpretty==0.8.10
-huey==2.4.3
-idna==2.7
-ipaddress==1.0.17
-ipython==7.16.3
-itsdangerous==1.1.0
-Jinja2==2.11.3
-livereload==1.0.0
-MarkupSafe==1.1.1
-mock==2.0.0
-moto==0.4.25
-oauth2client==4.1.3
-paramiko==2.4.2
-pbr==1.10.0
-ply==3.4
-psycopg2-binary==2.8.5
-pyasn1==0.4.8
-pyasn1-modules==0.2.8
-pycparser==2.14
-PyNaCl==1.3.0
-pyOpenSSL==20.0.1
-python-slugify==1.1.2
-redis==4.2.2
-redislite==6.0.674960
-requests==2.24.0
-robotframework==3.2.1
-robotframework-selenium2library==3.0.0
-robotframework-seleniumlibrary==4.4.0
-selenium==3.141.0
-sentinels==1.0.0
-simplejson==3.6.5
-six==1.15.0
-slimit==0.8.1
-tornado==3.1
-unicode-slugify==0.1.3
-unicodecsv==0.14.1
-Unidecode==0.4.17
-uritemplate==3.0.1
-urllib3==1.24.2
-Werkzeug==1.0.1
-xmltodict==0.10.2
-gunicorn==20.0.4
+boto3==1.34.*
+Flask==3.0.*
+Flask-Cors==4.0.*
+google-api-python-client==2.115.*
+gunicorn==21.2.*
+huey==2.5.*
+oauth2client==4.1.*
+psycopg2-binary==2.9.*
+requests==2.31.*
+slugify==0.0.*

scripts/makebuckets.sh

@@ -1,3 +1,3 @@
 #!/usr/bin/env sh
-aws --endpoint-url=http://localhost:4566 s3 mb s3://uploads.knilab.com
-aws --endpoint-url=http://localhost:4566 s3 mb s3://cdn.knilab.com
+aws --endpoint-url=http://localhost:4566 s3 mb s3://uploads.knilab.com --region us-east-1
+aws --endpoint-url=http://localhost:4566 s3 mb s3://cdn.knilab.com --region us-east-1

storymap/api.py

@@ -70,8 +70,6 @@ def create_app():
 app = create_app()
 app.config.from_envvar('FLASK_SETTINGS_FILE')
 settings = sys.modules[settings_module]
-# LOCAL_STORAGE_MODE is no longer supported. Use localstack instead.
-app.config['LOCAL_STORAGE_MODE'] = settings.LOCAL_STORAGE_MODE
 app.config['TEST_MODE'] = settings.TEST_MODE
 examples_json = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'examples.json')
 faq_json = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'faq.json')
@@ -288,7 +286,6 @@ def google_auth_verify():
                 "following information to [email protected]: " \
                 "stg-storymap.knightlab.com unauthorized %s" % uid)
 
-        print('upsert user', uid)
 
         # Upsert user record
         user = get_user(uid, db=db())
@@ -853,8 +850,6 @@ def storymap_image_save(user, id):
     import binascii
     try:
         name, content = _request_get_required('name', 'content')
-        print(content[:30])
-
         m = re.match('data:(.+);base64,(.+)', content)
         if m:
             content_type = m.group(1)

storymap/connection.py

@@ -12,11 +12,11 @@
 
 DEFAULT_USER_QUERY_LIMIT = 20
 
-# Get settings module
-settings = sys.modules[os.environ['FLASK_SETTINGS_MODULE']]
 
 
-def pg_conn():
+def pg_conn(settings=None):
+    if settings is None:
+        settings = sys.modules[os.environ['FLASK_SETTINGS_MODULE']]
     return psycopg2.connect(
         host=settings.DATABASES['pg']['HOST'],
         port=settings.DATABASES['pg']['PORT'],
@@ -37,9 +37,7 @@ def create_pg_user(uid, uname, *, db, migrated=1, storymaps=None):
     db.commit()
 
 
-def migrate_pg(*, db, drop_table=False):
-    # Checked max length of uname in mongo was 71 characters
-    raise Exception('Migration currently unavailable')
+def create_users_table(*, db, drop_table=False):
     if drop_table:
         with db.cursor() as cursor:
             cursor.execute('DROP TABLE IF EXISTS users;')
@@ -49,6 +47,14 @@ def migrate_pg(*, db, drop_table=False):
                 "migrated smallint, storymaps jsonb, " \
                 "CONSTRAINT unique_uid UNIQUE (uid))")
         db.commit()
+    else:
+        with db.cursor() as cursor:
+            cursor.execute(
+                "CREATE TABLE IF NOT EXISTS users " \
+                "(id serial PRIMARY KEY, uid varchar(32), uname varchar(100), " \
+                "migrated smallint, storymaps jsonb, " \
+                "CONSTRAINT unique_uid UNIQUE (uid))")
+        db.commit()
 
 
 def delete_test_user(*, db):

storymap/core/settings.py

@@ -5,7 +5,6 @@
 from os import environ as env
 
 TEST_MODE = env.get('TEST_MODE')
-LOCAL_STORAGE_MODE = False # rip this out if we can
 
 CORE_ROOT = dirname(abspath(__file__))
 PROJECT_ROOT = dirname(dirname(CORE_ROOT))
@@ -14,13 +13,12 @@
 STATIC_URL = env['STATIC_URL']
 CDN_URL = env['CDN_URL']
 
+
+# Default database `pg` is setup for local execution via docker-compose where the
+# postgresql service host is set to `pg`. To run migrations, set the `PG_HOST`
+# environment variable to localhost if executing the scription outside of the docker
+# compose stack.
 DATABASES = {
-    #'default': {
-    #    'ENGINE': env['DB_ENGINE__DEFAULT'],
-    #    'NAME': env['DB_NAME__DEFAULT'],
-    #    'HOST': env['DB_HOST__DEFAULT'],
-    #    'PORT': env['DB_PORT__DEFAULT']
-    #},
     'pg': {
         'HOST': env.get('PG_HOST', 'pg'),
         'PORT': env.get('PG_PORT', '5432'),
@@ -30,12 +28,8 @@
     }
 }
 
-if LOCAL_STORAGE_MODE:
-    AWS_STORAGE_BUCKET_NAME = env['AWS_LOCAL_STORAGE_BUCKET_NAME']
-    AWS_STORAGE_BUCKET_URL = env['AWS_LOCAL_STORAGE_BUCKET_URL']
-else:
-    AWS_STORAGE_BUCKET_NAME = env['AWS_STORAGE_BUCKET_NAME']
-    AWS_STORAGE_BUCKET_URL = env['AWS_STORAGE_BUCKET_URL']
+AWS_STORAGE_BUCKET_NAME = env['AWS_STORAGE_BUCKET_NAME']
+AWS_STORAGE_BUCKET_URL = env['AWS_STORAGE_BUCKET_URL']
 
 AWS_STORAGE_BUCKET_KEY = env['AWS_STORAGE_BUCKET_KEY']
 AWS_ACCESS_KEY_ID = env['AWS_ACCESS_KEY_ID']