adds basic testing scaffolding for unit tests and integration tests (…

…to s3) executable via fab

README.md

@@ -98,3 +98,17 @@ Apparently, Apple removed support for `openssl` in Mac OS X 10.11. Here's the so
 
 Users may be directed to our userinfo page to help with troubleshooting. This page provides information about the user's account and saved storymaps. The endpoint is `https://storymap.knightlab.com/userinfo/`
 
+
+## Using Atlassian localstack for development/testing
+
+NOTE: boto3 is required for localstack. Until transition to boto3 is complete, development and testing involving s3 connections still requires AWS credentials
+
+Be sure to have the aws cli installed (http://docs.aws.amazon.com/cli/latest/userguide/installing.html)
+
+In your StoryMap virtualenvironment, install localstack and create the test bucket:
+
+  * git clone https://github.com/atlassian/localstack.git
+  * cd localstack
+  * make clean install test
+  * make infra
+  * aws --endpoint-url=http://localhost:4572 s3 mb s3://test.knilab.com

core/settings.py

@@ -27,6 +27,7 @@
 AWS_STORAGE_BUCKET_KEY = env['AWS_STORAGE_BUCKET_KEY']
 AWS_ACCESS_KEY_ID = env['AWS_ACCESS_KEY_ID']
 AWS_SECRET_ACCESS_KEY = env['AWS_SECRET_ACCESS_KEY']
+AWS_ENDPOINT_URL = env.get('AWS_ENDPOINT_URL')
 GOOGLE_CLIENT_ID = env['GOOGLE_CLIENT_ID']
 GOOGLE_CLIENT_SECRET = env['GOOGLE_CLIENT_SECRET']
 ADMINS = env['ADMINS'].split()

env.sh.example

@@ -5,6 +5,7 @@ export PROJECT_NAME="StoryMapJS"
 export DB_NAME__DEFAULT="storymapjs"
 export APPLICATION_DOMAINS="not used for local development"
 export FLASK_SECRET_KEY="\xc2\xbd4\xe5UG7\x14\x7f@\xd5\xfa\x9a0\xd9m\xfc\x8b\xbcM\xb2\x1d\xc1e\x3c\xff"
+export AWS_ENDPOINT_URL="http://localhost:4572"
 export AWS_STORAGE_BUCKET_URL="//uploads.knilab.com/"
 export AWS_STORAGE_BUCKET_NAME="uploads.knilab.com"
 export AWS_STORAGE_BUCKET_KEY="storymapjs"

fabfile.py

@@ -2,6 +2,7 @@
 import sys
 import shutil
 from fabric.api import env
+from fabric.operations import local
 
 #
 # Project-specific settings, alter as needed
@@ -29,11 +30,19 @@ def add_paths(*args):
 from fablib import *
 
 @task
-def test(*args,**kwargs):
+def testui(*args,**kwargs):
     if os.path.isdir('robot_tests/logs'):
         shutil.rmtree('robot_tests/logs')
     os.execvp('robot', ('robot', '-d', 'robot_tests/logs') + args + ('robot_tests',))
 
+@task(alias='testint')
+def testntegration(*args, **kwargs):
+    local('python -m tests.integration_tests')
+
+@task
+def unittest(*args, **kwargs):
+    local('python -m tests.unit_tests')
+
 @task
 def prd(*args,**kwargs):
     abort( "you should be deploying with git, not the prd task")

storymap/storage.py

@@ -105,6 +105,7 @@ def get_contents_as_string(src_key):
 
 @_mock_in_test_mode
 def all_keys():
+    print settings.AWS_STORAGE_BUCKET_KEY
     for item in _bucket.list(prefix=settings.AWS_STORAGE_BUCKET_KEY):
         if item.name == key_prefix:
             continue

tests/integration_tests.py

@@ -0,0 +1,99 @@
+"""Integration testing requiring access to S3
+
+Be sure to set the following env vars:
+
+AWS_SECRET_ACCESS_KEY
+AWS_ACCESS_KEY_ID
+AWS_TEST_BUCKET (defaults to test.knilab.com)
+
+Note: we are in the process to transitioning to boto3 in order support localstack based testing and development. To use with localstack, be sure also to set:
+
+AWS_ENDPOINT_URL="http://localhost:4572"
+"""
+import importlib
+import json
+import os
+import sys
+import unittest
+import boto
+import botocore
+import boto3
+
+if __name__ == "__main__":
+    if not os.environ.get('FLASK_SETTINGS_MODULE', ''):
+        os.environ['FLASK_SETTINGS_MODULE'] = 'core.settings.loc'
+
+settings_module = os.environ.get('FLASK_SETTINGS_MODULE')
+try:
+    importlib.import_module(settings_module)
+except ImportError, e:
+    raise ImportError("Could not import settings '%s' (Is it on sys.path?): %s" % (settings_module, e))
+
+settings = sys.modules[os.environ['FLASK_SETTINGS_MODULE']]
+settings.TEST_MODE = False
+settings.AWS_STORAGE_BUCKET_NAME = os.environ.get('AWS_TEST_BUCKET', 'test.knilab.com')
+
+try:
+    from storymap.storage import all_keys, save_from_data
+except boto.exception.S3ResponseError:
+    raise Exception("""
+
+boto2 response error loading module storymap.storage. Check your environment variables
+
+During transition from boto2 to boto3, real connections to S3 are required to test the legacy code. Be sure AWS_SECRET_ACCESS_KEY and AWS_ACCESS_KEY_ID are set
+""")
+
+
+class StorageTestCase(unittest.TestCase):
+
+    def test_list_keys(self):
+        keys = all_keys()
+        # TODO: this is not yet testing anything
+
+    def test_save_from_data(self):
+        file_name = 'test1.json'
+        key_name = '%s/%s' % (settings.AWS_STORAGE_BUCKET_KEY, file_name)
+        content_type = 'application/json'
+        content = json.dumps({ 'test_key': 'test_value' })
+        save_from_data(key_name, content_type, content)
+        endpoint = settings.AWS_ENDPOINT_URL
+        if endpoint:
+            s3 = boto3.resource('s3', endpoint_url=endpoint)
+        else:
+            s3 = boto3.resource('s3')
+        obj = s3.Object(settings.AWS_STORAGE_BUCKET_NAME, key_name)
+        try:
+            self.assertEqual('test_value',
+                json.loads(obj.get()['Body'].read())['test_key'])
+        except botocore.exceptions.ConnectionError:
+            self.fail("""
+boto3 connection error in test. Check your environment variables
+
+Be sure AWS_ENDPOINT_URL points to a valid localized endpoint. If connecting to S3, be sure AWS_ENDPOINT_URL is blank or not set and that AWS_SECRET_ACCESS_KEY and AWS_ACCESS_KEY_ID are set
+""")
+        except Exception as e:
+            if 'NoSuchBucket' in e.message:
+                self.fail("""
+Could not connect. No such bucket: %s
+AWS endpoint: %s
+
+NOTE: StoryMap and these tests do not create the storage bucket. For testing, your endpoint should have a bucket named according to your AWS_TEST_BUCKET environment variable. With localstack, this bucket can be created with the following command:
+
+aws --endpoint-url=http://localhost:4572 s3 mb s3://%s
+""" % (obj.bucket_name, endpoint, settings.AWS_TEST_BUCKET))
+            elif 'NoSuchKey' in e.message:
+                self.fail("""
+No such key error
+
+The `save_from_data` function currently only saves to remote S3. To get this test passing, we will need to migrate to boto3 usage that allows for local storage (via localstack) or remote (to s3)
+""")
+            else:
+                raise
+
+
+def suite():
+    tests = ['test_list_keys', 'test_save_from_data']
+    return unittest.TestSuite(map(StorageTestCase, tests))
+
+if __name__=='__main__':
+    unittest.main()

tests/unit_tests.py

@@ -0,0 +1,10 @@
+import unittest
+
+class NothingTestCase(unittest.TestCase):
+
+    def test_nothing(self):
+        self.assertTrue(True)
+
+
+if __name__=='__main__':
+    unittest.main()