Merge pull request #9 from adamjmcgrath/cloud-storage

Cloud storage

requirements.txt

@@ -13,3 +13,4 @@ python-coveralls
 python-dateutil==2.1
 wtforms==0.6.3
 gaenv==0.1.8.post0
+GoogleAppEngineCloudStorageClient==1.9.22.1

scripts/cloud_storage_migration.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python
+# coding=utf-8
+#
+# Copyright 2011 Friday Film Club. All Rights Reserved.
+
+"""Migrate images from blobstore to cloud storage
+
+You'll need to `pip install python-magic`
+"""
+
+import os
+import sys
+sys.path.append('/usr/local/google_appengine')
+import dev_appserver
+import posixpath
+import cloudstorage
+import magic
+import re
+import getpass
+
+dev_appserver.fix_sys_path()
+
+dir = os.path.dirname(__file__)
+sys.path.insert(0, os.path.join(dir, '../src'))
+
+from google.appengine.ext.remote_api import remote_api_stub
+from google.appengine.ext import blobstore, ndb
+
+GCS_ROOT = '/ffcapp.appspot.com/images/'
+
+os.environ['SERVER_SOFTWARE'] = 'Development (remote_api)/1.0'
+
+import models
+
+APP_NAME = 's~ffcapp'
+RE_SPECIAL_CHARS_ = re.compile(r'[^a-zA-Z0-9 ]')
+os.environ['AUTH_DOMAIN'] = 'gmail.com'
+os.environ['USER_EMAIL'] = '[email protected]'
+
+def slugify(my_string):
+  """Remove special characters and replace spaces with hyphens."""
+  return '-'.join(re.sub(RE_SPECIAL_CHARS_, '', my_string).lower().split(' '))
+
+def auth_func():
+  return (os.environ['USER_EMAIL'], getpass.getpass())
+
+def save_image(blob_key, folder_name, file_name):
+  if not blob_key:
+    print 'ERROR: no blob %s/%s' % (folder_name, file_name)
+    return blob_key
+
+  img_file = blobstore.BlobInfo(blob_key)
+  img_file_o = img_file.open()
+  img_data = img_file_o.read()
+  type = magic.from_buffer(img_data, mime=True)
+
+  file_path = posixpath.join(
+    GCS_ROOT, folder_name, file_name)
+  gcs_file = cloudstorage.open(file_path, 'w', content_type=type)
+  print 'Saving file: %s' % file_path
+  gcs_file.write(img_data)
+  gcs_file.close()
+  img_file_o.close()
+  return blobstore.BlobKey(blobstore.create_gs_key('/gs' + file_path))
+
+
+def migrate_screenshot(q):
+  clue = q.clues[0]
+  if not clue:
+    return
+  title = q.answer_title
+  clue_entity = clue.get()
+  old_key = str(clue_entity.image)
+  clue_entity.image = save_image(
+    clue_entity.image, 'questions', slugify(title) + '-screenshot')
+  clue_entity.put()
+  print ('screenshot: %s | old: %s | new: %s' % (title, old_key, str(clue_entity.image)))
+
+
+def migrate_packshot(q):
+  title = q.answer_title
+  old_key = str(q.packshot)
+  q.packshot = save_image(
+    q.packshot, 'questions', slugify(title) + '-packshot')
+  q.put()
+  print ('packshot: %s | old: %s | new: %s' % (title, old_key, str(q.packshot)))
+
+
+def migrate_questions():
+  for q in models.Question.query():
+    migrate_packshot(q)
+    migrate_screenshot(q)
+
+def migrate_users(curs=None):
+  users, next_curs, more = models.User.query().fetch_page(500,
+                                                          start_cursor=curs)
+  for u in users:
+    old_key = str(u.pic)
+    u.pic = save_image(u.pic, 'profiles', u.username_lower)
+    print ('profile: %s | old: %s | new: %s' % (u.username_lower, old_key, str(u.pic)))
+
+  ndb.put_multi(users)
+
+  if more:
+    migrate_users(next_curs)
+
+
+def migrate_leagues(curs=None):
+  leagues, next_curs, more = models.League.query().fetch_page(500,
+                                                              start_cursor=curs)
+
+  for l in leagues:
+    old_key = str(l.pic)
+    l.pic = save_image(l.pic, 'leagues', l.name_slug)
+    print ('league: %s | old: %s | new: %s' % (l.name_slug, old_key, str(l.pic)))
+
+  ndb.put_multi(leagues)
+
+  if more:
+    migrate_leagues(next_curs)
+
+
+def main():
+  # Use 'localhost:8080' for dev server.
+  remote_api_stub.ConfigureRemoteApi(APP_NAME, '/_ah/remote_api',
+      auth_func, servername='ffcapp.appspot.com')
+
+  migrate_questions()
+  migrate_users()
+  migrate_leagues()
+
+
+if __name__ == '__main__':
+  main()

src/auth.py

@@ -90,7 +90,8 @@ def _login_user(self, data, auth_info, provider):
     if user:
       logging.info('Found existing user to log in')
       # existing user. just log them in and update token.
-      user.populate(**self._to_user_model_attrs(data, provider, False))
+      user.populate(**self._to_user_model_attrs(
+        data, provider, False, user.username_lower))
       user.put()
       self.auth.set_session(self.auth.store.user_to_dict(user), remember=True)
 
@@ -104,7 +105,8 @@ def _login_user(self, data, auth_info, provider):
         logging.info('Updating currently logged in user.')
         user = self.current_user
         user.auth_ids.append(auth_id)
-        user.populate(**self._to_user_model_attrs(data, provider, False))
+        user.populate(**self._to_user_model_attrs(
+          data, provider, False, user.username_lower))
         user.put()
         self.auth.set_session(self.auth.store.user_to_dict(user), remember=True)
 
@@ -116,7 +118,8 @@ def _login_user(self, data, auth_info, provider):
 
         # Authenticate the new user.
         user.auth_ids.append(auth_id)
-        user.populate(**self._to_user_model_attrs(data, provider, True))
+        user.populate(**self._to_user_model_attrs(
+          data, provider, True, user.username_lower))
         user.put()
         self.auth.set_session(self.auth.store.user_to_dict(user), remember=True)
         del self.session['username']
@@ -165,7 +168,7 @@ def _get_consumer_info_for(self, provider):
     return secrets.get_auth_config(
       provider, settings.get_environment(self.request.host))
 
-  def _to_user_model_attrs(self, data, provider, new_user):
+  def _to_user_model_attrs(self, data, provider, new_user, username):
     attrs_map = self.USER_ATTRS[provider]
     user_attrs = {}
     for k, v in data.iteritems():
@@ -182,7 +185,7 @@ def _to_user_model_attrs(self, data, provider, new_user):
 
         if new_user:
           if key == 'pic':
-            value = models.User.blob_from_url(value)
+            value = models.User.blob_from_url(value, username)
           user_attrs.setdefault(key, value)
 
     return user_attrs

src/forms.py

@@ -12,12 +12,12 @@
 import json
 import re
 import posixpath
+import cloudstorage
 
 import webapp2
 from webapp2_extras import auth
-from google.appengine.api import files
-from google.appengine.api import urlfetch
-from google.appengine.ext import ndb
+from google.appengine.api import images, urlfetch
+from google.appengine.ext import blobstore, ndb
 from google.appengine.ext.db import BadKeyError
 from wtforms import fields, Form, validators, widgets
 
@@ -26,6 +26,7 @@
 
 
 _USERNAME_RE = re.compile(r'^[\w\d_]{3,16}$')
+GCS_ROOT = '/ffcapp.appspot.com/images/'
 
 
 def validate_username(form, field):
@@ -94,24 +95,61 @@ def populate_obj(self, obj, name):
 class ImageField(fields.FileField):
   """An image field."""
 
+  def __init__(self, name, gcs_folder, img_size=None, **kwargs):
+    self.gcs_folder = gcs_folder
+    self.img_size = img_size
+    super(ImageField, self).__init__(name, **kwargs)
+
+  def file_name(self, req, obj):
+    img_file = req.POST.get(self.name)
+    return img_file.filename
+
   def populate_obj(self, obj, name):
-    """Populate the object represented by the film field."""
+    """Populate the question model with a GCS image."""
     req = webapp2.get_request()
-    img_file = req.get(self.name)
-    if not img_file:
+    if not req.get(self.name):
       return
-    file_name = files.blobstore.create(mime_type='application/octet-stream')
-    with files.open(file_name, 'a') as f:
-      f.write(img_file)
-    files.finalize(file_name)
 
-    setattr(obj, name, files.blobstore.get_blob_key(file_name))
+    img_file = req.POST.get(self.name)
+    file_name = posixpath.join(
+      GCS_ROOT, self.gcs_folder, self.file_name(req, obj))
+    gcs_file = cloudstorage.open(file_name, 'w', content_type=img_file.type)
+    logging.info('Saving file: %s' % file_name)
+    gcs_file.write(img_file.value)
+    gcs_file.close()
+
+    setattr(obj, name,
+            blobstore.BlobKey(blobstore.create_gs_key('/gs' + file_name)))
+
+
+class ProfileImageField(ImageField):
+
+  def file_name(self, req, obj):
+    return obj.username_lower
+
+
+class PackshotImageField(ImageField):
+
+  def file_name(self, req, obj):
+    return '%s-packshot' % models.slugify(obj.answer_title)
+
+
+class ScreenshotImageField(ImageField):
+
+  def file_name(self, req, obj):
+    return '%s-screenshot' % models.slugify(obj.question.get().answer_title)
+
+
+class LeagueImageField(ImageField):
+
+  def file_name(self, req, obj):
+    return obj.name_slug
 
 
 class ClueForm(Form):
   """A clue form."""
   text = fields.TextAreaField('Text')
-  image = ImageField('Image')
+  image = ScreenshotImageField('Image', 'questions')
 
 
 class ClueFormField(fields.FormField):
@@ -198,7 +236,41 @@ def populate_obj(self, entity, name):
     entity.users = [ndb.Key('User', int(key)) for key in self.data.split(',')]
 
 
-class Question(Form):
+class OrderedFieldForm(Form):
+  """
+  Set the order in which the fields populate the model.
+
+  So we can rely on the model having populated fields in subsequent
+  populate object calls.
+  """
+
+  # The field order list must contain the name of every field.
+  field_order = []
+
+  def populate_obj(self, obj):
+    """
+    Populates the attributes of the passed `obj` with data from the form's
+    fields.
+    """
+    items = sorted(
+      self._fields.items(), key=lambda tup: self.field_order.index(tup[0]))
+
+    for name, field in items:
+        field.populate_obj(obj, name)
+
+
+class Question(OrderedFieldForm):
+
+  field_order = [
+    'answer',
+    'clues',
+    'week',
+    'season',
+    'imdb_url',
+    'email_msg',
+    'packshot',
+  ]
+
   """A question form."""
   def __init__(self,  **kwargs):
     super(Question, self).__init__( **kwargs)
@@ -208,7 +280,7 @@ def __init__(self,  **kwargs):
   answer = FilmField('Film', [validators.Required()], id='film')
   clues = CluesFieldList(ClueFormField(ClueForm), min_entries=4)
   email_msg = fields.TextAreaField('Email Message')
-  packshot = ImageField('Image')
+  packshot = PackshotImageField('Image', 'questions')
   imdb_url = fields.TextField('IMDB Link',
                               default='http://www.imdb.com/title/XXX/')
   week = WeekField(choices=WeekField.week_choices())
@@ -220,17 +292,34 @@ class Registration(Form):
   username = fields.TextField('', [validate_username])
 
 
-class User(Form):
+class User(OrderedFieldForm):
+
+  field_order = [
+    'username',
+    'email',
+    'favourite_film',
+    'pic',
+  ]
+
   username = fields.TextField('', [validate_username])
   email = fields.TextField(validators=[validators.Email()])
-  pic = ImageField('pic')
+  pic = ProfileImageField('pic', 'profiles')
   favourite_film = FilmField()
 
 
-class League(Form):
+class League(OrderedFieldForm):
+
+  field_order = [
+    'id',
+    'name',
+    'pic',
+    'owner',
+    'users',
+  ]
+
   id = fields.HiddenField('')
   name = fields.TextField('', [validate_league_name])
-  pic = ImageField('pic')
+  pic = LeagueImageField('pic', 'leagues')
   owner = CurrentUserField()
   users = LeagueUsersField()