Uploading and loading thumbnails from s3

hgtv/__init__.py

@@ -29,4 +29,5 @@
 lastuser.init_app(app)
 lastuser.init_usermanager(UserManager(db, models.User))
 app.config['tz'] = timezone(app.config['TIMEZONE'])
-uploads.configure(app)
+
+uploads.thumbnails.init_app(app)

hgtv/models/channel.py

@@ -13,6 +13,7 @@
 
 from .video import PlaylistVideo, Video
 from ..models import db, BaseMixin, BaseScopedNameMixin, PLAYLIST_AUTO_TYPE
+from ..uploads import thumbnails
 
 
 __all__ = ['CHANNEL_TYPE', 'PLAYLIST_TYPE', 'Channel', 'Playlist', 'PlaylistRedirect']
@@ -53,6 +54,10 @@ class Channel(ProfileBase, db.Model):
     def __repr__(self):
         return '<Channel %s "%s">' % (self.name, self.title)
 
+    @property
+    def logo_url(self):
+        return thumbnails.get_url(self.channel_logo_filename)
+
     @property
     def current_action_permissions(self):
         """

hgtv/models/video.py

@@ -1,10 +1,12 @@
 # -*- coding: utf-8 -*-
 
+import os.path
 import urllib.parse, urllib.error
 from sqlalchemy.ext.associationproxy import association_proxy
 from werkzeug.utils import cached_property
 from flask import Markup, url_for, current_app
 from .tag import tags_videos
+from ..uploads import thumbnails
 from ..models import db, TimestampMixin, BaseIdNameMixin, PLAYLIST_AUTO_TYPE
 
 __all__ = ['PlaylistVideo', 'Video']
@@ -83,7 +85,7 @@ def url_user_playlists(self):
 
     @property
     def thumbnail(self):
-        return url_for('static', filename='thumbnails/' + self.thumbnail_path)
+        return thumbnails.get_url(self.thumbnail_path)
 
     @property
     def speaker_names(self):

hgtv/uploads.py

@@ -1,65 +1,111 @@
 #! /usr/bin/env python
 
+import boto3
+import botocore
+from datetime import datetime, timedelta
 from PIL import Image
 import os
 from werkzeug.datastructures import FileStorage
 from werkzeug.utils import secure_filename
 from io import BytesIO
-from flask import current_app
-from flask_uploads import (UploadSet, configure_uploads,
-     IMAGES, UploadNotAllowed)
+from hgtv import app
 
 
-thumbnails = UploadSet('thumbnails', IMAGES,
-    default_dest=lambda app: os.path.join(app.static_folder, 'thumbnails'))
+class UploadNotAllowed(Exception):
+    pass
 
 
-def configure(app):
-    thumbnails_dir = os.path.join(app.static_folder, 'thumbnails')
-    if not os.path.isdir(thumbnails_dir):
-        os.mkdir(thumbnails_dir)
-    configure_uploads(app, thumbnails)
+class S3Uploader:
+    def __init__(self, folder):
+        self.folder = folder
+
+    def init_app(self, app):
+        self._s3_resource = boto3.resource(
+            "s3",
+            aws_access_key_id=app.config["AWS_ACCESS_KEY"],
+            aws_secret_access_key=app.config["AWS_SECRET_KEY"],
+        )
+        self._s3_bucket = self._s3_resource.Bucket(app.config["AWS_BUCKET"])
+
+    def exists_in_s3(self, key):
+        try:
+            self._s3_bucket.Object(key).load()
+        except botocore.exceptions.ClientError:
+            return False
+        return True
+
+    def save(self, filestorage):
+        """
+        Uploads the given FileStorage object to S3 and returns the key.
+
+        :param filestorage: FileStorage object that needs to be uploded.
+        """
+        # check if it's already uploaded on S3
+        key = os.path.join(self.folder, filestorage.filename)
+        if not self.exists_in_s3(key):
+            # upload it to s3
+            self._s3_bucket.put_object(
+                ACL="public-read",
+                Key=key,
+                Body=filestorage.read(),
+                CacheControl="max-age=31536000",
+                ContentType=filestorage.content_type,
+                Expires=datetime.utcnow() + timedelta(days=365),
+            )
+        return filestorage.filename
+
+    def delete(self, thumbnail_name):
+        key = os.path.join(self.folder, thumbnail_name)
+        if self.exists_in_s3(key):
+            # upload it to s3
+            self._s3_resource.meta.client.delete_object(
+                Bucket=app.config["AWS_BUCKET"], Key=key
+            )
+
+    def get_url(self, thumbnail_name):
+        return os.path.join(app.config["MEDIA_DOMAIN"], self.folder, thumbnail_name)
+
+
+thumbnails = S3Uploader(folder="thumbnails")
 
 
 def return_werkzeug_filestorage(request, filename):
-    extension = request.headers['content-type'].split('/')[-1]
-    if extension not in current_app.config['ALLOWED_EXTENSIONS']:
+    extension = request.headers["content-type"].split("/")[-1]
+    if extension not in app.config["ALLOWED_EXTENSIONS"]:
         raise UploadNotAllowed("Unsupported file format")
-    new_filename = secure_filename(filename + '.' + extension)
+    new_filename = secure_filename(filename + "." + extension)
     if isinstance(request, FileStorage):
         tempfile = BytesIO(request.read())
     else:
         # this will be requests' Response object
         tempfile = BytesIO(request.content)
     tempfile.name = new_filename
     filestorage = FileStorage(
-        tempfile,
-        filename=new_filename,
-        content_type=request.headers['content-type']
+        tempfile, filename=new_filename, content_type=request.headers["content-type"]
     )
     return filestorage
 
 
 def resize_image(requestfile, maxsize=(320, 240)):
-    fileext = requestfile.filename.split('.')[-1].lower()
-    if fileext not in current_app.config['ALLOWED_EXTENSIONS']:
+    fileext = requestfile.filename.split(".")[-1].lower()
+    if fileext not in app.config["ALLOWED_EXTENSIONS"]:
         raise UploadNotAllowed("Unsupported file format")
     img = Image.open(requestfile)
     img.load()
     if img.size[0] > maxsize[0] or img.size[1] > maxsize[1]:
         img.thumbnail(maxsize, Image.ANTIALIAS)
-    boximg = Image.new('RGBA', (img.size[0], img.size[1]), (255, 255, 255, 0))
+    boximg = Image.new("RGBA", (img.size[0], img.size[1]), (255, 255, 255, 0))
     boximg.paste(img, (0, 0))
     savefile = BytesIO()
-    if fileext in ['jpg', 'jpeg']:
-        savefile.name = secure_filename(".".join(requestfile.filename.split('.')[:-1]) + ".png")
+    if fileext in ["jpg", "jpeg"]:
+        savefile.name = secure_filename(
+            ".".join(requestfile.filename.split(".")[:-1]) + ".png"
+        )
         boximg.save(savefile, format="PNG")
         content_type = "image/png"
     else:
         savefile.name = secure_filename(requestfile.filename)
         boximg.save(savefile)
         content_type = requestfile.content_type
     savefile.seek(0)
-    return FileStorage(savefile,
-        filename=savefile.name,
-        content_type=content_type)
+    return FileStorage(savefile, filename=savefile.name, content_type=content_type)

hgtv/views/channel.py

@@ -46,23 +46,15 @@ def channel_edit(channel):
         old_channel = channel
         form.populate_obj(channel)
         if form.delete_logo and form.delete_logo.data:
-            try:
-                if old_channel.channel_logo_filename:
-                    os.remove(os.path.join(app.static_folder, 'thumbnails', old_channel.channel_logo_filename))
-                    message = "Removed channel logo"
-            except OSError:
-                channel.channel_logo_filename = None
-                message = "Channel logo already Removed"
+            if old_channel.channel_logo_filename:
+                thumbnails.delete(old_channel.channel_logo_filename)
+                message = "Removed channel logo"
         else:
             if 'channel_logo' in request.files and request.files['channel_logo']:
                 try:
                     if old_channel.channel_logo_filename:
                         db.session.add(old_channel)
-                        try:
-                            os.remove(os.path.join(app.static_folder, 'thumbnails', old_channel.channel_logo_filename))
-                        except OSError:
-                            old_channel.channel_logo_filename = None
-                            message = "Unable to delete previous logo"
+                        thumbnails.delete(old_channel.channel_logo_filename)
                     image = resize_image(request.files['channel_logo'])
                     channel.channel_logo_filename = thumbnails.save(image)
                     message = "Channel logo uploaded"

hgtv/views/index.py

@@ -32,7 +32,7 @@ def index():
             'name': channel.name,
             'title': channel.title,
             'logo':
-                url_for('static', filename='thumbnails/' + channel.channel_logo_filename)
+                channel.logo_url
                 if channel.channel_logo_filename
                 else url_for('static', filename='img/sample-logo.png'),
             'banner_url': channel.channel_banner_url if channel.channel_banner_url else "",

hgtv/views/playlist.py

@@ -66,15 +66,16 @@ def process_playlist(playlist, playlist_url):
                             video = Video(playlist=playlist if playlist is not None else stream_playlist)
                             video.title = playlist_item['snippet']['title']
                             video.video_url = 'https://www.youtube.com/watch?v=' + playlist_item['snippet']['resourceId']['videoId']
-                            if playlist_item['snippet']['description']:
-                                video.description = markdown(playlist_item['snippet']['description'])
-                            for thumbnail in playlist_item['snippet']['thumbnails']['medium']:
-                                thumbnail_url_request = requests.get(playlist_item['snippet']['thumbnails']['medium']['url'])
-                                filestorage = return_werkzeug_filestorage(thumbnail_url_request,
-                                    filename=secure_filename(playlist_item['snippet']['title']) or 'name-missing')
-                                video.thumbnail_path = thumbnails.save(filestorage)
                             video.video_sourceid = playlist_item['snippet']['resourceId']['videoId']
                             video.video_source = 'youtube'
+                            if playlist_item['snippet']['description']:
+                                video.description = markdown(playlist_item['snippet']['description'])
+
+                            thumbnail_url_request = requests.get(playlist_item['snippet']['thumbnails']['medium']['url'])
+                            filestorage = return_werkzeug_filestorage(thumbnail_url_request,
+                                filename=(video.video_source + '-' + video.video_sourceid) or 'name-missing')
+                            video.thumbnail_path = thumbnails.save(filestorage)
+
                             video.make_name()
                             playlist.videos.append(video)
                             with db.session.no_autoflush:

hgtv/views/video.py

@@ -427,6 +427,7 @@ def video_delete(channel, playlist, video):
     if form.validate_on_submit():
         db.session.delete(video)
         db.session.commit()
+        thumbnails.delete(video.thumbnail_path)
         return {'status': 'ok', 'doc': _("Delete video {title}.".format(title=video.title)), 'result': {}}
     return {'status': 'error', 'errors': {'error': form.errors}}, 400