feat: so many changes

.vscode/settings.json

@@ -1,4 +1,8 @@
 {
+  "[python]": {
+    "editor.formatOnSave": true,
+    "editor.defaultFormatter": "charliermarsh.ruff"
+  },
   "files.associations": {
     "CNAME": "plaintext",
   },

Dockerfile

@@ -7,8 +7,15 @@ COPY . /app
 RUN pip install --upgrade pip
 RUN pip install --no-cache-dir -r requirements/prod.txt
 
+RUN adduser --disabled-password --gecos '' thekid
+
+RUN chown -R thekid:thekid /app && chmod -R 775 /app
+
+USER thekid
+
 ENV FLASK_APP=bereal.server
 
 EXPOSE 5000
 
-CMD ["gunicorn", "-k", "gevent", "-w", "4", "-t", "1200", "bereal.server:app"]
+# TODO(michaelfromyeg): remove the port line
+CMD ["gunicorn", "-b", ":5000", "-k", "gevent", "-w", "4", "-t", "600", "bereal.server:app"]

Makefile

@@ -1,9 +1,41 @@
-.PHONY: client server cli typecheck format
+.PHONY: client start-redis celery server cli typecheck format
 
 client:
 	@echo "Booting up the client..."
 	@cd client && npm start
 
+build:
+	@echo "Building the server..."
+	@docker-compose build
+
+up:
+	@echo "Booting up the server..."
+	@docker-compose up -d
+
+down:
+	@echo "Shutting down the server..."
+	@docker-compose down
+
+kill:
+	@echo "Killing the server..."
+	@docker-compose kill
+
+start-redis:
+	@echo "Booting up Redis..."
+	@redis-server /etc/redis/redis.conf
+
+check-redis:
+	@echo "Checking Redis..."
+	@redis-cli ping
+
+celery:
+	@echo "Booting up Celery..."
+	@celery -A bereal.celery worker --loglevel=DEBUG --logfile=celery.log -E
+
+flower:
+	@echo "Booting up Flower..."
+	@celery -A bereal.celery flower --address=0.0.0.0 --inspect --enable-events --loglevel=DEBUG --logfile=flower.log
+
 server:
 	@echo "Booting up the server..."
 	@python -m bereal.server

bereal/bereal.py

@@ -112,7 +112,7 @@ def download_image(date_str: str, url: str, base_path: str) -> None:
 
             if img_response.status_code == 200:
                 img_file.write(img_response.content)
-                logger.info("Downloaded %s", image_name)
+                logger.debug("Downloaded %s", image_name)
             else:
                 logger.warning(
                     "Failed to download %s with code %d; will continue", image_name, img_response.status_code

bereal/celery.py

@@ -0,0 +1,57 @@
+"""
+Celery stuff.
+"""
+import os
+from celery import Celery
+
+from .bereal import memories
+from .images import create_images, cleanup_images
+from .videos import build_slideshow
+from .utils import Mode, REDIS_HOST, REDIS_PORT, year2dates, CONTENT_PATH, EXPORTS_PATH
+from .logger import logger
+
+
+def make_celery(app_name=__name__, broker=f"redis://{REDIS_HOST}:{REDIS_PORT}/0") -> Celery:
+    """
+    Create a celery instance.
+    """
+    celery = Celery(app_name, broker=broker, backend=broker)
+    return celery
+
+
+bcelery = make_celery()
+
+
+@bcelery.task(time_limit=1200)
+def make_video(token: str, phone: str, year: str, song_path: str, mode: Mode) -> str:
+    """
+    Creating a video takes about ~15 min. This is a work-in-progress!
+    """
+    logger.info("%s", CONTENT_PATH)
+    logger.info("%s", EXPORTS_PATH)
+
+    logger.info("%s", os.getcwd())
+
+    logger.info("Starting make_video task; first, downloading images...")
+
+    sdate, edate = year2dates(year)
+    result = memories(phone, year, token, sdate, edate)
+
+    if not result:
+        raise Exception("Could not generate memories; try again later")
+
+    short_token = token[:10]
+    video_file = f"{short_token}-{phone}-{year}.mp4"
+
+    # TODO(michaelfromyeg): implement better error handling, everywhere
+    logger.info("Creating images for %s...", video_file)
+    image_folder = create_images(phone, year)
+
+    logger.info("Creating video %s from %s...", video_file, image_folder)
+    build_slideshow(phone, year, image_folder, song_path, video_file, mode)
+
+    logger.info("Cleaning up images")
+    cleanup_images(phone, year)
+
+    logger.info("Returning %s...", video_file)
+    return video_file

bereal/cli.py

@@ -8,7 +8,7 @@
 from typing import Any, Callable
 
 from .bereal import memories, send_code, verify_code
-from .images import create_images
+from .images import create_images, cleanup_images
 from .logger import logger
 from .utils import CONTENT_PATH, YEARS, Mode, str2mode, year2dates
 from .videos import build_slideshow
@@ -187,7 +187,7 @@ def step(idx: int, retval: dict[str, Any] | None) -> dict[str, Any] | None:
             build_slideshow(retval["image_folder"], retval["song_path"], video_file, retval["mode"])
 
             # TODO(michaelfromyeg): delete images in production
-            # cleanup_images(retval["phone"], retval["year"])
+            cleanup_images(retval["phone"], retval["year"])
         case _:
             raise ValueError(f"Invalid step: {idx}")
 

bereal/images.py

@@ -1,10 +1,8 @@
 """
 Combine two images together, with the secondary image in the top-left corner of the primary image.
 """
-import multiprocessing
 import os
 import shutil
-from multiprocessing import Pool
 
 from PIL import Image, ImageChops, ImageDraw, ImageFont
 
@@ -117,16 +115,22 @@ def create_images(
     # Get a list of primary filenames
     primary_filenames = os.listdir(primary_folder)
 
+    # NOTE(michaelfromyeg): because we're using celery, the below code is unusable
+    # specifically, "AssertionError: daemonic processes are not allowed to have children"
+
+    for primary_filename in primary_filenames:
+        process_image(primary_filename, primary_folder, secondary_folder, output_folder)
+
     # Use multiprocessing to process images in parallel
-    processes = max(1, multiprocessing.cpu_count() - 2)
-    with Pool(processes=processes) as pool:
-        pool.starmap(
-            process_image,
-            [
-                (primary_filename, primary_folder, secondary_folder, output_folder)
-                for primary_filename in primary_filenames
-            ],
-        )
+    # processes = max(1, multiprocessing.cpu_count() - 2)
+    # with Pool(processes=processes) as pool:
+    #     pool.starmap(
+    #         process_image,
+    #         [
+    #             (primary_filename, primary_folder, secondary_folder, output_folder)
+    #             for primary_filename in primary_filenames
+    #         ],
+    #     )
 
     return output_folder
 

bereal/server.py

@@ -5,22 +5,21 @@
 """
 from gevent import monkey
 
-# TODO(michaelfromyeg): move
 monkey.patch_all()
 
 import os
 import warnings
 from datetime import datetime, timedelta
 from typing import Any
 
-from flask import Flask, Response, abort, jsonify, request, send_from_directory
+from flask import Flask, Response, jsonify, request, send_from_directory
 from flask_apscheduler import APScheduler
 from flask_cors import CORS
 from flask_sqlalchemy import SQLAlchemy
-from itsdangerous import SignatureExpired, URLSafeTimedSerializer
+from itsdangerous import URLSafeTimedSerializer
 
-from .bereal import memories, send_code, verify_code
-from .images import create_images
+from .bereal import send_code, verify_code
+from .celery import bcelery, make_video
 from .logger import logger
 from .utils import (
     CONTENT_PATH,
@@ -30,17 +29,22 @@
     GIT_COMMIT_HASH,
     HOST,
     PORT,
+    REDIS_HOST,
+    REDIS_PORT,
     SECRET_KEY,
     str2mode,
-    year2dates,
 )
-from .videos import build_slideshow
 
 warnings.filterwarnings("ignore", category=UserWarning, module="tzlocal")
 
 app = Flask(__name__)
 serializer = URLSafeTimedSerializer(SECRET_KEY)
 
+logger.info("Running in %s mode", FLASK_ENV)
+
+logger.info("CONTENT_PATH: %s", CONTENT_PATH)
+logger.info("EXPORTS_PATH: %s", EXPORTS_PATH)
+
 if FLASK_ENV == "development":
     logger.info("Enabling CORS for development")
     CORS(app)
@@ -57,6 +61,9 @@
 scheduler.init_app(app)
 scheduler.start()
 
+app.config["CELERY_BROKER_URL"] = f"redis://{REDIS_HOST}:{REDIS_PORT}/0"
+bcelery.conf.update(app.config)
+
 
 class PhoneToken(db.Model):
     id = db.Column(db.Integer, primary_key=True)
@@ -98,7 +105,7 @@ def request_otp() -> tuple[Response, int]:
         return jsonify(
             {
                 "error": "Bad Request",
-                "message": "Invalid phone number; make sure not to include anything besides the digits (i.e., not '+' or '-' or spaces)",
+                "message": "Invalid phone number; BeReal is likely rate-limiting the service. Make sure not to include anything besides the digits (i.e., not '+' or '-' or spaces).",
             }
         ), 400
 
@@ -133,13 +140,11 @@ def create_video() -> tuple[Response, int]:
     """
     phone = request.form["phone"]
     token = request.form["token"]
-    short_token = token[:10]
 
     if token != get_token(phone):
         return jsonify({"error": "Bad Request", "message": "Invalid token"}), 400
 
     year = request.form["year"]
-    sdate, edate = year2dates(year)
 
     wav_file = request.files.get("file", None)
 
@@ -159,33 +164,46 @@ def create_video() -> tuple[Response, int]:
     else:
         song_path = DEFAULT_SONG_PATH
 
-    logger.debug("Downloading images locally...")
-    result = memories(phone, year, token, sdate, edate)
+    logger.debug("Queueing video task...")
+    task = make_video.delay(token, phone, year, song_path, mode)
+
+    return jsonify({"taskId": task.id}), 202
+
 
-    if not result:
-        return jsonify({"error": "Internal Server Error", "message": "Could not generate images; try again later"}), 500
+@app.route("/status/<task_id>", methods=["GET"])
+def task_status(task_id) -> tuple[Response, int]:
+    task = make_video.AsyncResult(task_id)
 
-    video_file = f"{short_token}-{phone}-{year}.mp4"
+    try:
+        if task.state == "PENDING":
+            response = {"status": "PENDING"}
+            return jsonify(response), 202
 
-    image_folder = create_images(phone, year)
-    build_slideshow(image_folder, song_path, video_file, mode)
+        if task.state == "FAILURE":
+            logger.error("Task %s failed: %s", task_id, task.info)
 
-    # TODO(michaelfromyeg): delete images in production
-    # cleanup_images(phone, year)
+            response = {
+                "status": "FAILURE",
+                "message": "An error occurred processing your task. Try again later.",
+                "error": str(task.info),
+            }
+            return jsonify(response), 500
 
-    return jsonify({"videoUrl": video_file}), 200
+        response = {"status": task.status, "result": task.result if task.state == "SUCCESS" else None}
+        return jsonify(response), 200
+    except Exception as e:
+        # Handle cases where task is not registered or result is not JSON serializable
+        response = {"status": "ERROR", "message": "An unexpected error occurred in creating the video", "error": str(e)}
+        return jsonify(response), 500
 
 
 @app.route("/video/<filename>", methods=["GET"])
 def get_video(filename: str) -> tuple[Response, int]:
     """
     Serve a video file.
     """
-    try:
-        return send_from_directory(EXPORTS_PATH, filename, mimetype="video/mp4"), 200
-    except SignatureExpired:
-        # TODO(michaelfromyeg): implement this
-        abort(403)
+    logger.debug("Serving video file %s/%s...", EXPORTS_PATH, filename)
+    return send_from_directory(EXPORTS_PATH, filename, mimetype="video/mp4"), 200
 
 
 @app.route("/favicon.ico")
@@ -259,13 +277,6 @@ def scheduled_task():
 
 
 if __name__ == "__main__":
-    OS_HOST: str | None = os.environ.get("HOST")
-    OS_PORT: str | None = os.environ.get("PORT")
-
-    host = OS_HOST or HOST or "localhost"
-    port = OS_PORT or PORT or 5000
-    port = int(port)
-
-    logger.info("Starting BeReal server on %s:%d...", host, port)
+    logger.info("Starting BeReal server on %s:%d...", HOST, PORT)
 
-    app.run(host=host, port=port, debug=FLASK_ENV == "development")
+    app.run(host=HOST, port=PORT, debug=FLASK_ENV == "development")