🚚(back) serve legacy videos from Scaleway S3

Videos are now served from the aws/ directory in Scaleway S3.
They are served using the django-storage already in place.
As the newer videos are already stored in Scaleway, we do not
rename these files and allow them to be served without requiring signed URLs.

Author: wilbrdt

CHANGELOG.md

@@ -11,6 +11,7 @@ Versioning](https://semver.org/spec/v2.0.0.html).
 ### Changed
 
 - Serve legacy classroom documents from Scaleway S3 after AWS migration
+- Serve legacy videos and related files from Scaleway S3 after AWS migration
 
 ## [5.9.1] - 2025-06-25
 

src/backend/marsha/core/models/video.py

@@ -293,28 +293,6 @@ class Meta:
             ),
         ]
 
-    def get_source_s3_key(self, stamp=None):
-        """Compute the S3 key in the source bucket (ID of the video + version stamp).
-
-        Parameters
-        ----------
-        stamp: Type[string]
-            Passing a value for this argument will return the source S3 key for the video assuming
-            its active stamp is set to this value. This is useful to create an upload policy for
-            this prospective version of the video, so that the client can upload the file to S3
-            and the confirmation lambda can set the `uploaded_on` field to this value only after
-            the video transcoding job is successful.
-
-        Returns
-        -------
-        string
-            The S3 key for the video in the source bucket, where uploaded videos are stored before
-            they are converted to the destination bucket.
-
-        """
-        stamp = stamp or self.uploaded_on_stamp()
-        return f"{self.pk}/video/{self.pk}/{stamp}"
-
     def get_storage_prefix(
         self,
         stamp=None,
@@ -341,6 +319,10 @@ def get_storage_prefix(
         """
         stamp = stamp or self.uploaded_on_stamp()
         base = base_dir
+
+        if base == AWS_STORAGE_BASE_DIRECTORY:
+            return f"{base}/{self.pk}"
+
         if base == DELETED_STORAGE_BASE_DIRECTORY:
             base = f"{base}/{VOD_STORAGE_BASE_DIRECTORY}"
 
@@ -693,8 +675,12 @@ def get_storage_prefix(
         """
         stamp = stamp or self.uploaded_on_stamp()
         base = base_dir
+
+        if base_dir == AWS_STORAGE_BASE_DIRECTORY:
+            return f"{base}/{self.video.pk}/timedtext"
+
         if base_dir == DELETED_STORAGE_BASE_DIRECTORY:
-            base = f"{DELETED_STORAGE_BASE_DIRECTORY}/{VOD_STORAGE_BASE_DIRECTORY}"
+            base = f"{base}/{VOD_STORAGE_BASE_DIRECTORY}"
 
         return f"{base}/{self.video.pk}/timedtext/{self.pk}/{stamp}"
 

src/backend/marsha/core/serializers/video.py

@@ -4,18 +4,17 @@
 from datetime import timedelta
 from hashlib import sha256
 from inspect import signature
-from urllib.parse import quote_plus
 
 from django.conf import settings
 from django.urls import reverse
 from django.utils import timezone
-from django.utils.text import slugify
 
 from rest_framework import serializers
 from sentry_sdk import capture_message
 
 from marsha.core.defaults import (
     AWS_PIPELINE,
+    AWS_STORAGE_BASE_DIRECTORY,
     ENDED,
     HARVESTED,
     IDLE,
@@ -28,7 +27,7 @@
     TMP_STORAGE_BASE_DIRECTORY,
 )
 from marsha.core.models import TimedTextTrack, Video
-from marsha.core.serializers.base import TimestampField, get_video_cloudfront_url_params
+from marsha.core.serializers.base import TimestampField
 from marsha.core.serializers.playlist import PlaylistLiteSerializer
 from marsha.core.serializers.shared_live_media import (
     SharedLiveMediaId3TagsSerializer,
@@ -37,7 +36,7 @@
 from marsha.core.serializers.thumbnail import ThumbnailSerializer
 from marsha.core.serializers.timed_text_track import TimedTextTrackSerializer
 from marsha.core.storage.storage_class import file_storage
-from marsha.core.utils import cloudfront_utils, jitsi_utils, time_utils, xmpp_utils
+from marsha.core.utils import jitsi_utils, time_utils, xmpp_utils
 from marsha.core.utils.time_utils import to_datetime
 
 
@@ -220,40 +219,30 @@ def get_vod_urls(self, obj):
             )
 
         if obj.transcode_pipeline == AWS_PIPELINE:
-            base = f"{settings.AWS_S3_URL_PROTOCOL}://{settings.CLOUDFRONT_DOMAIN}/{obj.pk}"
-            if settings.CLOUDFRONT_SIGNED_URLS_ACTIVE:
-                params = get_video_cloudfront_url_params(obj.pk)
-
-            filename = f"{slugify(obj.playlist.title)}_{stamp}.mp4"
-            content_disposition = quote_plus(f"attachment; filename={filename}")
+            base = obj.get_storage_prefix(base_dir=AWS_STORAGE_BASE_DIRECTORY)
 
             for resolution in obj.resolutions:
                 # MP4
-                mp4_url = (
-                    f"{base}/mp4/{stamp}_{resolution}.mp4"
-                    f"?response-content-disposition={content_disposition}"
-                )
+                mp4_url = file_storage.url(f"{base}/mp4/{stamp}_{resolution}.mp4")
 
                 # Thumbnails
                 urls["thumbnails"][resolution] = thumbnail_urls.get(
                     resolution,
-                    f"{base}/thumbnails/{stamp}_{resolution}.0000000.jpg",
+                    file_storage.url(
+                        f"{base}/thumbnails/{stamp}_{resolution}.0000000.jpg"
+                    ),
                 )
 
-                # Sign the urls of mp4 videos only if the functionality is activated
-                if settings.CLOUDFRONT_SIGNED_URLS_ACTIVE:
-                    mp4_url = cloudfront_utils.build_signed_url(mp4_url, params)
-
                 urls["mp4"][resolution] = mp4_url
 
             if obj.live_state != HARVESTED:
                 # Adaptive Bit Rate manifests
                 urls["manifests"] = {
-                    "hls": f"{base}/cmaf/{stamp}.m3u8",
+                    "hls": file_storage.url(f"{base}/cmaf/{stamp}.m3u8"),
                 }
 
                 # Previews
-                urls["previews"] = f"{base}/previews/{stamp}_100.jpg"
+                urls["previews"] = file_storage.url(f"{base}/previews/{stamp}_100.jpg")
 
         elif obj.transcode_pipeline == PEERTUBE_PIPELINE:
             base = obj.get_storage_prefix(stamp=stamp)

src/backend/marsha/core/tests/api/video/test_live_to_vod.py

@@ -46,13 +46,15 @@ def assert_user_can_call_live_to_vod(self, user, video):
         """Assert the user can convert the live."""
         self.assertNotEqual(video.upload_state, READY)
 
-        with mock.patch(
-            "marsha.websocket.utils.channel_layers_utils.dispatch_video_to_groups"
-        ) as mock_dispatch_video_to_groups, mock.patch.object(
-            api.video, "reopen_room_for_vod"
-        ) as mock_reopen_room, mock.patch(
-            "marsha.core.serializers.xmpp_utils.generate_jwt"
-        ) as mock_jwt_encode:
+        with (
+            mock.patch(
+                "marsha.websocket.utils.channel_layers_utils.dispatch_video_to_groups"
+            ) as mock_dispatch_video_to_groups,
+            mock.patch.object(api.video, "reopen_room_for_vod") as mock_reopen_room,
+            mock.patch(
+                "marsha.core.serializers.xmpp_utils.generate_jwt"
+            ) as mock_jwt_encode,
+        ):
             mock_jwt_encode.return_value = "xmpp_jwt"
             jwt_token = UserAccessTokenFactory(user=user)
             response = self.client.post(
@@ -155,6 +157,7 @@ def test_live_to_vod_by_playlist_admin(self):
     @override_settings(XMPP_CONVERSE_PERSISTENT_STORE="localStorage")
     @override_settings(XMPP_DOMAIN="conference.xmpp-server.com")
     @override_settings(XMPP_JWT_SHARED_SECRET="xmpp_shared_secret")
+    @override_settings(MEDIA_URL="https://abc.svc.edge.scw.cloud/")
     def test_api_video_instructor_harvested_live_to_vod(self):
         """An instructor can transform a harvested live to a vod."""
         video = factories.VideoFactory(
@@ -169,13 +172,15 @@ def test_api_video_instructor_harvested_live_to_vod(self):
         )
         jwt_token = InstructorOrAdminLtiTokenFactory(playlist=video.playlist)
 
-        with mock.patch(
-            "marsha.websocket.utils.channel_layers_utils.dispatch_video_to_groups"
-        ) as mock_dispatch_video_to_groups, mock.patch.object(
-            api.video, "reopen_room_for_vod"
-        ) as mock_reopen_room, mock.patch(
-            "marsha.core.serializers.xmpp_utils.generate_jwt"
-        ) as mock_jwt_encode:
+        with (
+            mock.patch(
+                "marsha.websocket.utils.channel_layers_utils.dispatch_video_to_groups"
+            ) as mock_dispatch_video_to_groups,
+            mock.patch.object(api.video, "reopen_room_for_vod") as mock_reopen_room,
+            mock.patch(
+                "marsha.core.serializers.xmpp_utils.generate_jwt"
+            ) as mock_jwt_encode,
+        ):
             mock_jwt_encode.return_value = "xmpp_jwt"
             response = self.client.post(
                 f"/api/videos/{video.id}/live-to-vod/",
@@ -212,29 +217,26 @@ def test_api_video_instructor_harvested_live_to_vod(self):
                 "title": video.title,
                 "urls": {
                     "mp4": {
-                        "240": f"https://abc.cloudfront.net/{video.id}/"
-                        "mp4/1569309880_240.mp4?response-content-disposition=attachment%3B+"
-                        "filename%3Dplaylist-002_1569309880.mp4",
-                        "480": f"https://abc.cloudfront.net/{video.id}/"
-                        "mp4/1569309880_480.mp4?response-content-disposition=attachment%3B+"
-                        "filename%3Dplaylist-002_1569309880.mp4",
-                        "720": f"https://abc.cloudfront.net/{video.id}/"
-                        "mp4/1569309880_720.mp4?response-content-disposition=attachment%3B+"
-                        "filename%3Dplaylist-002_1569309880.mp4",
+                        "240": f"https://abc.svc.edge.scw.cloud/aws/{video.id}/"
+                        "mp4/1569309880_240.mp4",
+                        "480": f"https://abc.svc.edge.scw.cloud/aws/{video.id}/"
+                        "mp4/1569309880_480.mp4",
+                        "720": f"https://abc.svc.edge.scw.cloud/aws/{video.id}/"
+                        "mp4/1569309880_720.mp4",
                     },
                     "thumbnails": {
-                        "240": f"https://abc.cloudfront.net/{video.id}/"
+                        "240": f"https://abc.svc.edge.scw.cloud/aws/{video.id}/"
                         "thumbnails/1569309880_240.0000000.jpg",
-                        "480": f"https://abc.cloudfront.net/{video.id}/"
+                        "480": f"https://abc.svc.edge.scw.cloud/aws/{video.id}/"
                         "thumbnails/1569309880_480.0000000.jpg",
-                        "720": f"https://abc.cloudfront.net/{video.id}/"
+                        "720": f"https://abc.svc.edge.scw.cloud/aws/{video.id}/"
                         "thumbnails/1569309880_720.0000000.jpg",
                     },
                     "manifests": {
-                        "hls": f"https://abc.cloudfront.net/{video.id}/"
+                        "hls": f"https://abc.svc.edge.scw.cloud/aws/{video.id}/"
                         "cmaf/1569309880.m3u8"
                     },
-                    "previews": f"https://abc.cloudfront.net/{video.id}/"
+                    "previews": f"https://abc.svc.edge.scw.cloud/aws/{video.id}/"
                     "previews/1569309880_100.jpg",
                 },
                 "should_use_subtitle_as_transcript": False,

src/backend/marsha/core/tests/api/video/test_retrieve.py

@@ -197,10 +197,7 @@ def test_api_video_read_detail_token_user(self):
             for resolution in resolutions
         }
 
-        mp4_template = (
-            "https://abc.svc.edge.scw.cloud/aws/{!s}/mp4/1533686400_{!s}.mp4"
-            "?response-content-disposition=attachment%3B+filename%3Dfoo-bar_1533686400.mp4"
-        )
+        mp4_template = "https://abc.svc.edge.scw.cloud/aws/{!s}/mp4/1533686400_{!s}.mp4"
         mp4_dict = {
             # pylint: disable=consider-using-f-string
             str(resolution): mp4_template.format(video.pk, resolution)
@@ -243,8 +240,7 @@ def test_api_video_read_detail_token_user(self):
                         "upload_state": "ready",
                         "source_url": (
                             "https://abc.svc.edge.scw.cloud/aws/a2f27fde-973a-4e89-8dca-cc59e01d255c/"
-                            "timedtext/source/1533686400_fr_cc?response-content-disposition=a"
-                            "ttachment%3B+filename%3Dfoo-bar_1533686400.srt"
+                            "timedtext/source/1533686400_fr_cc"
                         ),
                         "url": (
                             "https://abc.svc.edge.scw.cloud/aws/a2f27fde-973a-4e89-8dca-cc59e01d255c/"
@@ -344,11 +340,11 @@ def test_api_video_read_detail_token_user(self):
         )
 
     @override_settings(MEDIA_URL="https://abc.svc.edge.scw.cloud/")
-    def test_api_video_read_detail_token_user_nested_shared_live_media_urls_signed(
+    def test_api_video_read_detail_token_user_nested_shared_live_media(
         self,
     ):
         """
-        An instructor reading video details with nested shared live media and cloudfront signed
+        An instructor reading video details with nested shared live media
         urls activated should have the urls.media available
         """
         resolutions = [144, 240, 480, 720, 1080]
@@ -541,17 +537,13 @@ def test_api_video_read_detail_token_user_nested_shared_live_media_urls_signed(
             content, {"detail": "You do not have permission to perform this action."}
         )
 
-    @override_settings(
-        CLOUDFRONT_SIGNED_URLS_ACTIVE=True,
-        CLOUDFRONT_SIGNED_PUBLIC_KEY_ID="cloudfront-access-key-id",
-        MEDIA_URL="https://abc.svc.edge.scw.cloud/",
-    )
+    @override_settings(MEDIA_URL="https://abc.svc.edge.scw.cloud")
     def test_api_video_read_detail_token_student_user_nested_shared_live_media_urls_signed(
         self,
     ):
         """
-        A student reading video details with nested shared live media and cloudfront signed
-        urls activated should not have the urls.media available
+        A student reading video details with nested shared live media should not have
+        the urls.media available
         """
         resolutions = [144, 240, 480, 720, 1080]
         video = factories.VideoFactory(
@@ -605,19 +597,6 @@ def test_api_video_read_detail_token_student_user_nested_shared_live_media_urls_
             for resolution in resolutions
         }
 
-        expected_cloudfront_signature = (
-            "Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNl"
-            "IjoiaHR0cHM6Ly9hYmMuY2xvdWRmcm9udC5uZXQvZDlkNzA0OWMtNWEzZi00MDcwLWE0"
-            "OTQtZTZiZjBiZDhiOWZiLyoiLCJDb25kaXRpb24iOnsiRGF0ZUxlc3NUaGFuIjp7IkFX"
-            "UzpFcG9jaFRpbWUiOjE2MzgyMzc2MDB9fX1dfQ__&Signature=IVWMFfS7WQVTKLZl~"
-            "gKgGES~BS~wVLBIOncSE6yVgg9zIrEI1Epq3AVkOsI7z10dyjgInNbPviArnxmlV~DQe"
-            "N-ykgEWmGy7aT4lRCx61oXuHFtNkq8Qx-we~UY87mZ4~UTqmM~JVuuLduMiRQB-I3XKa"
-            "RQGRlsok5yGu0RhvLcZntVFp6QgYui3WtGvxSs2LjW0IakR1qepSDl9LXI-F2bgl9Vd1"
-            "U9eapPBhhoD0okebXm7NGg9gUMLXlmUo-RvsrAzzEteKctPp0Xzkydk~tcnMkJs4jfbQ"
-            "xKrpyF~N9OuCRYCs68ONhHvypOYU3K-wQEoAFlERBLiaOzDZUzlyA__&Key-Pair-Id="
-            "cloudfront-access-key-id"
-        )
-
         self.assertEqual(
             content,
             {
@@ -648,40 +627,35 @@ def test_api_video_read_detail_token_student_user_nested_shared_live_media_urls_
                 "urls": {
                     "mp4": {
                         "144": (
-                            "https://abc.cloudfront.net/d9d7049c-5a3f-4070-a494-e6bf0bd8b9fb/mp4/"
-                            "1533686400_144.mp4?response-content-disposition=attachment%3B+filen"
-                            f"ame%3Dfoo-bar_1533686400.mp4&{expected_cloudfront_signature}"
+                            "https://abc.svc.edge.scw.cloud/aws/d9d7049c-5a3f-4070-a494-e6bf0bd8b9fb/mp4/"
+                            "1533686400_144.mp4"
                         ),
                         "240": (
-                            "https://abc.cloudfront.net/d9d7049c-5a3f-4070-a494-e6bf0bd8b9fb/mp4/"
-                            "1533686400_240.mp4?response-content-disposition=attachment%3B+filen"
-                            f"ame%3Dfoo-bar_1533686400.mp4&{expected_cloudfront_signature}"
+                            "https://abc.svc.edge.scw.cloud/aws/d9d7049c-5a3f-4070-a494-e6bf0bd8b9fb/mp4/"
+                            "1533686400_240.mp4"
                         ),
                         "480": (
-                            "https://abc.cloudfront.net/d9d7049c-5a3f-4070-a494-e6bf0bd8b9fb/mp4/"
-                            "1533686400_480.mp4?response-content-disposition=attachment%3B+filen"
-                            f"ame%3Dfoo-bar_1533686400.mp4&{expected_cloudfront_signature}"
+                            "https://abc.svc.edge.scw.cloud/aws/d9d7049c-5a3f-4070-a494-e6bf0bd8b9fb/mp4/"
+                            "1533686400_480.mp4"
                         ),
                         "720": (
-                            "https://abc.cloudfront.net/d9d7049c-5a3f-4070-a494-e6bf0bd8b9fb/mp4/"
-                            "1533686400_720.mp4?response-content-disposition=attachment%3B+filen"
-                            f"ame%3Dfoo-bar_1533686400.mp4&{expected_cloudfront_signature}"
+                            "https://abc.svc.edge.scw.cloud/aws/d9d7049c-5a3f-4070-a494-e6bf0bd8b9fb/mp4/"
+                            "1533686400_720.mp4"
                         ),
                         "1080": (
-                            "https://abc.cloudfront.net/d9d7049c-5a3f-4070-a494-e6bf0bd8b9fb/mp4/"
-                            "1533686400_1080.mp4?response-content-disposition=attachment%3B+filen"
-                            f"ame%3Dfoo-bar_1533686400.mp4&{expected_cloudfront_signature}"
+                            "https://abc.svc.edge.scw.cloud/aws/d9d7049c-5a3f-4070-a494-e6bf0bd8b9fb/mp4/"
+                            "1533686400_1080.mp4"
                         ),
                     },
                     "thumbnails": thumbnails_dict,
                     "manifests": {
                         "hls": (
-                            "https://abc.cloudfront.net/d9d7049c-5a3f-4070-a494-e6bf0bd8b9fb/"
+                            "https://abc.svc.edge.scw.cloud/aws/d9d7049c-5a3f-4070-a494-e6bf0bd8b9fb/"
                             "cmaf/1533686400.m3u8"
                         ),
                     },
                     "previews": (
-                        "https://abc.cloudfront.net/d9d7049c-5a3f-4070-a494-e6bf0bd8b9fb/"
+                        "https://abc.svc.edge.scw.cloud/aws/d9d7049c-5a3f-4070-a494-e6bf0bd8b9fb/"
                         "previews/1533686400_100.jpg"
                     ),
                 },
@@ -846,7 +820,6 @@ def test_api_video_read_detail_as_instructor_in_read_only(self):
         )
         self.assertEqual(response.status_code, 200)
 
-    @override_settings(CLOUDFRONT_SIGNED_URLS_ACTIVE=False)
     def test_api_video_read_detail_token_user_no_active_stamp(self):
         """A video with no active stamp should not fail and its "urls" should be set to `None`."""
         video = factories.VideoFactory(
@@ -913,7 +886,6 @@ def test_api_video_read_detail_token_user_no_active_stamp(self):
             },
         )
 
-    @override_settings(CLOUDFRONT_SIGNED_URLS_ACTIVE=False)
     def test_api_video_read_detail_token_user_not_sucessfully_uploaded(self):
         """A video that has never been uploaded successfully should have no url."""
         state = random.choice(["pending", "error", "ready"])