🚚(back) serve legacy timed text tracks from Scaleway S3

Timed text tracks are now served from the aws/ directory in Scaleway S3.
They are served using the django-storage already in place.
As the newer timed text tracks are already stored in Scaleway, we do not
rename these files and allow them to be served without requiring signed URLs.

Author: wilbrdt

src/backend/marsha/core/models/video.py

@@ -623,32 +623,6 @@ class Meta:
             )
         ]
 
-    def get_source_s3_key(self, stamp=None):
-        """Compute the S3 key in the source bucket.
-
-        It is built from the video ID + ID of the timed text track + version stamp + language +
-        closed captioning flag.
-
-        Parameters
-        ----------
-        stamp: Type[string]
-            Passing a value for this argument will return the source S3 key for the timed text
-            track assuming its active stamp is set to this value. This is useful to create an
-            upload policy for this prospective version of the track, so that the client can
-            upload the file to S3 and the confirmation lambda can set the `uploaded_on` field
-            to this value only after the file upload and processing is successful.
-
-        Returns
-        -------
-        string
-            The S3 key for the timed text files in the source bucket, where uploaded files are
-            stored before they are converted and copied to the destination bucket.
-
-        """
-        stamp = stamp or self.uploaded_on_stamp()
-        mode = f"_{self.mode}" if self.mode else ""
-        return f"{self.video.pk}/timedtexttrack/{self.pk}/{stamp}_{self.language}{mode}"
-
     def get_storage_prefix(
         self,
         stamp=None,

src/backend/marsha/core/serializers/timed_text_track.py

@@ -1,17 +1,14 @@
 """Structure of TimedTextTrack related models API responses with DRF serializers."""
 
-from urllib.parse import quote_plus
-
 from django.conf import settings
-from django.utils.text import slugify
 
 from rest_framework import serializers
 
-from marsha.core.defaults import CELERY_PIPELINE
+from marsha.core.defaults import AWS_STORAGE_BASE_DIRECTORY, CELERY_PIPELINE
 from marsha.core.models import TimedTextTrack
-from marsha.core.serializers.base import TimestampField, get_video_cloudfront_url_params
+from marsha.core.serializers.base import TimestampField
 from marsha.core.storage.storage_class import file_storage
-from marsha.core.utils import cloudfront_utils, time_utils
+from marsha.core.utils import time_utils
 
 
 class TimedTextTrackSerializer(serializers.ModelSerializer):
@@ -80,51 +77,8 @@ def create(self, validated_data):
 
         return super().create(validated_data)
 
-    def _sign_url(self, url, video_id):
-        """Generate a presigned cloudfront url.
-
-        Parameters
-        ----------
-        url: string
-            The url to sign
-
-        Returns:
-        string
-            The signed url
-
-        """
-        params = get_video_cloudfront_url_params(video_id)
-        return cloudfront_utils.build_signed_url(url, params)
-
-    def _generate_url(self, obj, object_path, extension=None, content_disposition=None):
-        """Generate an url to fetch a timed text track file depending on argument passed.
-
-        Parameters:
-        obj : Type[models.TimedTextTrack]
-            The timed text track that we want to serialize
-
-        object_patch: string
-            The path in the path the timed text track is stored
-
-        extension: string or None
-            If the timed text track need an extension in the url, add it to the end
-
-        content_disposition: string or None
-            Add a response-content-disposition query string to url if present
-        """
-        base = f"{settings.AWS_S3_URL_PROTOCOL}://{settings.CLOUDFRONT_DOMAIN}/{obj.video_id}"
-        stamp = time_utils.to_timestamp(obj.uploaded_on)
-        mode = f"_{obj.mode}" if obj.mode else ""
-        url = f"{base}/{object_path}/{stamp}_{obj.language:s}{mode:s}"
-        if extension:
-            url = f"{url}.{extension}"
-
-        if content_disposition:
-            url = f"{url}?response-content-disposition={content_disposition}"
-        return url
-
     def get_source_url(self, obj):
-        """Source url of the timed text track, signed with a CloudFront key if activated.
+        """Source url of the timed text track.
 
         This is the url of the uploaded file without any modification.
 
@@ -150,21 +104,15 @@ def get_source_url(self, obj):
 
             return file_storage.url(f"{base}/source.{obj.extension}")
 
-        # Default AWS fallback
-        filename = f"{slugify(obj.video.playlist.title)}_{stamp}.{obj.extension}"
-        url = self._generate_url(
-            obj,
-            "timedtext/source",
-            content_disposition=quote_plus(f"attachment; filename={filename}"),
-        )
+        # Default fallback to location under "aws" directory
+        base = obj.get_storage_prefix(base_dir=AWS_STORAGE_BASE_DIRECTORY)
+        stamp = time_utils.to_timestamp(obj.uploaded_on)
+        mode = f"_{obj.mode}" if obj.mode else ""
 
-        # Sign the url only if the functionality is activated
-        if settings.CLOUDFRONT_SIGNED_URLS_ACTIVE:
-            url = self._sign_url(url, obj.video_id)
-        return url
+        return file_storage.url(f"{base}/source/{stamp}_{obj.language:s}{mode:s}")
 
     def get_url(self, obj):
-        """Url of the timed text track, signed with a CloudFront key if activated.
+        """Url of the timed text track.
 
         Parameters
         ----------
@@ -187,10 +135,9 @@ def get_url(self, obj):
 
             return file_storage.url(f"{base}/{stamp}.vtt")
 
-        # Default AWS fallback
-        url = self._generate_url(obj, "timedtext", extension="vtt")
+        # Default fallback to location under "aws" directory
+        base = obj.get_storage_prefix(base_dir=AWS_STORAGE_BASE_DIRECTORY)
+        stamp = time_utils.to_timestamp(obj.uploaded_on)
+        mode = f"_{obj.mode}" if obj.mode else ""
 
-        # Sign the url only if the functionality is activated
-        if settings.CLOUDFRONT_SIGNED_URLS_ACTIVE:
-            url = self._sign_url(url, obj.video_id)
-        return url
+        return file_storage.url(f"{base}/{stamp}_{obj.language:s}{mode:s}.vtt")

src/backend/marsha/core/tests/api/timed_text_tracks/test_retrieve.py

@@ -3,20 +3,21 @@
 from datetime import datetime, timezone as baseTimezone
 import json
 import random
-from unittest import mock
 
 from django.test import TestCase, override_settings
 
 from marsha.core import factories, models
-from marsha.core.api import timezone
 from marsha.core.defaults import AWS_PIPELINE
 from marsha.core.factories import TimedTextTrackFactory, UserFactory, VideoFactory
 from marsha.core.simple_jwt.factories import (
     InstructorOrAdminLtiTokenFactory,
     StudentLtiTokenFactory,
     UserAccessTokenFactory,
 )
-from marsha.core.tests.testing_utils import RSA_KEY_MOCK
+
+
+# flake8: noqa: E501
+# pylint: disable=line-too-long
 
 
 class TimedTextTrackRetrieveAPITest(TestCase):
@@ -55,7 +56,7 @@ def test_api_timed_text_track_read_detail_student(self):
             content, {"detail": "You do not have permission to perform this action."}
         )
 
-    @override_settings(CLOUDFRONT_SIGNED_URLS_ACTIVE=False)
+    @override_settings(MEDIA_URL="https://abc.svc.edge.scw.cloud/")
     def test_api_timed_text_track_read_detail_token_user(self):
         """A token user associated to a video can read a timed text track related to this video."""
         timed_text_track = TimedTextTrackFactory(
@@ -91,12 +92,11 @@ def test_api_timed_text_track_read_detail_token_user(self):
                 "language": "fr",
                 "upload_state": "ready",
                 "source_url": (
-                    "https://abc.cloudfront.net/b8d40ed7-95b8-4848-98c9-50728dfee25d/"
-                    "timedtext/source/1533686400_fr_cc?response-content-disposition=a"
-                    "ttachment%3B+filename%3Dfoo_1533686400.srt"
+                    "https://abc.svc.edge.scw.cloud/aws/b8d40ed7-95b8-4848-98c9-50728dfee25d/"
+                    "timedtext/source/1533686400_fr_cc"
                 ),
                 "url": (
-                    "https://abc.cloudfront.net/b8d40ed7-95b8-4848-98c9-50728dfee25d/"
+                    "https://abc.svc.edge.scw.cloud/aws/b8d40ed7-95b8-4848-98c9-50728dfee25d/"
                     "timedtext/1533686400_fr_cc.vtt"
                 ),
                 "video": str(timed_text_track.video.id),
@@ -116,7 +116,7 @@ def test_api_timed_text_track_read_detail_token_user(self):
             content, {"detail": "You do not have permission to perform this action."}
         )
 
-    @override_settings(CLOUDFRONT_SIGNED_URLS_ACTIVE=False)
+    @override_settings(MEDIA_URL="https://abc.svc.edge.scw.cloud/")
     def test_api_timed_text_track_without_extension_read_detail_token_user(self):
         """A timed text track without extension should return empty source url."""
         timed_text_track = TimedTextTrackFactory(
@@ -152,7 +152,7 @@ def test_api_timed_text_track_without_extension_read_detail_token_user(self):
                 "upload_state": "ready",
                 "source_url": None,
                 "url": (
-                    "https://abc.cloudfront.net/b8d40ed7-95b8-4848-98c9-50728dfee25d/"
+                    "https://abc.svc.edge.scw.cloud/aws/b8d40ed7-95b8-4848-98c9-50728dfee25d/"
                     "timedtext/1533686400_fr_cc.vtt"
                 ),
                 "video": str(timed_text_track.video.id),
@@ -172,7 +172,7 @@ def test_api_timed_text_track_without_extension_read_detail_token_user(self):
             content, {"detail": "You do not have permission to perform this action."}
         )
 
-    @override_settings(CLOUDFRONT_SIGNED_URLS_ACTIVE=False)
+    @override_settings(MEDIA_URL="https://abc.svc.edge.scw.cloud/")
     def test_api_timed_text_track_read_detail_admin_user(self):
         """Admin user associated to a video can read a timed text track related to this video."""
         timed_text_track = TimedTextTrackFactory(
@@ -209,12 +209,11 @@ def test_api_timed_text_track_read_detail_admin_user(self):
                 "language": "fr",
                 "upload_state": "ready",
                 "source_url": (
-                    "https://abc.cloudfront.net/b8d40ed7-95b8-4848-98c9-50728dfee25d/timedtext/"
-                    "source/1533686400_fr_cc?response-content-disposition=attachment%3B+filenam"
-                    "e%3Dfoo_1533686400.srt"
+                    "https://abc.svc.edge.scw.cloud/aws/b8d40ed7-95b8-4848-98c9-50728dfee25d/timedtext/"
+                    "source/1533686400_fr_cc"
                 ),
                 "url": (
-                    "https://abc.cloudfront.net/b8d40ed7-95b8-4848-98c9-50728dfee25d/timedtext/"
+                    "https://abc.svc.edge.scw.cloud/aws/b8d40ed7-95b8-4848-98c9-50728dfee25d/timedtext/"
                     "1533686400_fr_cc.vtt"
                 ),
                 "video": str(timed_text_track.video.id),
@@ -258,7 +257,6 @@ def test_api_timed_text_track_read_instructor_in_read_only(self):
         )
         self.assertEqual(response.status_code, 403)
 
-    @override_settings(CLOUDFRONT_SIGNED_URLS_ACTIVE=False)
     def test_api_timed_text_track_read_detail_token_user_no_active_stamp(self):
         """A timed text track with no active stamp should not fail.
 
@@ -279,7 +277,6 @@ def test_api_timed_text_track_read_detail_token_user_no_active_stamp(self):
         content = json.loads(response.content)
         self.assertIsNone(content["url"])
 
-    @override_settings(CLOUDFRONT_SIGNED_URLS_ACTIVE=False)
     def test_api_timed_text_track_read_detail_token_user_not_ready(self):
         """A timed_text_track that has never been uploaded successfully should have no url."""
         timed_text_track = TimedTextTrackFactory(
@@ -299,66 +296,6 @@ def test_api_timed_text_track_read_detail_token_user_not_ready(self):
         content = json.loads(response.content)
         self.assertIsNone(content["url"])
 
-    @override_settings(
-        CLOUDFRONT_SIGNED_URLS_ACTIVE=True,
-        CLOUDFRONT_SIGNED_PUBLIC_KEY_ID="cloudfront-access-key-id",
-    )
-    @mock.patch("builtins.open", new_callable=mock.mock_open, read_data=RSA_KEY_MOCK)
-    def test_api_timed_text_track_read_detail_token_user_signed_urls(self, _mock_open):
-        """Activating signed urls should add Cloudfront query string authentication parameters."""
-        timed_text_track = TimedTextTrackFactory(
-            video__pk="b8d40ed7-95b8-4848-98c9-50728dfee25d",
-            video__playlist__title="foo",
-            mode="cc",
-            language="fr",
-            uploaded_on=datetime(2018, 8, 8, tzinfo=baseTimezone.utc),
-            upload_state="ready",
-            process_pipeline=AWS_PIPELINE,
-            extension="srt",
-        )
-        jwt_token = InstructorOrAdminLtiTokenFactory(
-            playlist=timed_text_track.video.playlist
-        )
-
-        # Get the timed_text_track via the API using the JWT token
-        # fix the time so that the url signature is deterministic and can be checked
-        now = datetime(2018, 8, 8, tzinfo=baseTimezone.utc)
-        with mock.patch.object(timezone, "now", return_value=now):
-            response = self.client.get(
-                self._get_url(timed_text_track.video, timed_text_track),
-                HTTP_AUTHORIZATION=f"Bearer {jwt_token}",
-            )
-        self.assertEqual(response.status_code, 200)
-        content = json.loads(response.content)
-
-        expected_cloudfront_signature = (
-            "Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6"
-            "Ly9hYmMuY2xvdWRmcm9udC5uZXQvYjhkNDBlZDctOTViOC00ODQ4LTk4YzktNTA3MjhkZmVlM"
-            "jVkLyoiLCJDb25kaXRpb24iOnsiRGF0ZUxlc3NUaGFuIjp7IkFXUzpFcG9jaFRpbWUiOjE1Mz"
-            "M2OTM2MDB9fX1dfQ__&Signature=PkRZOcfOxbalcuNG9XN6wO72enDenSetWgTthNjR4Nsy"
-            "UvCao1rZ9s4MZbqU61NDxB8Q3yDoWZUm-PP0uFa6v2Rz9g6XSTCA~-x8Yhh72-jc1J5NZOavh"
-            "~HT6lbC2HnPAesaxbVG4EejSDuXjncE8kBiUdT6YNotAv1JzbqidXuOBdkSjR32PEav98PT0r"
-            "UKmXohNAL-RFdwHL1cKGhy17CoxABn4ToDJ-t0Z4cT4husb5HebH~6nOmhlDDdFMSdmD7FjZ~"
-            "qaJwagJ3sAqG1ph9NcTX45bDn2rcrDXUy0jHWxBPYUId6NGbKCITp1SFj0QAsoxsXnh90Ibkr"
-            "GQ4XUA__&Key-Pair-Id=cloudfront-access-key-id"
-        )
-
-        self.assertEqual(
-            content["url"],
-            (
-                "https://abc.cloudfront.net/b8d40ed7-95b8-4848-98c9-50728dfee25d/timedtext"
-                f"/1533686400_fr_cc.vtt?{expected_cloudfront_signature}"
-            ),
-        )
-        self.assertEqual(
-            content["source_url"],
-            (
-                "https://abc.cloudfront.net/b8d40ed7-95b8-4848-98c9-50728dfee25d/timedtext"
-                "/source/1533686400_fr_cc?response-content-disposition=attachment%3B+filen"
-                f"ame%3Dfoo_1533686400.srt&{expected_cloudfront_signature}"
-            ),
-        )
-
     def test_api_timed_text_track_read_detail_staff_or_user(self):
         """Users authenticated via a session are not allowed to read a timed text track detail."""
         for user in [UserFactory(), UserFactory(is_staff=True)]:

src/backend/marsha/core/tests/serializers/test_timed_text_track.py

@@ -12,6 +12,7 @@
 class TimedTextTrackSerializerTest(TestCase):
     """Test the TimedTextTrack serializer."""
 
+    @override_settings(MEDIA_URL="https://abc.svc.edge.scw.cloud/")
     def test_timed_text_track_serializer_urls_with_aws_pipeline(self):
         """The TimedTextTrackSerializer should return AWS URLs."""
         date = datetime(2022, 1, 1, tzinfo=baseTimezone.utc)
@@ -32,18 +33,15 @@ def test_timed_text_track_serializer_urls_with_aws_pipeline(self):
         )
         serializer = TimedTextTrackSerializer(timed_text_track)
         self.assertEqual(
-            f"https://abc.cloudfront.net/{video.pk}/timedtext/1640995200_fr_ts.vtt",
+            f"https://abc.svc.edge.scw.cloud/aws/{video.pk}/timedtext/1640995200_fr_ts.vtt",
             serializer.data["url"],
         )
         self.assertEqual(
-            f"https://abc.cloudfront.net/{video.pk}/timedtext/source/1640995200_fr_ts"
-            "?response-content-disposition=attachment%3B+filename%3Dplaylist-001_1640995200.srt",
+            f"https://abc.svc.edge.scw.cloud/aws/{video.pk}/timedtext/source/1640995200_fr_ts",
             serializer.data["source_url"],
         )
 
-    @override_settings(
-        MEDIA_URL="https://abc.cloudfront.net/",
-    )
+    @override_settings(MEDIA_URL="https://abc.svc.edge.scw.cloud/")
     def test_timed_text_track_serializer_urls_with_celery_pipeline(self):
         """The TimedTextTrackSerializer should return videos storage URLs."""
         date = datetime(2022, 1, 1, tzinfo=baseTimezone.utc)
@@ -63,12 +61,12 @@ def test_timed_text_track_serializer_urls_with_celery_pipeline(self):
         )
         serializer = TimedTextTrackSerializer(timed_text_track)
         self.assertEqual(
-            f"https://abc.cloudfront.net/vod/{video.pk}/timedtext/"
+            f"https://abc.svc.edge.scw.cloud/vod/{video.pk}/timedtext/"
             f"{timed_text_track.pk}/1640995200/1640995200.vtt",
             serializer.data["url"],
         )
         self.assertEqual(
-            f"https://abc.cloudfront.net/vod/{video.pk}/timedtext/"
+            f"https://abc.svc.edge.scw.cloud/vod/{video.pk}/timedtext/"
             f"{timed_text_track.pk}/1640995200/source.srt",
             serializer.data["source_url"],
         )