Skip to content

Commit c800fe3

Browse files
wilbrdtwilbrdt
committed
🧱(backend) serve peertube videos via Scaleway Edge Services
Peertube-transcoded videos are stored in Scaleway S3. To migrate away from Cloudfront, let's update Marsha so the videos are served using Scaleway Edge Services. Note that Cloudfront is still used to served other media files (document, deposited files, classroom document etc.) and static files.
1 parent e592732 commit c800fe3

File tree

9 files changed

+47
-22
lines changed

9 files changed

+47
-22
lines changed

‎.circleci/config.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,7 @@ jobs:
267267
DJANGO_AWS_ACCESS_KEY_ID: aws-access-key-id
268268
DJANGO_AWS_SECRET_ACCESS_KEY: aws-secret-access-key
269269
DJANGO_CLOUDFRONT_DOMAIN: abc.cloudfront.net
270+
DJANGO_SCW_EDGE_SERVICE_DOMAIN: abc.svc.edge.scw.cloud
270271
DJANGO_UPDATE_STATE_SHARED_SECRETS: dummy,secret
271272
DJANGO_AWS_MEDIALIVE_ROLE_ARN: aws:medialive:arn:role
272273
DJANGO_AWS_MEDIAPACKAGE_HARVEST_JOB_ARN: aws:mediapackage:arn:role
@@ -349,6 +350,7 @@ jobs:
349350
DJANGO_AWS_ACCESS_KEY_ID: aws-access-key-id
350351
DJANGO_AWS_SECRET_ACCESS_KEY: aws-secret-access-key
351352
DJANGO_CLOUDFRONT_DOMAIN: abc.cloudfront.net
353+
DJANGO_SCW_EDGE_SERVICE_DOMAIN: abc.svc.edge.scw.cloud
352354
DJANGO_UPDATE_STATE_SHARED_SECRETS: dummy,secret
353355
DJANGO_AWS_MEDIALIVE_ROLE_ARN: aws:medialive:arn:role
354356
DJANGO_AWS_MEDIAPACKAGE_HARVEST_JOB_ARN: aws:mediapackage:arn:role
@@ -589,6 +591,7 @@ jobs:
589591
DJANGO_AWS_ACCESS_KEY_ID: aws-access-key-id
590592
DJANGO_AWS_SECRET_ACCESS_KEY: aws-secret-access-key
591593
DJANGO_CLOUDFRONT_DOMAIN: abc.cloudfront.net
594+
DJANGO_SCW_EDGE_SERVICE_DOMAIN: abc.svc.edge.scw.cloud
592595
DJANGO_UPDATE_STATE_SHARED_SECRETS: dummy,secret
593596
DJANGO_AWS_MEDIALIVE_ROLE_ARN: aws:medialive:arn:role
594597
DJANGO_AWS_MEDIAPACKAGE_HARVEST_JOB_ARN: aws:mediapackage:arn:role

‎CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,10 @@ Versioning](https://semver.org/spec/v2.0.0.html).
88

99
## [Unreleased]
1010

11+
### Added
12+
13+
- Add SCW_EDGE_SERVICE_DOMAIN setting for serving videos with SCW Edge Services
14+
1115
### Changed
1216

1317
- Increase connection timeout on Nginx for peertube runner success request

‎docs/env.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,15 @@ that will be used to distribute processed files to end users.
291291
- Required: Yes
292292
- Default: None
293293

294+
#### DJANGO_SCW_EDGE_SERVICE_DOMAIN
295+
296+
The domain for the Scaleway Edge Service.
297+
This is the domain that will be used to distribute videos from Scaleway S3 to end users.
298+
299+
- Type: string
300+
- Required: Yes
301+
- Default: None
302+
294303
### XMPP settings
295304

296305
#### DJANGO_LIVE_CHAT_ENABLED

‎env.d/test‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ DJANGO_AWS_MEDIAPACKAGE_HARVEST_JOB_ARN=aws:mediapackage:arn:role
1515

1616
# AWS
1717
DJANGO_CLOUDFRONT_DOMAIN=abc.cloudfront.net
18+
DJANGO_SCW_EDGE_SERVICE_DOMAIN=abc.svc.edge.scw.cloud
1819
DJANGO_UPDATE_STATE_SHARED_SECRETS=dummy,secret
1920

2021
# BBB

‎src/backend/marsha/core/serializers/video.py‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -207,13 +207,7 @@ def get_vod_urls(self, obj):
207207

208208
urls = {"mp4": {}, "thumbnails": {}, "manifests": {}}
209209

210-
base = f"{settings.AWS_S3_URL_PROTOCOL}://{settings.CLOUDFRONT_DOMAIN}/{obj.pk}"
211210
stamp = time_utils.to_timestamp(obj.uploaded_on)
212-
if settings.CLOUDFRONT_SIGNED_URLS_ACTIVE:
213-
params = get_video_cloudfront_url_params(obj.pk)
214-
215-
filename = f"{slugify(obj.playlist.title)}_{stamp}.mp4"
216-
content_disposition = quote_plus(f"attachment; filename={filename}")
217211

218212
# Trying to recover the transcoding pipeline
219213
if obj.transcode_pipeline is None:
@@ -228,6 +222,13 @@ def get_vod_urls(self, obj):
228222
)
229223

230224
if obj.transcode_pipeline == AWS_PIPELINE:
225+
base = f"{settings.AWS_S3_URL_PROTOCOL}://{settings.CLOUDFRONT_DOMAIN}/{obj.pk}"
226+
if settings.CLOUDFRONT_SIGNED_URLS_ACTIVE:
227+
params = get_video_cloudfront_url_params(obj.pk)
228+
229+
filename = f"{slugify(obj.playlist.title)}_{stamp}.mp4"
230+
content_disposition = quote_plus(f"attachment; filename={filename}")
231+
231232
for resolution in obj.resolutions:
232233
# MP4
233234
mp4_url = (
@@ -255,6 +256,7 @@ def get_vod_urls(self, obj):
255256

256257
# Previews
257258
urls["previews"] = f"{base}/previews/{stamp}_100.jpg"
259+
258260
elif obj.transcode_pipeline == PEERTUBE_PIPELINE:
259261
base = obj.get_videos_storage_prefix(stamp=stamp)
260262
for resolution in obj.resolutions:

‎src/backend/marsha/core/storage/s3.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ class S3VideoStorage(S3Storage):
3232

3333
object_parameters = settings.VIDEOS_STORAGE_S3_OBJECT_PARAMETERS
3434

35-
custom_domain = settings.CLOUDFRONT_DOMAIN
35+
custom_domain = settings.SCW_EDGE_SERVICE_DOMAIN
3636
url_protocol = "https:"
3737

3838
if settings.CLOUDFRONT_SIGNED_URLS_ACTIVE:

‎src/backend/marsha/core/tests/serializers/test_video.py‎

Lines changed: 18 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ def test_video_serializer_urls_with_aws_pipeline(self):
4545
)
4646

4747
@override_settings(
48-
MEDIA_URL="https://abc.cloudfront.net/",
48+
MEDIA_URL="https://abc.svc.edge.scw.cloud/",
4949
)
5050
def test_video_serializer_urls_with_peertube_pipeline(self):
5151
"""The VideoBaseSerializer should return the right URLs."""
@@ -63,20 +63,20 @@ def test_video_serializer_urls_with_peertube_pipeline(self):
6363
serializer = VideoBaseSerializer(video)
6464

6565
self.assertEqual(
66-
f"https://abc.cloudfront.net/vod/{video.pk}/video/1640995200/thumbnail.jpg",
66+
f"https://abc.svc.edge.scw.cloud/vod/{video.pk}/video/1640995200/thumbnail.jpg",
6767
serializer.data["urls"]["thumbnails"][1080],
6868
)
6969
self.assertEqual(
70-
f"https://abc.cloudfront.net/vod/{video.pk}/video/1640995200/master.m3u8?"
70+
f"https://abc.svc.edge.scw.cloud/vod/{video.pk}/video/1640995200/master.m3u8?"
7171
"v=25301066da0654a2967176ea47c26f6735b2cfcbd206f49fe1379f5103bb7b1f",
7272
serializer.data["urls"]["manifests"]["hls"],
7373
)
7474
self.assertEqual(
75-
f"https://abc.cloudfront.net/vod/{video.pk}/video/1640995200/thumbnail.jpg",
75+
f"https://abc.svc.edge.scw.cloud/vod/{video.pk}/video/1640995200/thumbnail.jpg",
7676
serializer.data["urls"]["previews"],
7777
)
7878
self.assertTrue(
79-
f"https://abc.cloudfront.net/vod/{video.pk}/"
79+
f"https://abc.svc.edge.scw.cloud/vod/{video.pk}/"
8080
"video/1640995200/1640995200-1080-fragmented.mp4"
8181
in serializer.data["urls"]["mp4"][1080]
8282
)
@@ -117,7 +117,7 @@ def test_video_serializer_urls_with_no_pipeline(self):
117117
)
118118

119119
@override_settings(
120-
MEDIA_URL="https://abc.cloudfront.net/",
120+
MEDIA_URL="https://abc.svc.edge.scw.cloud/",
121121
)
122122
def test_video_serializer_urls_with_no_pipeline_recovered_to_peertube(self):
123123
"""The VideoBaseSerializer should return the right URLs with Peertube pipeline."""
@@ -129,29 +129,32 @@ def test_video_serializer_urls_with_no_pipeline_recovered_to_peertube(self):
129129
uploaded_on=date,
130130
)
131131

132-
with mock.patch(
133-
"marsha.core.serializers.video.capture_message"
134-
) as sentry_capture_message, mock.patch(
135-
"marsha.core.serializers.video.video_storage"
136-
) as mock_video_storage:
132+
with (
133+
mock.patch(
134+
"marsha.core.serializers.video.capture_message"
135+
) as sentry_capture_message,
136+
mock.patch(
137+
"marsha.core.serializers.video.video_storage"
138+
) as mock_video_storage,
139+
):
137140
mock_video_storage.url = storage_class.video_storage.url
138141
mock_video_storage.exists.return_value = True
139142

140143
serializer = VideoBaseSerializer(video)
141144
self.assertEqual(
142-
f"https://abc.cloudfront.net/vod/{video.pk}/video/1640995200/thumbnail.jpg",
145+
f"https://abc.svc.edge.scw.cloud/vod/{video.pk}/video/1640995200/thumbnail.jpg",
143146
serializer.data["urls"]["thumbnails"][1080],
144147
)
145148
self.assertEqual(
146-
f"https://abc.cloudfront.net/vod/{video.pk}/video/1640995200/master.m3u8",
149+
f"https://abc.svc.edge.scw.cloud/vod/{video.pk}/video/1640995200/master.m3u8",
147150
serializer.data["urls"]["manifests"]["hls"],
148151
)
149152
self.assertEqual(
150-
f"https://abc.cloudfront.net/vod/{video.pk}/video/1640995200/thumbnail.jpg",
153+
f"https://abc.svc.edge.scw.cloud/vod/{video.pk}/video/1640995200/thumbnail.jpg",
151154
serializer.data["urls"]["previews"],
152155
)
153156
self.assertTrue(
154-
f"https://abc.cloudfront.net/vod/{video.pk}/"
157+
f"https://abc.svc.edge.scw.cloud/vod/{video.pk}/"
155158
"video/1640995200/1640995200-1080-fragmented.mp4"
156159
in serializer.data["urls"]["mp4"][1080]
157160
)

‎src/backend/marsha/settings.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -385,6 +385,8 @@ class Base(Configuration):
385385
{"ContentDisposition": "attachment"}
386386
)
387387

388+
SCW_EDGE_SERVICE_DOMAIN = values.Value(None)
389+
388390
# LTI Config
389391
LTI_CONFIG_TITLE = values.Value("Marsha")
390392
LTI_CONFIG_TITLES = values.DictValue(

‎src/tray/templates/services/app/secret.yml.j2‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ data:
1414
DJANGO_UPDATE_STATE_SHARED_SECRETS: "{{ MARSHA_VAULT.DJANGO_UPDATE_STATE_SHARED_SECRETS | default('secret') | b64encode }}"
1515
DJANGO_CLOUDFRONT_ACCESS_KEY_ID: "{{ MARSHA_VAULT.DJANGO_CLOUDFRONT_ACCESS_KEY_ID | default('secret') | b64encode }}"
1616
DJANGO_CLOUDFRONT_DOMAIN: "{{ MARSHA_VAULT.DJANGO_CLOUDFRONT_DOMAIN | default('foo.com') | b64encode }}"
17+
DJANGO_SCW_EDGE_SERVICE_DOMAIN: "{{ MARSHA_VAULT.DJANGO_SCW_EDGE_SERVICE_DOMAIN | default('foo.com') | b64encode }}"
1718
DJANGO_JWT_SIGNING_KEY: "{{ MARSHA_VAULT.DJANGO_JWT_SIGNING_KEY | default('secret') | b64encode }}"
1819
DJANGO_SECRET_KEY: "{{ MARSHA_VAULT.DJANGO_SECRET_KEY | default('supersecret') | b64encode }}"
1920
DJANGO_AWS_MEDIALIVE_ROLE_ARN: "{{ MARSHA_VAULT.DJANGO_AWS_MEDIALIVE_ROLE_ARN | default('secret') | b64encode }}"

0 commit comments

Comments
 (0)