From 1215e24a32f871835fc70451f2c6d56b93a7457f Mon Sep 17 00:00:00 2001
From: Naomi Kirby <oskirby@gmail.com>
Date: Wed, 4 Dec 2024 14:56:14 -0800
Subject: [PATCH 1/4] Allow beetmover to receive optional exclude patterns
---
beetmoverscript/src/beetmoverscript/gcloud.py | 3 +-
beetmoverscript/src/beetmoverscript/script.py | 4 ++-
beetmoverscript/tests/test_gcloud.py | 35 +++++++++++++++++++
beetmoverscript/tests/test_script.py | 32 ++++++++++++++++-
4 files changed, 71 insertions(+), 3 deletions(-)
diff --git a/beetmoverscript/src/beetmoverscript/gcloud.py b/beetmoverscript/src/beetmoverscript/gcloud.py
index a464f9913..326dc594d 100644
--- a/beetmoverscript/src/beetmoverscript/gcloud.py
+++ b/beetmoverscript/src/beetmoverscript/gcloud.py
@@ -203,6 +203,7 @@ async def push_to_releases_gcs(context):
# Weed out RELEASE_EXCLUDE matches, but allow partners specified in the payload
push_partners = context.task["payload"].get("partners", [])
+ exclude = context.task["payload"].get("exclude", []) + list(RELEASE_EXCLUDE)
for blob_path in candidates_blobs.keys():
if "/partner-repacks/" in blob_path:
@@ -214,7 +215,7 @@ async def push_to_releases_gcs(context):
)
else:
log.debug("Excluding partner repack {}".format(blob_path))
- elif not matches_exclude(blob_path, RELEASE_EXCLUDE):
+ elif not matches_exclude(blob_path, exclude):
blobs_to_copy[blob_path] = blob_path.replace(candidates_prefix, releases_prefix)
else:
log.debug("Excluding {}".format(blob_path))
diff --git a/beetmoverscript/src/beetmoverscript/script.py b/beetmoverscript/src/beetmoverscript/script.py
index f1e600621..158e11aa4 100755
--- a/beetmoverscript/src/beetmoverscript/script.py
+++ b/beetmoverscript/src/beetmoverscript/script.py
@@ -226,6 +226,8 @@ async def push_to_releases_s3(context):
# Weed out RELEASE_EXCLUDE matches, but allow partners specified in the payload
push_partners = context.task["payload"].get("partners", [])
+ exclude = context.task["payload"].get("exclude", []) + list(RELEASE_EXCLUDE)
+
for k in candidates_keys_checksums.keys():
if "/partner-repacks/" in k:
partner_match = get_partner_match(k, candidates_prefix, push_partners)
@@ -236,7 +238,7 @@ async def push_to_releases_s3(context):
)
else:
log.debug("Excluding partner repack {}".format(k))
- elif not matches_exclude(k, RELEASE_EXCLUDE):
+ elif not matches_exclude(k, exclude):
context.artifacts_to_beetmove[k] = k.replace(candidates_prefix, releases_prefix)
else:
log.debug("Excluding {}".format(k))
diff --git a/beetmoverscript/tests/test_gcloud.py b/beetmoverscript/tests/test_gcloud.py
index a4ccc4292..e3b0034d3 100644
--- a/beetmoverscript/tests/test_gcloud.py
+++ b/beetmoverscript/tests/test_gcloud.py
@@ -10,6 +10,7 @@
from scriptworker.exceptions import ScriptWorkerTaskException
import beetmoverscript.gcloud
+from beetmoverscript.utils import get_candidates_prefix, get_releases_prefix
from . import get_fake_valid_task, noop_sync
@@ -233,6 +234,40 @@ def fake_list_bucket_objects_gcs_same(client, bucket, prefix):
await beetmoverscript.gcloud.push_to_releases_gcs(context)
+@pytest.mark.parametrize(
+ "candidate_blobs,exclude,results",
+ [
+ ({"foo/bar.zip": "md5hash", "foo/baz.exe": "abcd", "foo/qux.js": "shasum"}, [], ["foo/baz.exe", "foo/qux.js"]),
+ ({"foo/bar.zip": "md5hash", "foo/baz.exe": "abcd", "foo/qux.js": "shasum"}, [r"^.*\.exe$"], ["foo/qux.js"]),
+ ({"foo/bar.zip": "md5hash", "foo/baz.exe": "abcd", "foo/qux.js": "shasum"}, [r"^.*\.exe$", r"^.*\.js"], []),
+ ],
+)
+@pytest.mark.asyncio
+async def test_push_to_releases_gcs_exclude(context, monkeypatch, candidate_blobs, exclude, results):
+ context.gcs_client = FakeClient()
+ context.task = get_fake_valid_task("task_push_to_releases.json")
+
+ payload = context.task["payload"]
+ payload["exclude"] = exclude
+ test_candidate_prefix = get_candidates_prefix(payload["product"], payload["version"], payload["build_number"])
+ test_release_prefix = get_releases_prefix(payload["product"], payload["version"])
+
+ def fake_list_bucket_objects_gcs_same(client, bucket, prefix):
+ if "candidates" in prefix:
+ return {f"{prefix}{key}": value for (key, value) in candidate_blobs.items()}
+ if "releases" in prefix:
+ return {}
+
+ expect_blobs = {f"{test_candidate_prefix}{key}": f"{test_release_prefix}{key}" for key in results}
+ def fake_move_artifacts(client, bucket_name, blobs_to_copy, candidates_blobs, releases_blobs):
+ assert blobs_to_copy == expect_blobs
+
+ monkeypatch.setattr(beetmoverscript.gcloud, "list_bucket_objects_gcs", fake_list_bucket_objects_gcs_same)
+ monkeypatch.setattr(beetmoverscript.gcloud, "move_artifacts", fake_move_artifacts)
+
+ await beetmoverscript.gcloud.push_to_releases_gcs(context)
+
+
def test_list_bucket_objects_gcs():
beetmoverscript.gcloud.list_bucket_objects_gcs(FakeClient(), "foobucket", "prefix")
diff --git a/beetmoverscript/tests/test_script.py b/beetmoverscript/tests/test_script.py
index 076766941..69292d5ad 100644
--- a/beetmoverscript/tests/test_script.py
+++ b/beetmoverscript/tests/test_script.py
@@ -29,7 +29,7 @@
setup_mimetypes,
)
from beetmoverscript.task import get_release_props, get_upstream_artifacts
-from beetmoverscript.utils import generate_beetmover_manifest, is_promotion_action
+from beetmoverscript.utils import get_candidates_prefix, get_releases_prefix, generate_beetmover_manifest, is_promotion_action
from . import get_fake_valid_config, get_fake_valid_task, get_test_jinja_env, noop_async, noop_sync
@@ -69,6 +69,36 @@ def fake_list(*args):
else:
await push_to_releases_s3(context)
+# push_to_releases_s3_exclude {{{1
+@pytest.mark.parametrize(
+ "candidates_keys,exclude,results",
+ [
+ ({"foo.zip": "x", "bar.exe": "y", "baz.js": "z"}, [], ["bar.exe", "baz.js"]),
+ ({"foo.zip": "x", "bar.exe": "y", "baz.js": "z"}, [r"^.*\.exe$"], ["baz.js"]),
+ ({"foo.zip": "x", "bar.exe": "y", "baz.js": "z"}, [r"^.*\.exe$", r"^.*\.js"], []),
+ ],
+)
+@pytest.mark.asyncio
+async def test_push_to_releases_s3_exclude(context, mocker, candidates_keys, exclude, results):
+ payload = {"product": "devedition", "build_number": 33, "version": "99.0b44", "exclude": exclude}
+ context.task = {"payload": payload}
+ test_candidate_prefix = get_candidates_prefix(payload["product"], payload["version"], payload["build_number"])
+ test_release_prefix = get_releases_prefix(payload["product"], payload["version"])
+
+ objects = [{f"{test_candidate_prefix}{key}": candidates_keys[key] for key in candidates_keys}, {}]
+
+ expect_artifacts = {f"{test_candidate_prefix}{key}": f"{test_release_prefix}{key}" for key in results}
+ def check(ctx, _, r):
+ assert ctx.artifacts_to_beetmove == expect_artifacts
+
+ def fake_list(*args):
+ return objects.pop(0)
+
+ mocker.patch.object(boto3, "resource")
+ mocker.patch.object(beetmoverscript.script, "list_bucket_objects", new=fake_list)
+ mocker.patch.object(beetmoverscript.script, "copy_beets", new=check)
+
+ await push_to_releases_s3(context)
# copy_beets {{{1
@pytest.mark.parametrize("releases_keys,raises", (({}, False), ({"to2": "from2_md5"}, False), ({"to1": "to1_md5"}, True)))
From a9ea7439907633afe4afe597546158da0f865098 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
<66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 4 Dec 2024 22:58:58 +0000
Subject: [PATCH 2/4] style: pre-commit.ci auto fixes [...]
---
beetmoverscript/tests/test_gcloud.py | 1 +
beetmoverscript/tests/test_script.py | 3 +++
2 files changed, 4 insertions(+)
diff --git a/beetmoverscript/tests/test_gcloud.py b/beetmoverscript/tests/test_gcloud.py
index e3b0034d3..a5e58d611 100644
--- a/beetmoverscript/tests/test_gcloud.py
+++ b/beetmoverscript/tests/test_gcloud.py
@@ -259,6 +259,7 @@ def fake_list_bucket_objects_gcs_same(client, bucket, prefix):
return {}
expect_blobs = {f"{test_candidate_prefix}{key}": f"{test_release_prefix}{key}" for key in results}
+
def fake_move_artifacts(client, bucket_name, blobs_to_copy, candidates_blobs, releases_blobs):
assert blobs_to_copy == expect_blobs
diff --git a/beetmoverscript/tests/test_script.py b/beetmoverscript/tests/test_script.py
index 69292d5ad..b0a5fffe5 100644
--- a/beetmoverscript/tests/test_script.py
+++ b/beetmoverscript/tests/test_script.py
@@ -69,6 +69,7 @@ def fake_list(*args):
else:
await push_to_releases_s3(context)
+
# push_to_releases_s3_exclude {{{1
@pytest.mark.parametrize(
"candidates_keys,exclude,results",
@@ -88,6 +89,7 @@ async def test_push_to_releases_s3_exclude(context, mocker, candidates_keys, exc
objects = [{f"{test_candidate_prefix}{key}": candidates_keys[key] for key in candidates_keys}, {}]
expect_artifacts = {f"{test_candidate_prefix}{key}": f"{test_release_prefix}{key}" for key in results}
+
def check(ctx, _, r):
assert ctx.artifacts_to_beetmove == expect_artifacts
@@ -100,6 +102,7 @@ def fake_list(*args):
await push_to_releases_s3(context)
+
# copy_beets {{{1
@pytest.mark.parametrize("releases_keys,raises", (({}, False), ({"to2": "from2_md5"}, False), ({"to1": "to1_md5"}, True)))
def test_copy_beets(context, mocker, releases_keys, raises):
From 08f8e3aec1526242e68d068fbfe5d26355811968 Mon Sep 17 00:00:00 2001
From: Naomi Kirby <oskirby@gmail.com>
Date: Mon, 9 Dec 2024 09:38:01 -0800
Subject: [PATCH 3/4] Update the beetmover schema
---
.../src/beetmoverscript/data/beetmover_task_schema.json | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/beetmoverscript/src/beetmoverscript/data/beetmover_task_schema.json b/beetmoverscript/src/beetmoverscript/data/beetmover_task_schema.json
index a8583803b..024e34546 100644
--- a/beetmoverscript/src/beetmoverscript/data/beetmover_task_schema.json
+++ b/beetmoverscript/src/beetmoverscript/data/beetmover_task_schema.json
@@ -85,6 +85,14 @@
},
"minItems": 1,
"uniqueItems": true
+ },
+ "exclude": {
+ "type": "array",
+ "minItems": 0,
+ "uniqueItems": false,
+ "items": {
+ "type": "string"
+ }
}
},
"required": ["upload_date", "upstreamArtifacts", "releaseProperties"]
From bb05034cd86256b99b15cc27399f167e2bcf6c2f Mon Sep 17 00:00:00 2001
From: Naomi Kirby <oskirby@gmail.com>
Date: Mon, 9 Dec 2024 10:16:10 -0800
Subject: [PATCH 4/4] Actually, update all the schemas
---
.../data/artifactMap_beetmover_task_schema.json | 8 ++++++++
.../beetmoverscript/data/maven_beetmover_task_schema.json | 8 ++++++++
.../data/release_beetmover_task_schema.json | 8 ++++++++
3 files changed, 24 insertions(+)
diff --git a/beetmoverscript/src/beetmoverscript/data/artifactMap_beetmover_task_schema.json b/beetmoverscript/src/beetmoverscript/data/artifactMap_beetmover_task_schema.json
index 675443455..19a3c183e 100644
--- a/beetmoverscript/src/beetmoverscript/data/artifactMap_beetmover_task_schema.json
+++ b/beetmoverscript/src/beetmoverscript/data/artifactMap_beetmover_task_schema.json
@@ -89,6 +89,14 @@
"minItems": 1,
"uniqueItems": true
},
+ "exclude": {
+ "type": "array",
+ "minItems": 0,
+ "uniqueItems": false,
+ "items": {
+ "type": "string"
+ }
+ },
"artifactMap": {
"type": "array",
"items": {
diff --git a/beetmoverscript/src/beetmoverscript/data/maven_beetmover_task_schema.json b/beetmoverscript/src/beetmoverscript/data/maven_beetmover_task_schema.json
index 702e7528c..70b956613 100644
--- a/beetmoverscript/src/beetmoverscript/data/maven_beetmover_task_schema.json
+++ b/beetmoverscript/src/beetmoverscript/data/maven_beetmover_task_schema.json
@@ -82,6 +82,14 @@
},
"minItems": 1,
"uniqueItems": true
+ },
+ "exclude": {
+ "type": "array",
+ "minItems": 0,
+ "uniqueItems": false,
+ "items": {
+ "type": "string"
+ }
}
},
"required": ["upstreamArtifacts", "releaseProperties"]
diff --git a/beetmoverscript/src/beetmoverscript/data/release_beetmover_task_schema.json b/beetmoverscript/src/beetmoverscript/data/release_beetmover_task_schema.json
index a0b7bfe13..0cfe8678c 100644
--- a/beetmoverscript/src/beetmoverscript/data/release_beetmover_task_schema.json
+++ b/beetmoverscript/src/beetmoverscript/data/release_beetmover_task_schema.json
@@ -53,6 +53,14 @@
"minItems": 0,
"uniqueItems": true
},
+ "exclude": {
+ "type": "array",
+ "minItems": 0,
+ "uniqueItems": false,
+ "items": {
+ "type": "string"
+ }
+ },
"partners": {
"type": "array",
"minItems": 0,