Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
remove REV from integration test task definitions
Browse files Browse the repository at this point in the history
This might be the most hacky thing we're doing here so far, but it's necessary for translations integration tests, where cached ancestor tasks will have a `REV` that doesn't match the `REF`, leading to errors such as:
```
[vcs 2024-12-20T16:35:08.992Z] executing ['git', 'config', '--global', '--add', 'safe.directory', '/builds/worker/checkouts/vcs']
[vcs 2024-12-20T16:35:08.994Z] executing ['git', 'clone', 'https://github.com/mozilla-releng/staging-firefox-translations-training', '/builds/worker/checkouts/vcs']
[vcs 2024-12-20T16:35:08.995Z] Cloning into '/builds/worker/checkouts/vcs'...
[vcs 2024-12-20T16:35:10.747Z] executing ['git', 'fetch', '--tags', '--force', 'https://github.com/mozilla-releng/staging-firefox-translations-training', 'refs/heads/main']
[vcs 2024-12-20T16:35:10.949Z] From https://github.com/mozilla-releng/staging-firefox-translations-training
[vcs 2024-12-20T16:35:10.949Z]  * branch            main       -> FETCH_HEAD
[vcs 2024-12-20T16:35:10.955Z] executing ['git', 'fetch', '--no-tags', 'https://github.com/mozilla-releng/staging-firefox-translations-training', 'refs/heads/main']
[vcs 2024-12-20T16:35:11.128Z] From https://github.com/mozilla-releng/staging-firefox-translations-training
[vcs 2024-12-20T16:35:11.128Z]  * branch            main       -> FETCH_HEAD
[vcs 2024-12-20T16:35:11.133Z] executing ['git', 'checkout', '-f', '-B', 'refs/heads/main', '876ad1a2de046271a45c979551c8650e5e1b75a2']
[vcs 2024-12-20T16:35:11.135Z] fatal: reference is not a tree: 876ad1a2de046271a45c979551c8650e5e1b75a2
```

Removing the `REV` will cause the `REF` to be used without issues:
```
[vcs 2024-12-20T18:15:48.936Z] executing ['git', 'config', '--global', '--add', 'safe.directory', '/builds/worker/checkouts/vcs']
[vcs 2024-12-20T18:15:48.938Z] executing ['git', 'clone', 'https://github.com/mozilla-releng/staging-firefox-translations-training', '/builds/worker/checkouts/vcs']
[vcs 2024-12-20T18:15:48.939Z] Cloning into '/builds/worker/checkouts/vcs'...
[vcs 2024-12-20T18:15:50.544Z] executing ['git', 'fetch', '--tags', '--force', 'https://github.com/mozilla-releng/staging-firefox-translations-training', 'refs/heads/main']
[vcs 2024-12-20T18:15:50.758Z] From https://github.com/mozilla-releng/staging-firefox-translations-training
[vcs 2024-12-20T18:15:50.758Z]  * branch            main       -> FETCH_HEAD
[vcs 2024-12-20T18:15:50.763Z] executing ['git', 'fetch', '--no-tags', 'https://github.com/mozilla-releng/staging-firefox-translations-training', 'refs/heads/main']
[vcs 2024-12-20T18:15:50.926Z] From https://github.com/mozilla-releng/staging-firefox-translations-training
[vcs 2024-12-20T18:15:50.926Z]  * branch            main       -> FETCH_HEAD
[vcs 2024-12-20T18:15:50.931Z] executing ['git', 'checkout', '-f', '-B', 'refs/heads/main', 'FETCH_HEAD']
[vcs 2024-12-20T18:15:51.929Z] Switched to a new branch 'refs/heads/main'
```
bhearsum committed Jan 29, 2025
1 parent 6d76009 commit 96fa36c
Showing 2 changed files with 26 additions and 2 deletions.
21 changes: 20 additions & 1 deletion taskcluster/fxci_config_taskgraph/util/integration.py
Original file line number Diff line number Diff line change
@@ -101,6 +101,20 @@ def _rewrite_task_datestamps(task_def):
return task_def


def _remove_task_revisions(task_def):
"""Rewrite revisions in task payloads to ensure that tasks do not refer to
out of date revisions."""
to_remove = set()
for k in task_def.get("payload", {}).get("env", {}):
if k.endswith("_REV"):
to_remove.add(k)

for k in to_remove:
del task_def["payload"]["env"][k]

return task_def


def find_tasks(
decision_index_path: str,
include_attrs: dict[str, list[str]],
@@ -153,6 +167,11 @@ def find_tasks(
# All datestamps come in as absolute ones, many of which
# will be in the past. We need to rewrite these to relative
# ones to make the task reschedulable.
tasks[upstream_task_id] = _rewrite_task_datestamps(task_def)
# We also need to remove absolute revisions from payloads
# to avoid issues with revisions not matching the refs
# that are given.
tasks[upstream_task_id] = _remove_task_revisions(
_rewrite_task_datestamps(task_def)
)

return tasks
7 changes: 6 additions & 1 deletion taskcluster/test/test_transforms_integration_test.py
Original file line number Diff line number Diff line change
@@ -424,7 +424,8 @@ def run_include_deps_test(run_test, *args, **kwargs):
"task": "toolchain1",
},
]
)
),
"FOO_REV": "abc123",
},
},
"tags": {},
@@ -533,6 +534,10 @@ def run_include_deps_test(run_test, *args, **kwargs):
assert isinstance(a["expires"], dict)
assert "relative-datestamp" in a["expires"]

# verify that `FOO_REV` is gone
if t["payload"].get("env", {}).get("FOO_REV"):
assert False, "FOO_REV should've been removed"

return ret


0 comments on commit 96fa36c

Please sign in to comment.