[PR #1453/2ad56c8d backport][0.40] Honor PYTEST_MARK in make paralleltest

.ci/gen_certs.py

@@ -6,8 +6,8 @@
 # ///
 
 import argparse
-import os
 import sys
+from pathlib import Path
 
 import trustme
 
@@ -17,43 +17,41 @@ def main() -> None:
     parser.add_argument(
         "-d",
         "--dir",
-        default=os.getcwd(),
+        default=".",
         help="Directory where certificates and keys are written to. Defaults to cwd.",
     )
 
     args = parser.parse_args(sys.argv[1:])
-    cert_dir = args.dir
+    cert_dir = Path(args.dir)
 
-    if not os.path.isdir(cert_dir):
+    if not cert_dir.is_dir():
         raise ValueError(f"--dir={cert_dir} is not a directory")
 
     key_type = trustme.KeyType["ECDSA"]
 
     # Generate the CA certificate
     ca = trustme.CA(key_type=key_type)
     # Write the certificate the client should trust
-    ca_cert_path = os.path.join(cert_dir, "ca.pem")
+    ca_cert_path = cert_dir / "ca.pem"
     ca.cert_pem.write_to_path(path=ca_cert_path)
 
     # Generate the server certificate
     server_cert = ca.issue_cert("localhost", "127.0.0.1", "::1", key_type=key_type)
     # Write the certificate and private key the server should use
-    server_key_path = os.path.join(cert_dir, "server.key")
-    server_cert_path = os.path.join(cert_dir, "server.pem")
+    server_key_path = cert_dir / "server.key"
+    server_cert_path = cert_dir / "server.pem"
     server_cert.private_key_pem.write_to_path(path=server_key_path)
-    with open(server_cert_path, mode="w") as f:
-        f.truncate()
+    server_cert_path.write_text("")
     for blob in server_cert.cert_chain_pems:
         blob.write_to_path(path=server_cert_path, append=True)
 
     # Generate the client certificate
     client_cert = ca.issue_cert("admin@example.com", common_name="admin", key_type=key_type)
     # Write the certificate and private key the client should use
-    client_key_path = os.path.join(cert_dir, "client.key")
-    client_cert_path = os.path.join(cert_dir, "client.pem")
+    client_key_path = cert_dir / "client.key"
+    client_cert_path = cert_dir / "client.pem"
     client_cert.private_key_pem.write_to_path(path=client_key_path)
-    with open(client_cert_path, mode="w") as f:
-        f.truncate()
+    client_cert_path.write_text("")
     for blob in client_cert.cert_chain_pems:
         blob.write_to_path(path=client_cert_path, append=True)
 

.ci/run_container.sh

@@ -69,6 +69,9 @@ else
 fi
 export PULP_CONTENT_ORIGIN
 
+PULP_SECRET_KEY="$(python3 -c "import secrets; print(secrets.token_urlsafe(50))")"
+export PULP_SECRET_KEY
+
 "${CONTAINER_RUNTIME}" \
   run ${RM:+--rm} \
   --env S6_KEEP_ENV=1 \
@@ -79,6 +82,7 @@ export PULP_CONTENT_ORIGIN
   ${PULP_DOMAIN_ENABLED:+--env PULP_DOMAIN_ENABLED} \
   ${PULP_ENABLED_PLUGINS:+--env PULP_ENABLED_PLUGINS} \
   --env PULP_CONTENT_ORIGIN \
+  --env PULP_SECRET_KEY \
   --detach \
   --name "pulp-ephemeral" \
   --volume "${PULP_CLI_TEST_TMPDIR}/settings:/etc/pulp${SELINUX:+:Z}" \

.ci/scripts/check_click_for_mypy.py

@@ -5,7 +5,7 @@
 #     "packaging>=25.0,<25.1",
 # ]
 # ///
-
+import sys
 from importlib import metadata
 
 from packaging.version import Version
@@ -15,4 +15,4 @@
     if click_version < Version("8.1.1"):
         print("🚧 Linting with mypy is currently only supported with click>=8.1.1. 🚧")
         print("🔧 Please run `pip install click>=8.1.1` first. 🔨")
-        exit(1)
+        sys.exit(1)

.ci/scripts/collect_changes.py

@@ -8,15 +8,17 @@
 # ///
 
 import itertools
-import os
 import re
+import typing as t
+from pathlib import Path
 
 import tomllib
 from git import GitCommandError, Repo
+from packaging.version import Version
 from packaging.version import parse as parse_version
 
 # Read Towncrier settings
-with open("pyproject.toml", "rb") as fp:
+with Path("pyproject.toml").open("rb") as fp:
     tc_settings = tomllib.load(fp)["tool"]["towncrier"]
 
 CHANGELOG_FILE = tc_settings.get("filename", "NEWS.rst")
@@ -51,37 +53,36 @@
 )
 
 
-def get_changelog(repo, branch):
+def get_changelog(repo: Repo, branch: str) -> str:
     branch_tc_settings = tomllib.loads(repo.git.show(f"{branch}:pyproject.toml"))["tool"][
         "towncrier"
     ]
     branch_changelog_file = branch_tc_settings.get("filename", "NEWS.rst")
     return repo.git.show(f"{branch}:{branch_changelog_file}") + "\n"
 
 
-def _tokenize_changes(splits):
+def _tokenize_changes(splits: list[str]) -> t.Iterator[list[Version | str]]:
     assert len(splits) % 3 == 0
     for i in range(len(splits) // 3):
         title = splits[3 * i]
         version = parse_version(splits[3 * i + 1])
         yield [version, title + splits[3 * i + 2]]
 
 
-def split_changelog(changelog):
+def split_changelog(changelog: str) -> tuple[str, list[list[Version | str]]]:
     preamble, rest = changelog.split(START_STRING, maxsplit=1)
     split_rest = re.split(TITLE_REGEX, rest)
     return preamble + START_STRING + split_rest[0], list(_tokenize_changes(split_rest[1:]))
 
 
-def main():
-    repo = Repo(os.getcwd())
+def main() -> None:
+    repo = Repo(Path.cwd())
     remote = repo.remotes[0]
     branches = [ref for ref in remote.refs if re.match(r"^([0-9]+)\.([0-9]+)$", ref.remote_head)]
     branches.sort(key=lambda ref: parse_version(ref.remote_head), reverse=True)
     branches = [ref.name for ref in branches]
 
-    with open(CHANGELOG_FILE) as f:
-        main_changelog = f.read()
+    main_changelog = Path(CHANGELOG_FILE).read_text()
     preamble, main_changes = split_changelog(main_changelog)
     old_length = len(main_changes)
 
@@ -92,7 +93,7 @@ def main():
         except GitCommandError:
             print("No changelog found on this branch.")
             continue
-        dummy, changes = split_changelog(changelog)
+        _dummy, changes = split_changelog(changelog)
         new_changes = sorted(main_changes + changes, key=lambda x: x[0], reverse=True)
         # Now remove duplicates (retain the first one)
         main_changes = [new_changes[0]]
@@ -103,7 +104,7 @@ def main():
     new_length = len(main_changes)
     if old_length < new_length:
         print(f"{new_length - old_length} new versions have been added.")
-        with open(CHANGELOG_FILE, "w") as fp:
+        with Path(CHANGELOG_FILE).open("w") as fp:
             fp.write(preamble)
             fp.writelines(change[1] for change in main_changes)
 

.ci/scripts/pr_labels.py

@@ -19,7 +19,7 @@
 def main():
     assert len(sys.argv) == 3
 
-    with open("pyproject.toml", "rb") as fp:
+    with Path("pyproject.toml").open("rb") as fp:
         PYPROJECT_TOML = tomllib.load(fp)
     BLOCKING_REGEX = re.compile(r"DRAFT|WIP|NO\s*MERGE|DO\s*NOT\s*MERGE|EXPERIMENT")
     ISSUE_REGEX = re.compile(r"(?:fixes|closes)[\s:]+#(\d+)")

.ci/scripts/validate_commit_message.py

@@ -12,9 +12,8 @@
 from pathlib import Path
 
 import tomllib
-from github import Github
 
-with open("pyproject.toml", "rb") as fp:
+with Path("pyproject.toml").open("rb") as fp:
     PYPROJECT_TOML = tomllib.load(fp)
 KEYWORDS = ["fixes", "closes"]
 BLOCKING_REGEX = [
@@ -36,11 +35,13 @@
 if any(re.match(pattern, message) for pattern in BLOCKING_REGEX):
     sys.exit("This PR is not ready for consumption.")
 
-g = Github(os.environ.get("GITHUB_TOKEN"))
-repo = g.get_repo("pulp/pulp-cli")
-
 
 def check_status(issue: str) -> None:
+    from github import Github
+
+    g = Github(os.environ.get("GITHUB_TOKEN"))
+    repo = g.get_repo("pulp/pulp-cli")
+
     gi = repo.get_issue(int(issue))
     if gi.pull_request:
         sys.exit(f"Error: issue #{issue} is a pull request.")

.gitignore

@@ -4,6 +4,7 @@ build/
 site/
 dist/
 *.po~
+uv.lock
 
 tests/cli.toml
 GPG-PRIVATE-KEY-fixture-signing

CHANGES/+paralleltest-pytest-mark.misc

@@ -0,0 +1 @@
+Honor PYTEST_MARK in `make paralleltest`, matching livetest, so plugin CI can filter parallel livetests.

Makefile

@@ -3,6 +3,8 @@ LANGUAGES=de
 GLUE_PLUGINS=$(notdir $(wildcard pulp-glue/src/pulp_glue/*))
 CLI_PLUGINS=$(notdir $(wildcard src/pulpcore/cli/*))
 
+PYTEST_MARK ?= live
+
 .PHONY: info
 info:
 	@echo Pulp glue
@@ -29,22 +31,20 @@ _autofix:
 
 .PHONY: autofix
 autofix:
-	uv lock
 	uv run --isolated --group lint $(MAKE) _autofix
 
 .PHONY: _lint
 _lint:
 	find tests .ci -name '*.sh' -print0 | xargs -0 shellcheck -x
 	ruff format --check --diff
-	ruff check
+	ruff check --output-format concise
 	.ci/scripts/check_click_for_mypy.py
 	mypy
 	cd pulp-glue; mypy
 	@echo "🙊 Code 🙈 LGTM 🙉 !"
 
 .PHONY: lint
 lint:
-	uv lock --check
 	uv run --isolated --group lint $(MAKE) _lint
 
 tests/cli.toml:
@@ -53,14 +53,12 @@ tests/cli.toml:
 
 .PHONY: _test
 _test: | tests/cli.toml
-	pytest -v tests pulp-glue/tests cookiecutter/pulp_filter_extension.py
+	pytest -v tests pulp-glue/tests
 
 .PHONY: test
 test:
 	uv run $(MAKE) _test
 
-PYTEST_MARK ?= live
-
 .PHONY: _livetest
 _livetest: | tests/cli.toml
 	pytest -v tests pulp-glue/tests -m "$(PYTEST_MARK)"
@@ -71,15 +69,15 @@ livetest:
 
 .PHONY: _paralleltest
 _paralleltest: | tests/cli.toml
-	pytest -v tests pulp-glue/tests -m live -n 8
+	pytest -v tests pulp-glue/tests -m "$(PYTEST_MARK)" -n 8
 
 .PHONY: paralleltest
 paralleltest:
 	uv run $(MAKE) _paralleltest
 
 .PHONY: _unittest
 _unittest:
-	pytest -v tests pulp-glue/tests cookiecutter/pulp_filter_extension.py -m "not live"
+	pytest -v tests pulp-glue/tests -m "not live"
 
 .PHONY: unittest
 unittest:
@@ -95,11 +93,11 @@ unittest_glue:
 
 .PHONY: docs
 docs:
-	pulp-docs build
+	uv run --only-group docs pulp-docs build --draft --no-blog
 
 .PHONY: servedocs
 servedocs:
-	pulp-docs serve -w CHANGES.md -w pulp-glue/pulp_glue -w pulp_cli/generic.py
+	uv run --only-group docs pulp-docs serve --draft --no-blog -w CHANGES.md -w src -w pulp-glue/src
 
 pulp-glue/pulp_glue/%/locale/messages.pot: pulp-glue/pulp_glue/%/*.py
 	xgettext -d $* -o $@ pulp-glue/pulp_glue/$*/*.py

pyproject.toml

@@ -70,23 +70,30 @@ lint = [
   "types-toml",
 ]
 test = [
-  "jinja2>=3.1.4,<3.2",
   "pygments>=2.19.2",
-  "pytest>=7.0.0,<9.1",
+  "pytest>=7.0.0,<9.2",
   "pytest-xdist>=3.8.0,<3.9",
   "python-gnupg>=0.5.0,<0.6",
   "secretstorage>=3.5.0",
   "trustme>=1.1.0,<1.3",
 ]
+docs = [
+  "pulp-docs",
+]
 
 [tool.uv.sources]
 # This section is managed by the cookiecutter templates.
 pulp-glue = { workspace = true }
+pulp-docs = { git = "https://github.com/pulp/pulp-docs" }
 
 [tool.uv.workspace]
 # This section is managed by the cookiecutter templates.
 members = ["pulp-glue"]
 
+[tool.uv.dependency-groups]
+# This section is managed by the cookiecutter templates.
+docs = {requires-python = ">=3.11"}
+
 [tool.uv.build-backend]
 # This section is managed by the cookiecutter templates.
 module-name = ["pulpcore.cli", "pulp_cli", "pytest_pulp_cli"]
@@ -123,7 +130,6 @@ paralleltests = true
 # This section is managed by the cookiecutter templates.
 current_version = "0.40.2.dev"
 commit = false
-pre_commit_hooks = ["uv lock", "git add uv.lock"]
 tag = false
 parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)(\\.(?P<release>[a-z]+))?"
 serialize = [