Migrate to Django 5 storages

Author: gerrod3

.ci/assets/ci_constraints.txt

@@ -1,5 +1,6 @@
 # Pulpcore versions without the openapi command do no longer work in the CI
-pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.*
+# Pulpcore versions without the django 5 storage compatibility will fail, >3.63,<3.70
+pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.*,!=3.64.*,!=3.65.*,!=3.66.*,!=3.67.*,!=3.68.*,!=3.69.*
 
 
 tablib!=3.6.0

.ci/scripts/check_release.py

@@ -1,11 +1,20 @@
 #!/usr/bin/env python
+# /// script
+# requires-python = ">=3.13"
+# dependencies = [
+#     "gitpython>=3.1.46,<3.2.0",
+#     "packaging>=26.0,<26.1",
+#     "pyyaml>=6.0.3,<6.1.0",
+# ]
+# ///
 
 import argparse
 import re
 import os
 import tomllib
-import yaml
 from pathlib import Path
+
+import yaml
 from packaging.version import Version
 from git import Repo
 

.github/workflows/ci.yml

@@ -94,13 +94,13 @@ jobs:
       run_docs: ${{ needs.check-changes.outputs.run_docs }}
 
   lint:
-    needs:
-      - "check-changes"
-    if: needs.check-changes.outputs.run_tests == '1'
     uses: "./.github/workflows/lint.yml"
 
   build:
-    needs: "lint"
+    needs:
+      - "check-changes"
+      - "lint"
+    if: needs.check-changes.outputs.run_tests == '1'
     uses: "./.github/workflows/build.yml"
 
   test:

.github/workflows/scripts/install.sh

@@ -95,7 +95,7 @@ if [ "$TEST" = "s3" ]; then
   sed -i -e '$a s3_test: true\
 minio_access_key: "'$MINIO_ACCESS_KEY'"\
 minio_secret_key: "'$MINIO_SECRET_KEY'"\
-pulp_scenario_settings: {"AWS_ACCESS_KEY_ID": "AKIAIT2Z5TDYPX3ARJBA", "AWS_DEFAULT_ACL": "@none None", "AWS_S3_ADDRESSING_STYLE": "path", "AWS_S3_ENDPOINT_URL": "http://minio:9000", "AWS_S3_REGION_NAME": "eu-central-1", "AWS_S3_SIGNATURE_VERSION": "s3v4", "AWS_SECRET_ACCESS_KEY": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "AWS_STORAGE_BUCKET_NAME": "pulp3", "DEFAULT_FILE_STORAGE": "storages.backends.s3boto3.S3Boto3Storage", "MEDIA_ROOT": ""}\
+pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}}\
 pulp_scenario_env: {}\
 ' vars/main.yaml
   export PULP_API_ROOT="/rerouted/djnd/"
@@ -109,7 +109,7 @@ if [ "$TEST" = "azure" ]; then
       - ./azurite:/etc/pulp\
     command: "azurite-blob --skipApiVersionCheck --blobHost 0.0.0.0"' vars/main.yaml
   sed -i -e '$a azure_test: true\
-pulp_scenario_settings: {"AZURE_ACCOUNT_KEY": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "AZURE_ACCOUNT_NAME": "devstoreaccount1", "AZURE_CONNECTION_STRING": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "AZURE_CONTAINER": "pulp-test", "AZURE_LOCATION": "pulp3", "AZURE_OVERWRITE_FILES": true, "AZURE_URL_EXPIRATION_SECS": 120, "DEFAULT_FILE_STORAGE": "storages.backends.azure_storage.AzureStorage", "MEDIA_ROOT": ""}\
+pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.azure_storage.AzureStorage", "OPTIONS": {"account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "account_name": "devstoreaccount1", "azure_container": "pulp-test", "connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "expiration_secs": 120, "location": "pulp3", "overwrite_files": true}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}}\
 pulp_scenario_env: {}\
 ' vars/main.yaml
 fi

pulp_python/app/models.py

@@ -4,7 +4,6 @@
 from django.contrib.postgres.fields import ArrayField
 from django.core.exceptions import ObjectDoesNotExist
 from django.db import models
-from django.conf import settings
 from pulpcore.plugin.models import (
     Content,
     Publication,
@@ -13,6 +12,7 @@
     Repository,
 )
 from pulpcore.plugin.responses import ArtifactResponse
+from pulpcore.plugin.util import get_domain
 
 from pathlib import PurePath
 from .utils import (
@@ -77,7 +77,7 @@ def content_handler(self, path):
         elif len(path.parts) and path.parts[0] == "simple":
             # Temporary fix for PublishedMetadata not being properly served from remote storage
             # https://github.com/pulp/pulp_python/issues/413
-            if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem":
+            if get_domain().storage_class != "pulpcore.app.models.storage.FileSystem":
                 if self.publication or self.repository:
                     try:
                         publication = self.publication or Publication.objects.filter(

pulp_python/app/utils.py

@@ -3,7 +3,6 @@
 import tempfile
 import json
 from collections import defaultdict
-from django.core.files.storage import default_storage as storage
 from django.conf import settings
 from jinja2 import Template
 from packaging.utils import canonicalize_name
@@ -144,8 +143,8 @@ def get_project_metadata_from_artifact(filename, artifact):
     # because pkginfo validates that the filename has a valid extension before
     # reading it
     with tempfile.NamedTemporaryFile('wb', dir=".", suffix=filename) as temp_file:
-        artifact_file = storage.open(artifact.file.name)
-        shutil.copyfileobj(artifact_file, temp_file)
+        artifact.file.seek(0)
+        shutil.copyfileobj(artifact.file, temp_file)
         temp_file.flush()
         metadata = DIST_TYPES[packagetype](temp_file.name)
         metadata.packagetype = packagetype

template_config.yml

@@ -2,18 +2,21 @@
 # were not present before running plugin-template have been added with their default values.
 
 # generated with plugin_template
+#
+# After editing this file please always reapply the plugin template before committing any changes.
 
-api_root: /pulp/
+---
+api_root: "/pulp/"
 black: false
 check_commit_message: true
 check_gettext: true
 check_manifest: true
 check_stray_pulpcore_imports: true
-ci_base_image: ghcr.io/pulp/pulp-ci-centos9
+ci_base_image: "ghcr.io/pulp/pulp-ci-centos9"
 ci_env: {}
-ci_trigger: '{pull_request: {branches: [''*'']}}'
-cli_package: pulp-cli
-cli_repo: https://github.com/pulp/pulp-cli.git
+ci_trigger: "{pull_request: {branches: ['*']}}"
+cli_package: "pulp-cli"
+cli_repo: "https://github.com/pulp/pulp-cli.git"
 core_import_allowed: []
 deploy_client_to_pypi: true
 deploy_client_to_rubygems: true
@@ -23,54 +26,62 @@ docker_fixtures: false
 extra_files: []
 flake8: true
 flake8_ignore: []
-github_org: pulp
+github_org: "pulp"
 latest_release_branch: null
 lint_requirements: true
 os_required_packages: []
 parallel_test_workers: 8
-plugin_app_label: python
-plugin_default_branch: main
-plugin_name: pulp_python
+plugin_app_label: "python"
+plugin_default_branch: "main"
+plugin_name: "pulp_python"
 plugins:
-- app_label: python
-  name: pulp_python
-post_job_template: null
-pre_job_template: null
+  - app_label: "python"
+    name: "pulp_python"
 pulp_env: {}
 pulp_env_azure: {}
 pulp_env_gcp: {}
 pulp_env_s3: {}
-pulp_scheme: https
+pulp_scheme: "https"
 pulp_settings:
-  allowed_export_paths: /tmp
-  allowed_import_paths: /tmp
+  allowed_export_paths: "/tmp"
+  allowed_import_paths: "/tmp"
   orphan_protection_time: 0
-  pypi_api_hostname: https://pulp:443
+  pypi_api_hostname: "https://pulp:443"
 pulp_settings_azure:
-  AZURE_ACCOUNT_KEY: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
-  AZURE_ACCOUNT_NAME: devstoreaccount1
-  AZURE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;
-  AZURE_CONTAINER: pulp-test
-  AZURE_LOCATION: pulp3
-  AZURE_OVERWRITE_FILES: true
-  AZURE_URL_EXPIRATION_SECS: 120
-  DEFAULT_FILE_STORAGE: storages.backends.azure_storage.AzureStorage
-  MEDIA_ROOT: ''
+  MEDIA_ROOT: ""
+  STORAGES:
+    default:
+      BACKEND: "storages.backends.azure_storage.AzureStorage"
+      OPTIONS:
+        account_key: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+        account_name: "devstoreaccount1"
+        azure_container: "pulp-test"
+        connection_string: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;"
+        expiration_secs: 120
+        location: "pulp3"
+        overwrite_files: true
+    staticfiles:
+      BACKEND: "django.contrib.staticfiles.storage.StaticFilesStorage"
 pulp_settings_gcp: null
 pulp_settings_s3:
-  AWS_ACCESS_KEY_ID: AKIAIT2Z5TDYPX3ARJBA
-  AWS_DEFAULT_ACL: '@none None'
-  AWS_S3_ADDRESSING_STYLE: path
-  AWS_S3_ENDPOINT_URL: http://minio:9000
-  AWS_S3_REGION_NAME: eu-central-1
-  AWS_S3_SIGNATURE_VERSION: s3v4
-  AWS_SECRET_ACCESS_KEY: fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS
-  AWS_STORAGE_BUCKET_NAME: pulp3
-  DEFAULT_FILE_STORAGE: storages.backends.s3boto3.S3Boto3Storage
-  MEDIA_ROOT: ''
+  MEDIA_ROOT: ""
+  STORAGES:
+    default:
+      BACKEND: "storages.backends.s3boto3.S3Boto3Storage"
+      OPTIONS:
+        access_key: "AKIAIT2Z5TDYPX3ARJBA"
+        addressing_style: "path"
+        bucket_name: "pulp3"
+        default_acl: "@none"
+        endpoint_url: "http://minio:9000"
+        region_name: "eu-central-1"
+        secret_key: "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS"
+        signature_version: "s3v4"
+    staticfiles:
+      BACKEND: "django.contrib.staticfiles.storage.StaticFilesStorage"
 pydocstyle: true
-release_email: pulp-infra@redhat.com
-release_user: pulpbot
+release_email: "pulp-infra@redhat.com"
+release_user: "pulpbot"
 stalebot: true
 stalebot_days_until_close: 30
 stalebot_days_until_stale: 90
@@ -86,4 +97,5 @@ test_performance: false
 test_reroute: true
 test_s3: true
 use_issue_template: true
+...