Remove django-extensions package

* Add custom UUIDField and use it for uuid model attributes
* Remove django_extensions from existing migrations
* Add migrations for current changes
* Update requirements

Co-authored-by: Douglas Cerna (Soy Douglas) <[email protected]>

Author: Dhwaniartefact

hack/submodules/archivematica-storage-service

@@ -64,8 +64,6 @@ django-cas-ng==3.6.0
     # via -r requirements.txt
 django-csp==3.7
     # via -r requirements.txt
-django-extensions==1.7.9
-    # via -r requirements.txt
 django-forms-bootstrap==3.1.0
     # via -r requirements.txt
 django-prometheus==2.0.0
@@ -251,7 +249,6 @@ six==1.16.0
     # via
     #   -r requirements.txt
     #   amclient
-    #   django-extensions
     #   jsonschema
     #   mozilla-django-oidc
     #   opf-fido

requirements-dev.txt

@@ -8,7 +8,6 @@ brotli
 clamd
 django-autoslug==1.9.8
 django-csp
-django-extensions==1.7.9
 django-forms-bootstrap>=3.0.0,<4.0.0
 django-prometheus>=2.0,<2.1
 django-tastypie==0.14.3

requirements.in

@@ -46,8 +46,6 @@ django-cas-ng==3.6.0
     # via -r requirements.in
 django-csp==3.7
     # via -r requirements.in
-django-extensions==1.7.9
-    # via -r requirements.in
 django-forms-bootstrap==3.1.0
     # via -r requirements.in
 django-prometheus==2.0.0
@@ -152,7 +150,6 @@ requests==2.27.1
 six==1.16.0
     # via
     #   amclient
-    #   django-extensions
     #   jsonschema
     #   mozilla-django-oidc
     #   opf-fido

requirements.txt

@@ -67,7 +67,7 @@ def process_xml_metadata(mets, sip_dir, sip_uuid, sip_type, xml_validation):
                     continue
                 valid, errors = _validate_xml(tree, schema_uri)
                 _add_validation_event(
-                    mets, metadata_file.uuid, schema_uri, valid, errors
+                    mets, str(metadata_file.uuid), schema_uri, valid, errors
                 )
                 if not valid:
                     xml_metadata_errors += errors
@@ -251,5 +251,5 @@ def _add_validation_event(mets, file_uuid, schema_uri, valid, errors):
         "eventOutcomeDetailNote": "\n".join([str(err) for err in errors]),
     }
     event_object = insertIntoEvents(file_uuid, **event_data)
-    metadata_fsentry = mets.get_file(file_uuid=file_uuid)
+    metadata_fsentry = mets.get_file(file_uuid=str(file_uuid))
     metadata_fsentry.add_premis_event(createmets2.createEvent(event_object))

src/MCPClient/lib/clientScripts/archivematicaCreateMETSMetadataXML.py

@@ -72,7 +72,11 @@ def update_object(job, mets):
     # Iterate through original files
     for fsentry in mets.all_files():
         # Only update original files
-        if fsentry.use != "original" or fsentry.type != "Item" or not fsentry.file_uuid:
+        if (
+            fsentry.use != "original"
+            or fsentry.type != "Item"
+            or not str(fsentry.file_uuid)
+        ):
             continue
 
         # Copy techMD
@@ -101,7 +105,7 @@ def update_object(job, mets):
             fixity.find("premis:messageDigest", namespaces=ns.NSMAP).text = f.checksum
 
         # If FileID exists, update file ID
-        if models.FileID.objects.filter(file_id=fsentry.file_uuid):
+        if models.FileID.objects.filter(file_id=str(fsentry.file_uuid)):
             job.pyprint("Updating format for", fsentry.file_uuid)
             modified = True
             # Delete old formats
@@ -119,7 +123,7 @@ def update_object(job, mets):
 
         # If FPCommand output exists, update objectCharacteristicsExtension
         if models.FPCommandOutput.objects.filter(
-            file_id=fsentry.file_uuid,
+            file_id=str(fsentry.file_uuid),
             rule__purpose__in=["characterization", "default_characterization"],
         ).exists():
             job.pyprint(
@@ -143,7 +147,7 @@ def update_object(job, mets):
 
         # If Derivation exists, update relationships
         if models.Derivation.objects.filter(
-            source_file_id=fsentry.file_uuid, event__isnull=False
+            source_file_id=str(fsentry.file_uuid), event__isnull=False
         ):
             job.pyprint("Updating relationships for", fsentry.file_uuid)
             modified = True
@@ -375,7 +379,7 @@ def add_events(job, mets, sip_uuid):
     for event in models.Event.objects.filter(file_uuid__sip__uuid=sip_uuid).iterator():
         job.pyprint("Adding", event.event_type, "event to file", event.file_uuid_id)
         try:
-            fsentry = fsentries[event.file_uuid_id]
+            fsentry = fsentries[str(event.file_uuid_id)]
         except KeyError:
             job.pyprint(
                 "File with UUID",
@@ -433,7 +437,7 @@ def add_new_files(job, mets, sip_uuid, sip_dir):
                 if rel_path != old_mets_rel_path:
                     job.pyprint(rel_path, "not found in METS, must be new file")
                     f = models.File.objects.get(
-                        currentlocation=current_loc, sip_id=sip_uuid
+                        currentlocation=current_loc, sip_id=str(sip_uuid)
                     )
                     new_files.append(f)
                     if rel_path == "objects/metadata/metadata.csv":
@@ -455,20 +459,20 @@ def add_new_files(job, mets, sip_uuid, sip_dir):
 
     for f in new_files:
         # Create amdSecs
-        job.pyprint("Adding amdSec for", f.currentlocation, "(", f.uuid, ")")
+        job.pyprint("Adding amdSec for", f.currentlocation, "(", str(f.uuid), ")")
         amdsec, amdid = createmets2.getAMDSec(
             job,
-            fileUUID=f.uuid,
+            fileUUID=str(f.uuid),
             filePath=None,  # Only needed if use=original
             use=f.filegrpuse,
-            sip_uuid=sip_uuid,
+            sip_uuid=str(sip_uuid),
             transferUUID=None,  # Only needed if use=original
             itemdirectoryPath=None,  # Only needed if use=original
             typeOfTransfer=None,  # Only needed if use=original
             baseDirectoryPath=sip_dir,
             state=state,
         )
-        job.pyprint(f.uuid, "has amdSec with ID", amdid)
+        job.pyprint(str(f.uuid), "has amdSec with ID", amdid)
 
         # Create parent directories if needed
         dirs = os.path.dirname(
@@ -485,12 +489,12 @@ def add_new_files(job, mets, sip_uuid, sip_dir):
         derived_from = None
         if f.original_file_set.exists():
             original_f = f.original_file_set.get().source_file
-            derived_from = mets.get_file(file_uuid=original_f.uuid)
+            derived_from = mets.get_file(file_uuid=str(original_f.uuid))
         entry = metsrw.FSEntry(
             path=f.currentlocation.replace("%SIPDirectory%", "", 1),
             use=f.filegrpuse,
             type="Item",
-            file_uuid=f.uuid,
+            file_uuid=str(f.uuid),
             derived_from=derived_from,
         )
         metsrw_amdsec = metsrw.AMDSec.parse(amdsec)
@@ -514,7 +518,7 @@ def delete_files(mets, sip_uuid):
         sip_id=sip_uuid, event__event_type="deletion"
     ).values_list("uuid", flat=True)
     for file_uuid in deleted_files:
-        df = mets.get_file(file_uuid=file_uuid)
+        df = mets.get_file(file_uuid=str(file_uuid))
         df.use = "deleted"
         df.path = None
         df.label = None
@@ -554,7 +558,7 @@ def update_metadata_csv(job, mets, metadata_csv, sip_uuid, sip_dir, state):
         file_obj = None
         try:
             file_obj = models.File.objects.get(
-                sip_id=sip_uuid, originallocation__endswith="%" + f
+                sip_id=str(sip_uuid), originallocation__endswith="%" + f
             )
         except models.File.DoesNotExist:
             try:
@@ -564,7 +568,7 @@ def update_metadata_csv(job, mets, metadata_csv, sip_uuid, sip_dir, state):
             except models.File.DoesNotExist:
                 pass
         if file_obj is not None:
-            fsentry = mets.get_file(file_uuid=file_obj.uuid)
+            fsentry = mets.get_file(file_uuid=str(file_obj.uuid))
         else:
             fsentry = _get_directory_fsentry(mets, f)
         if fsentry is None:

src/MCPClient/lib/clientScripts/archivematicaCreateMETSReingest.py

@@ -269,7 +269,7 @@ def createRightsStatement(job, statement, fileUUID, state):
     ).text = "UUID"
     etree.SubElement(
         linkingObjectIdentifier, ns.premisBNS + "linkingObjectIdentifierValue"
-    ).text = fileUUID
+    ).text = str(fileUUID)
     return rightsStatement
 
 

src/MCPClient/lib/clientScripts/archivematicaCreateMETSRights.py

@@ -219,9 +219,12 @@ def version_attrs(self):
 
 
 def file_already_scanned(file_uuid):
-    return Event.objects.filter(
-        file_uuid_id=file_uuid, event_type="virus check"
-    ).exists()
+    return (
+        file_uuid != "None"
+        and Event.objects.filter(
+            file_uuid_id=file_uuid, event_type="virus check"
+        ).exists()
+    )
 
 
 def queue_event(file_uuid, date, scanner, passed, queue):
@@ -274,7 +277,6 @@ def get_scanner():
     """
     choice = str(mcpclient_settings.CLAMAV_CLIENT_BACKEND).lower()
     if choice not in SCANNERS_NAMES:
-
         logger.warning(
             "Unexpected antivirus scanner (CLAMAV_CLIENT_BACKEND):" ' "%s"; using %s.',
             choice,

src/MCPClient/lib/clientScripts/archivematica_clamscan.py

@@ -230,7 +230,9 @@ def _add_identifier(object_elem, identifier, bns=ns.premisBNS):
     idfr_type, idfr_val = identifier
     objectIdentifier = etree.SubElement(object_elem, bns + "objectIdentifier")
     etree.SubElement(objectIdentifier, bns + "objectIdentifierType").text = idfr_type
-    etree.SubElement(objectIdentifier, bns + "objectIdentifierValue").text = idfr_val
+    etree.SubElement(objectIdentifier, bns + "objectIdentifierValue").text = str(
+        idfr_val
+    )
     return object_elem
 
 
@@ -527,7 +529,7 @@ def create_premis_object(fileUUID):
     :param str fileUUID: UUID of the File to create an object for
     :return: premis:object Element, suitable for inserting into mets:xmlData
     """
-    f = File.objects.get(uuid=fileUUID)
+    f = File.objects.get(uuid=str(fileUUID))
     # PREMIS:OBJECT
     object_elem = etree.Element(ns.premisBNS + "object", nsmap={"premis": ns.premisNS})
     object_elem.set(ns.xsiBNS + "type", "premis:file")
@@ -657,7 +659,7 @@ def create_premis_object_derivations(fileUUID):
         ).text = "UUID"
         etree.SubElement(
             relatedObjectIdentifier, ns.premisBNS + "relatedObjectIdentifierValue"
-        ).text = derivation.derived_file_id
+        ).text = str(derivation.derived_file_id)
 
         relatedEventIdentifier = etree.SubElement(
             relationship, ns.premisBNS + "relatedEventIdentifier"
@@ -667,7 +669,7 @@ def create_premis_object_derivations(fileUUID):
         ).text = "UUID"
         etree.SubElement(
             relatedEventIdentifier, ns.premisBNS + "relatedEventIdentifierValue"
-        ).text = derivation.event_id
+        ).text = str(derivation.event_id)
 
         elements.append(relationship)
 
@@ -691,7 +693,7 @@ def create_premis_object_derivations(fileUUID):
         ).text = "UUID"
         etree.SubElement(
             relatedObjectIdentifier, ns.premisBNS + "relatedObjectIdentifierValue"
-        ).text = derivation.source_file_id
+        ).text = str(derivation.source_file_id)
 
         relatedEventIdentifier = etree.SubElement(
             relationship, ns.premisBNS + "relatedEventIdentifier"
@@ -701,7 +703,7 @@ def create_premis_object_derivations(fileUUID):
         ).text = "UUID"
         etree.SubElement(
             relatedEventIdentifier, ns.premisBNS + "relatedEventIdentifierValue"
-        ).text = derivation.event_id
+        ).text = str(derivation.event_id)
 
         elements.append(relationship)
 
@@ -714,7 +716,7 @@ def createDigiprovMD(fileUUID, state):
     """
     ret = []
 
-    events = Event.objects.filter(file_uuid_id=fileUUID)
+    events = Event.objects.filter(file_uuid_id=str(fileUUID))
     for event_record in events:
         state.globalDigiprovMDCounter += 1
         digiprovMD = etree.Element(
@@ -730,7 +732,7 @@ def createDigiprovMD(fileUUID, state):
         xmlData.append(createEvent(event_record))
 
     for agent in Agent.objects.extend_queryset_with_preservation_system(
-        Agent.objects.filter(event__file_uuid_id=fileUUID).distinct()
+        Agent.objects.filter(event__file_uuid_id=str(fileUUID)).distinct()
     ):
         state.globalDigiprovMDCounter += 1
         digiprovMD = etree.Element(
@@ -761,10 +763,9 @@ def createEvent(event_record):
     etree.SubElement(
         eventIdentifier, ns.premisBNS + "eventIdentifierType"
     ).text = "UUID"
-    etree.SubElement(
-        eventIdentifier, ns.premisBNS + "eventIdentifierValue"
-    ).text = event_record.event_id
-
+    etree.SubElement(eventIdentifier, ns.premisBNS + "eventIdentifierValue").text = str(
+        event_record.event_id
+    )
     etree.SubElement(event, ns.premisBNS + "eventType").text = event_record.event_type
     etree.SubElement(
         event, ns.premisBNS + "eventDateTime"
@@ -895,7 +896,7 @@ def getAMDSec(
             )
             AMD.append(digiprovMD)
 
-    for a in createDigiprovMD(fileUUID, state):
+    for a in createDigiprovMD(str(fileUUID), state):
         AMD.append(a)
 
     return ret
@@ -1136,7 +1137,7 @@ def createFileSec(
             GROUPID = ""
             if f.filegrpuuid:
                 # GROUPID was determined elsewhere
-                GROUPID = "Group-%s" % (f.filegrpuuid)
+                GROUPID = f"Group-{f.filegrpuuid}"
                 if use == "TRIM file metadata":
                     use = "metadata"
 
@@ -1147,7 +1148,7 @@ def createFileSec(
                 "maildirFile",
             ):
                 # These files are in a group defined by themselves
-                GROUPID = "Group-%s" % (f.uuid)
+                GROUPID = f"Group-{f.uuid}"
                 if use == "maildirFile":
                     use = "original"
                 # Check for CSV-based Dublincore dmdSec
@@ -1193,7 +1194,7 @@ def createFileSec(
                 }
                 original_file = File.objects.filter(**kwargs).first()
                 if original_file is not None:
-                    GROUPID = "Group-" + original_file.uuid
+                    GROUPID = f"Group-{original_file.uuid}"
 
             elif use in ("preservation", "text/ocr", "derivative"):
                 # Derived files should be in the original file's group
@@ -1205,7 +1206,7 @@ def createFileSec(
                         " where the derived file is {}".format(f.uuid)
                     )
                     raise
-                GROUPID = "Group-" + d.source_file_id
+                GROUPID = f"Group-{d.source_file_id}"
 
             elif use == "service":
                 # Service files are in the original file's group
@@ -1223,10 +1224,10 @@ def createFileSec(
                     "currentlocation__startswith": fileFileIDPath,
                 }
                 original_file = File.objects.get(**kwargs)
-                GROUPID = "Group-" + original_file.uuid
+                GROUPID = f"Group-{original_file.uuid}"
 
             elif use == "TRIM container metadata":
-                GROUPID = "Group-%s" % (f.uuid)
+                GROUPID = f"Group-{f.uuid}"
                 use = "metadata"
 
             # Special DSPACEMETS processing
@@ -1765,7 +1766,7 @@ def main(
 
         # Get the <dmdSec> for the entire AIP; it is associated to the root
         # <mets:div> in the physical structMap.
-        sip_mdl = SIP.objects.filter(uuid=sipUUID).first()
+        sip_mdl = SIP.objects.filter(uuid=str(sipUUID)).first()
         if sip_mdl:
             aipDmdSec = getDirDmdSec(sip_mdl, sip_dir_name)
             state.globalDmdSecCounter += 1