Fix tape full verify (#2327)

Author: henrikek

ESSArch_Core/storage/backends/tape.py

@@ -278,14 +278,14 @@ def post_mark_as_full(cls, storage_medium):
         """Called after a medium has been successfully marked as full"""
 
         logger = logging.getLogger('essarch.storage.backends.tape')
-        drive = str(storage_medium.drive)
+        drive = storage_medium.tape_drive
 
         logger.debug('Release lock for drive {} and storage medium {} ({})'.format(
-            drive.pk, storage_medium.medium_id, str(storage_medium.pk)))
+            drive.pk, storage_medium.medium_id, storage_medium.pk))
         cache.delete_pattern(drive.get_lock_key())
 
         logger.debug('Queueing unmount of storage medium {} ({})'.format(
-            storage_medium.medium_id, str(storage_medium.pk)))
+            storage_medium.medium_id, storage_medium.pk))
         rq, _ = RobotQueue.objects.get_or_create(
             user=User.objects.get(username='system'),
             storage_medium=storage_medium,
@@ -295,5 +295,5 @@ def post_mark_as_full(cls, storage_medium):
 
         while RobotQueue.objects.filter(id=rq.id).exists():
             logger.debug('Wait for the unmount request to complete for storage medium {} ({})'.format(
-                storage_medium.medium_id, str(storage_medium.pk)))
+                storage_medium.medium_id, storage_medium.pk))
             time.sleep(1)

ESSArch_Core/storage/models.py

@@ -2,6 +2,7 @@
 import logging
 import os
 import pickle
+import shutil
 import tarfile
 import uuid
 from datetime import timedelta
@@ -382,7 +383,8 @@ def _create_storage_medium(self):
 
         if storage_type == TAPE:
             slot = TapeSlot.objects.filter(status=20, storage_medium__isnull=True,
-                                           medium_id__startswith=self.target).exclude(medium_id__exact='').first()
+                                           medium_id__startswith=self.target
+                                           ).exclude(medium_id__exact='').natural_sort().first()
             if slot is None:
                 raise ValueError("No tape available for allocation")
             medium = StorageMedium.objects.create(medium_id=slot.medium_id, storage_target=self, status=20,
@@ -669,12 +671,13 @@ def deactivate(self) -> None:
     def mark_as_full(self):
         logger = logging.getLogger('essarch.storage.models')
         logger.debug('Marking storage medium as full: "{}"'.format(str(self.pk)))
+        logger.info('Storage medium is full, start to verify: "{}"'.format(self.medium_id))
         objs = self.storage.annotate(
             content_location_value_int=Cast('content_location_value', models.IntegerField())
         ).order_by('content_location_value_int')
 
         if objs.count() > 3:
-            objs = [objs.first(), objs[objs.count() / 2], objs.last()]
+            objs = [objs.first(), objs[int(objs.count() / 2)], objs.last()]
 
         try:
             for obj in objs:
@@ -684,11 +687,15 @@ def mark_as_full(self):
             logger.exception('Failed to verify storage medium: "{}"'.format(str(self.pk)))
             raise
         else:
+            verifydir = Path.objects.get(entity='verify').value
+            tmppath = os.path.join(verifydir, self.storage_target.target)
+            shutil.rmtree(tmppath)
             self.status = 30
             storage_backend = self.storage_target.get_storage_backend()
             storage_backend.post_mark_as_full(self)
         finally:
             self.save(update_fields=['status'])
+            logger.info('Storage medium is full, content verified success: "{}"'.format(self.medium_id))
 
     class Meta:
         permissions = (
@@ -985,7 +992,7 @@ def verify(self):
             drive.last_change = timezone.now()
             drive.save(update_fields=['last_change'])
 
-            filename = os.path.join(tmppath, self.ip.object_identifier_value + '.tar'),
+            filename = os.path.join(tmppath, self.ip.object_identifier_value + '.tar')
             algorithm = self.ip.get_message_digest_algorithm_display()
             options = {'expected': self.ip.message_digest, 'algorithm': algorithm}
 
@@ -1084,6 +1091,11 @@ def __str__(self):
         return self.device
 
 
+class TapeSlotQueryset(models.QuerySet):
+    def natural_sort(self):
+        return natural_sort(self, 'medium_id')
+
+
 class TapeSlot(models.Model):
     STATUS_CHOICES = (
         (0, 'Inactive'),
@@ -1103,6 +1115,8 @@ class TapeSlot(models.Model):
     robot = models.ForeignKey('Robot', models.PROTECT, related_name='tape_slots')
     status = models.IntegerField(choices=STATUS_CHOICES, default=20)
 
+    objects = TapeSlotQueryset.as_manager()
+
     @classmethod
     @transaction.atomic
     @retry(retry=retry_if_exception_type(RequestException), reraise=True, stop=stop_after_attempt(5),

requirements/base.txt

@@ -1,5 +1,5 @@
 asgiref==3.8.1
-boto3==1.34.160
+boto3==1.35.11
 celery[tblib]==5.4.0
 cffi==1.17.0
 channels==4.1.0
@@ -9,7 +9,7 @@ click==8.1.3
 cryptography==42.0.8
 daphne==4.1.2
 dj-rest-auth[with_social]==6.0.0
-django==5.0.8
+django==5.0.9
 django-allauth==0.61.1
 django-cors-headers==4.4.0
 django-countries-plus==2.2.0