Merge pull request #405 from epam/release/1.13

1.13.1

Author: bohdan-onsha

CHANGELOG.md

@@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+# [1.13.1] - 2024-08-05
+- Speed up deletion of s3 bucket with lots of objects
+
 # [1.13.0] - 2024-07-10
 - Added possibility to configure `FunctionResponseTypes` for lambda functions
 - Updated maven plugin version to 1.12.0 with support of `FunctionResponseTypes`

setup.py

@@ -27,7 +27,7 @@
 
 setup(
     name='aws-syndicate',
-    version='1.13.0',
+    version='1.13.1',
     packages=find_packages(),
     include_package_data=True,
     install_requires=[

syndicate/connection/s3_connection.py

@@ -176,8 +176,11 @@ def create_bucket(self, bucket_name, acl=None, location=None):
     def remove_bucket(self, bucket_name):
         """ Remove bucket by name. To remove bucket it must be empty."""
         bucket = self.resource.Bucket(bucket_name)
-        for each in bucket.objects.all():
-            each.delete()
+        bucket_versioning = self.resource.BucketVersioning(bucket_name)
+        if bucket_versioning.status == 'Enabled':
+            bucket.object_versions.delete()
+        else:
+            bucket.objects.all().delete()
         bucket.delete()
 
     def delete_bucket(self, bucket_name):

syndicate/core/resources/s3_resource.py

@@ -183,25 +183,8 @@ def remove_buckets(self, args):
     def _remove_bucket(self, arn, config):
         bucket_name = config['resource_name']
         try:
-            errors = []
-            keys = self.s3_conn.list_object_versions(bucket_name)
-            if keys:
-                for s3_keys in chunks(keys, 1000):
-                    errors.extend(self._delete_objects(bucket_name, s3_keys))
-
-            markers = self.s3_conn.list_object_markers(bucket_name)
-            if markers:
-                for s3_markers in chunks(markers, 1000):
-                    errors.extend(
-                        self._delete_objects(bucket_name, s3_markers))
-
-            if errors:
-                raise AssertionError('Error occurred while deleting S3 objects'
-                                     ' from {0} bucket. Not deleted keys: '
-                                     '{1}'.format(bucket_name, str(errors)))
-            else:
-                self.s3_conn.delete_bucket(bucket_name)
-                _LOG.info('S3 bucket {0} was removed.'.format(bucket_name))
+            self.s3_conn.remove_bucket(bucket_name=bucket_name)
+            _LOG.info('S3 bucket {0} was removed.'.format(bucket_name))
         except ClientError as e:
             if e.response['Error']['Code'] == 'NoSuchBucket':
                 _LOG.warn('S3 bucket {0} is not found'.format(bucket_name))