[COST-5216] Missed delete filtering optimization

koku/masu/test/util/aws/test_common.py

@@ -567,23 +567,32 @@ def test_clear_s3_files(self):
             self.account_id, Provider.PROVIDER_AWS, self.aws_provider_uuid, start_date, Config.CSV_DATA_TYPE
         )
         expected_key = "not_matching_key"
-        mock_object = Mock(metadata={metadata_key: "this will be deleted"}, key=expected_key)
         not_matching_summary = Mock()
-        not_matching_summary.Object.return_value = mock_object
+        not_matching_summary.key = expected_key
+
         not_expected_key = "matching_key"
-        mock_object = Mock(metadata={metadata_key: metadata_value}, key=not_expected_key)
         matching_summary = Mock()
-        matching_summary.Object.return_value = mock_object
+        matching_summary.key = not_expected_key
+
+        def mock_head_object(Bucket, Key):
+            if Key == expected_key:
+                return {"Metadata": {metadata_key: "this will be deleted"}}
+            elif Key == not_expected_key:
+                return {"Metadata": {metadata_key: metadata_value}}
+            raise ClientError({}, "Error")
+
         with patch("masu.util.aws.common.get_s3_resource") as mock_s3:
             mock_s3.return_value.Bucket.return_value.objects.filter.return_value = [
                 not_matching_summary,
                 matching_summary,
             ]
-            with patch("masu.util.aws.common.delete_s3_objects") as mock_delete:
-                utils.clear_s3_files(
-                    s3_csv_path, self.aws_provider_uuid, start_date, "manifestid", 1, context, "requiest_id"
-                )
-                mock_delete.assert_called
+            with patch("boto3.client") as mock_s3_client:
+                mock_s3_client.return_value.head_object.side_effect = mock_head_object
+                with patch("masu.util.aws.common.delete_s3_objects") as mock_delete:
+                    utils.clear_s3_files(
+                        s3_csv_path, self.aws_provider_uuid, start_date, "manifestid", 1, context, "request_id"
+                    )
+                    mock_delete.assert_called()
 
     def test_clear_s3_files_gcp(self):
         """Test clearing s3 GCP ingress files."""
@@ -595,21 +604,30 @@ def test_clear_s3_files_gcp(self):
         start_date = self.dh.this_month_start.replace(tzinfo=None)
         s3_csv_path = get_path_prefix(self.account_id, "GCP", prov_uuid, start_date, Config.CSV_DATA_TYPE)
         expected_key = "not_matching_key"
-        mock_object = Mock(metadata={metadata_key: "this will be deleted"}, key=expected_key)
         not_matching_summary = Mock()
-        not_matching_summary.Object.return_value = mock_object
+        not_matching_summary.key = expected_key
+
         not_expected_key = "matching_key"
-        mock_object = Mock(metadata={metadata_key: metadata_value}, key=not_expected_key)
         matching_summary = Mock()
-        matching_summary.Object.return_value = mock_object
+        matching_summary.key = not_expected_key
+
+        def mock_head_object(Bucket, Key):
+            if Key == expected_key:
+                return {"Metadata": {metadata_key: "this will be deleted"}}
+            elif Key == not_expected_key:
+                return {"Metadata": {metadata_key: metadata_value}}
+            raise ClientError({}, "Error")
+
         with patch("masu.util.aws.common.get_s3_resource") as mock_s3:
             mock_s3.return_value.Bucket.return_value.objects.filter.return_value = [
                 not_matching_summary,
                 matching_summary,
             ]
-            with patch("masu.util.aws.common.delete_s3_objects") as mock_delete:
-                utils.clear_s3_files(s3_csv_path, prov_uuid, start_date, "manifestid", 1, context, "requiest_id")
-                mock_delete.assert_called
+            with patch("boto3.client") as mock_s3_client:
+                mock_s3_client.return_value.head_object.side_effect = mock_head_object
+                with patch("masu.util.aws.common.delete_s3_objects") as mock_delete:
+                    utils.clear_s3_files(s3_csv_path, prov_uuid, start_date, "manifestid", 1, context, "request_id")
+                    mock_delete.assert_called()
 
     def test_remove_s3_objects_matching_metadata(self):
         """Test remove_s3_objects_matching_metadata."""

koku/masu/util/aws/common.py

@@ -865,22 +865,12 @@ def clear_s3_files(
         s3_prefixes.append(parquet_ocp_on_cloud_path_s3 + path)
     to_delete = []
     for prefix in s3_prefixes:
-        for obj_summary in _get_s3_objects(prefix):
-            try:
-                existing_object = obj_summary.Object()
-                metadata_value = existing_object.metadata.get(metadata_key)
-                if str(metadata_value) != str(manifest_id):
-                    to_delete.append(existing_object.key)
-            except (ClientError) as err:
-                LOG.warning(
-                    log_json(
-                        request_id,
-                        msg="unable to get matching object, likely deleted by another worker",
-                        context=context,
-                        bucket=settings.S3_BUCKET_NAME,
-                    ),
-                    exc_info=err,
-                )
+        to_delete.extend(
+            get_s3_objects_not_matching_metadata(
+                request_id, prefix, metadata_key=metadata_key, metadata_value_check=str(manifest_id), context=context
+            )
+        )
+
     delete_s3_objects(request_id, to_delete, context)
     manifest_accessor = ReportManifestDBAccessor()
     manifest = manifest_accessor.get_manifest_by_id(manifest_id)