diff --git a/koku/masu/test/util/aws/test_common.py b/koku/masu/test/util/aws/test_common.py index 20849322ee..8ab8dd7e14 100644 --- a/koku/masu/test/util/aws/test_common.py +++ b/koku/masu/test/util/aws/test_common.py @@ -567,23 +567,32 @@ def test_clear_s3_files(self): self.account_id, Provider.PROVIDER_AWS, self.aws_provider_uuid, start_date, Config.CSV_DATA_TYPE ) expected_key = "not_matching_key" - mock_object = Mock(metadata={metadata_key: "this will be deleted"}, key=expected_key) not_matching_summary = Mock() - not_matching_summary.Object.return_value = mock_object + not_matching_summary.key = expected_key + not_expected_key = "matching_key" - mock_object = Mock(metadata={metadata_key: metadata_value}, key=not_expected_key) matching_summary = Mock() - matching_summary.Object.return_value = mock_object + matching_summary.key = not_expected_key + + def mock_head_object(Bucket, Key): + if Key == expected_key: + return {"Metadata": {metadata_key: "this will be deleted"}} + elif Key == not_expected_key: + return {"Metadata": {metadata_key: metadata_value}} + raise ClientError({}, "Error") + with patch("masu.util.aws.common.get_s3_resource") as mock_s3: mock_s3.return_value.Bucket.return_value.objects.filter.return_value = [ not_matching_summary, matching_summary, ] - with patch("masu.util.aws.common.delete_s3_objects") as mock_delete: - utils.clear_s3_files( - s3_csv_path, self.aws_provider_uuid, start_date, "manifestid", 1, context, "requiest_id" - ) - mock_delete.assert_called + with patch("boto3.client") as mock_s3_client: + mock_s3_client.return_value.head_object.side_effect = mock_head_object + with patch("masu.util.aws.common.delete_s3_objects") as mock_delete: + utils.clear_s3_files( + s3_csv_path, self.aws_provider_uuid, start_date, "manifestid", 1, context, "request_id" + ) + mock_delete.assert_called() def test_clear_s3_files_gcp(self): """Test clearing s3 GCP ingress files.""" @@ -595,21 +604,30 @@ def test_clear_s3_files_gcp(self): start_date = self.dh.this_month_start.replace(tzinfo=None) s3_csv_path = get_path_prefix(self.account_id, "GCP", prov_uuid, start_date, Config.CSV_DATA_TYPE) expected_key = "not_matching_key" - mock_object = Mock(metadata={metadata_key: "this will be deleted"}, key=expected_key) not_matching_summary = Mock() - not_matching_summary.Object.return_value = mock_object + not_matching_summary.key = expected_key + not_expected_key = "matching_key" - mock_object = Mock(metadata={metadata_key: metadata_value}, key=not_expected_key) matching_summary = Mock() - matching_summary.Object.return_value = mock_object + matching_summary.key = not_expected_key + + def mock_head_object(Bucket, Key): + if Key == expected_key: + return {"Metadata": {metadata_key: "this will be deleted"}} + elif Key == not_expected_key: + return {"Metadata": {metadata_key: metadata_value}} + raise ClientError({}, "Error") + with patch("masu.util.aws.common.get_s3_resource") as mock_s3: mock_s3.return_value.Bucket.return_value.objects.filter.return_value = [ not_matching_summary, matching_summary, ] - with patch("masu.util.aws.common.delete_s3_objects") as mock_delete: - utils.clear_s3_files(s3_csv_path, prov_uuid, start_date, "manifestid", 1, context, "requiest_id") - mock_delete.assert_called + with patch("boto3.client") as mock_s3_client: + mock_s3_client.return_value.head_object.side_effect = mock_head_object + with patch("masu.util.aws.common.delete_s3_objects") as mock_delete: + utils.clear_s3_files(s3_csv_path, prov_uuid, start_date, "manifestid", 1, context, "request_id") + mock_delete.assert_called() def test_remove_s3_objects_matching_metadata(self): """Test remove_s3_objects_matching_metadata.""" diff --git a/koku/masu/util/aws/common.py b/koku/masu/util/aws/common.py index aaf9791e08..1232e2fe37 100644 --- a/koku/masu/util/aws/common.py +++ b/koku/masu/util/aws/common.py @@ -865,22 +865,12 @@ def clear_s3_files( s3_prefixes.append(parquet_ocp_on_cloud_path_s3 + path) to_delete = [] for prefix in s3_prefixes: - for obj_summary in _get_s3_objects(prefix): - try: - existing_object = obj_summary.Object() - metadata_value = existing_object.metadata.get(metadata_key) - if str(metadata_value) != str(manifest_id): - to_delete.append(existing_object.key) - except (ClientError) as err: - LOG.warning( - log_json( - request_id, - msg="unable to get matching object, likely deleted by another worker", - context=context, - bucket=settings.S3_BUCKET_NAME, - ), - exc_info=err, - ) + to_delete.extend( + get_s3_objects_not_matching_metadata( + request_id, prefix, metadata_key=metadata_key, metadata_value_check=str(manifest_id), context=context + ) + ) + delete_s3_objects(request_id, to_delete, context) manifest_accessor = ReportManifestDBAccessor() manifest = manifest_accessor.get_manifest_by_id(manifest_id)