Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

S3: Sending notification to EventBridge ObjectCreated:Post #7368

Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion moto/s3/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -2048,6 +2048,7 @@ def put_object(
lock_legal_status: Optional[str] = None,
lock_until: Optional[str] = None,
checksum_value: Optional[str] = None,
request_method: Optional[str] = "PUT",
) -> FakeKey:
if storage is not None and storage not in STORAGE_CLASS:
raise InvalidStorageClass(storage=storage)
Expand Down Expand Up @@ -2096,9 +2097,18 @@ def put_object(
keys = [new_key]
bucket.keys.setlist(key_name, keys)

# Send event notification
if request_method == "POST":
notify_event_name = (
notifications.S3NotificationEvent.OBJECT_CREATED_POST_EVENT
)
else: # PUT request
notify_event_name = (
notifications.S3NotificationEvent.OBJECT_CREATED_PUT_EVENT
)
notifications.send_event(
self.account_id,
notifications.S3NotificationEvent.OBJECT_CREATED_PUT_EVENT,
notify_event_name,
bucket,
new_key,
)
Expand Down
1 change: 1 addition & 0 deletions moto/s3/responses.py
Original file line number Diff line number Diff line change
Expand Up @@ -2259,6 +2259,7 @@ def _key_response_post(
multipart=multipart,
encryption=multipart.sse_encryption,
kms_key_id=multipart.kms_key_id,
request_method=request.method,
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This would send a POST event immediately after the CompleteMultipartUpload-action is finished. If I understand it correctly, AWS will only send a single event in this case. So we should avoid sending a POST event completely here.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh that's true, thanks!

)
key.set_metadata(multipart.metadata)

Expand Down
95 changes: 93 additions & 2 deletions tests/test_s3/test_s3_eventbridge_integration.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,23 @@
import json
from io import BytesIO
from unittest import SkipTest
from uuid import uuid4

import boto3

from moto import mock_aws
from moto import mock_aws, settings
from moto.core import DEFAULT_ACCOUNT_ID as ACCOUNT_ID
from tests.test_s3.test_s3_multipart import reduced_min_part_size

REGION_NAME = "us-east-1"
REDUCED_PART_SIZE = 256


@mock_aws
def test_pub_object_notification():
def test_put_object_notification_ObjectCreated_PUT():
if not settings.TEST_DECORATOR_MODE:
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's the reason behind skipping the tests in ServerMode?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry, this part was added by mistake.

raise SkipTest(("Doesn't quite work right with the Proxy or Server"))

s3_res = boto3.resource("s3", region_name=REGION_NAME)
s3_client = boto3.client("s3", region_name=REGION_NAME)
events_client = boto3.client("events", region_name=REGION_NAME)
Expand Down Expand Up @@ -57,3 +64,87 @@ def test_pub_object_notification():
assert event_message["region"] == REGION_NAME
assert event_message["detail"]["bucket"]["name"] == bucket_name
assert event_message["detail"]["reason"] == "ObjectCreated"


@mock_aws
@reduced_min_part_size
def test_put_object_notification_ObjectCreated_POST():
if not settings.TEST_DECORATOR_MODE:
raise SkipTest(("Doesn't quite work right with the Proxy or Server"))

s3_res = boto3.resource("s3", region_name=REGION_NAME)
s3_client = boto3.client("s3", region_name=REGION_NAME)
events_client = boto3.client("events", region_name=REGION_NAME)
logs_client = boto3.client("logs", region_name=REGION_NAME)

rule_name = "test-rule"
events_client.put_rule(
Name=rule_name, EventPattern=json.dumps({"account": [ACCOUNT_ID]})
)
log_group_name = "/test-group"
logs_client.create_log_group(logGroupName=log_group_name)
events_client.put_targets(
Rule=rule_name,
Targets=[
{
"Id": "test",
"Arn": f"arn:aws:logs:{REGION_NAME}:{ACCOUNT_ID}:log-group:{log_group_name}",
}
],
)

# Create S3 bucket
bucket_name = str(uuid4())
s3_res.create_bucket(Bucket=bucket_name)

# Put Notification
s3_client.put_bucket_notification_configuration(
Bucket=bucket_name,
NotificationConfiguration={"EventBridgeConfiguration": {}},
)

# Create multipart upload request (an object is uploaded via POST request).
key_name = "the-key"
part1 = b"0" * REDUCED_PART_SIZE
part2 = b"1"
multipart = s3_client.create_multipart_upload(Bucket=bucket_name, Key=key_name)
up1 = s3_client.upload_part(
Body=BytesIO(part1),
PartNumber=4,
Bucket=bucket_name,
Key=key_name,
UploadId=multipart["UploadId"],
)
up2 = s3_client.upload_part(
Body=BytesIO(part2),
PartNumber=2,
Bucket=bucket_name,
Key=key_name,
UploadId=multipart["UploadId"],
)

s3_client.complete_multipart_upload(
Bucket=bucket_name,
Key=key_name,
MultipartUpload={
"Parts": [
{"ETag": up1["ETag"], "PartNumber": 4},
{"ETag": up2["ETag"], "PartNumber": 2},
]
},
UploadId=multipart["UploadId"],
)

events = sorted(
logs_client.filter_log_events(logGroupName=log_group_name)["events"],
key=lambda item: item["eventId"],
)
assert len(events) == 1
event_message = json.loads(events[0]["message"])
assert event_message["detail-type"] == "Object Created"
assert event_message["source"] == "aws.s3"
assert event_message["account"] == ACCOUNT_ID
assert event_message["region"] == REGION_NAME
assert event_message["detail"]["bucket"]["name"] == bucket_name
# NOTE: We cannot know if an object is created via PUT or POST request from EventBridge message.
assert event_message["detail"]["reason"] == "ObjectCreated"