From 380b7aaaf4b6c18648eb4b35287eed43336c3ccd Mon Sep 17 00:00:00 2001 From: Satoshi Kita Date: Tue, 13 Dec 2022 23:24:45 +0900 Subject: [PATCH 1/4] support BatchGetItem and BatchWriteItem --- src/Kitar/Dynamodb/Model/Model.php | 4 ++ src/Kitar/Dynamodb/Query/Builder.php | 58 ++++++++++++++++++++++++++ src/Kitar/Dynamodb/Query/Grammar.php | 48 +++++++++++++++++++++ src/Kitar/Dynamodb/Query/Processor.php | 25 +++++++++++ 4 files changed, 135 insertions(+) diff --git a/src/Kitar/Dynamodb/Model/Model.php b/src/Kitar/Dynamodb/Model/Model.php index 0c6cdff..70afc78 100644 --- a/src/Kitar/Dynamodb/Model/Model.php +++ b/src/Kitar/Dynamodb/Model/Model.php @@ -354,6 +354,10 @@ public function __call($method, $parameters) "putItem", "deleteItem", "updateItem", + "batchGetItem", + "batchPutItem", + "batchDeleteItem", + "batchWriteItem", "scan", "filter", "filterIn", diff --git a/src/Kitar/Dynamodb/Query/Builder.php b/src/Kitar/Dynamodb/Query/Builder.php index a99c18d..a1b51ed 100644 --- a/src/Kitar/Dynamodb/Query/Builder.php +++ b/src/Kitar/Dynamodb/Query/Builder.php @@ -41,6 +41,20 @@ class Builder extends BaseBuilder 'remove' => [] ]; + /** + * Keys array for BatchGetItem + * https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchGetItem.html + * @var array + */ + public $batch_get_keys = []; + + /** + * RequestItems array for BatchWriteItem + * https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchWriteItem.html + * @var array + */ + public $batch_write_request_items = []; + /** * ScanIndexForward option. */ @@ -312,6 +326,48 @@ public function updateItem($item) return $this->process('updateItem', 'processSingleItem'); } + public function batchGetItem($keys) + { + $this->batch_get_keys = $keys; + + return $this->process('batchGetItem', 'processBatchGetItems'); + } + + public function batchPutItem($items) + { + $this->batch_write_request_items = collect($items)->map(function ($item) { + return [ + 'PutRequest' => [ + 'Item' => $item, + ], + ]; + })->toArray(); + + return $this->batchWriteItem(); + } + + public function batchDeleteItem($keys) + { + $this->batch_write_request_items = collect($keys)->map(function ($key) { + return [ + 'DeleteRequest' => [ + 'Key' => $key, + ], + ]; + })->toArray(); + + return $this->batchWriteItem(); + } + + public function batchWriteItem($request_items = []) + { + if (! empty($request_items)) { + $this->batch_write_request_items = $request_items; + } + + return $this->process('batchWriteItem', null); + } + /** * @inheritdoc */ @@ -539,6 +595,8 @@ protected function process($query_method, $processor_method) $this->grammar->compileKey($this->key), $this->grammar->compileItem($this->item), $this->grammar->compileUpdates($this->updates), + $this->grammar->compileBatchGetRequestItems($this->from, $this->batch_get_keys), + $this->grammar->compileBatchWriteRequestItems($this->from, $this->batch_write_request_items), $this->grammar->compileDynamodbLimit($this->limit), $this->grammar->compileScanIndexForward($this->scan_index_forward), $this->grammar->compileExclusiveStartKey($this->exclusive_start_key), diff --git a/src/Kitar/Dynamodb/Query/Grammar.php b/src/Kitar/Dynamodb/Query/Grammar.php index 4e5910a..16413f3 100644 --- a/src/Kitar/Dynamodb/Query/Grammar.php +++ b/src/Kitar/Dynamodb/Query/Grammar.php @@ -135,6 +135,54 @@ public function compileUpdates($updates) ]; } + public function compileBatchGetRequestItems($table_name, $keys) + { + if (empty($keys)) { + return []; + } + + $marshaler = $this->marshaler; + $marshaled_items = collect($keys)->map(function ($key) use ($marshaler) { + return $marshaler->marshalItem($key); + })->toArray(); + + $table_name = $this->tablePrefix . $table_name; + + return [ + 'RequestItems' => [ + $table_name => [ + 'Keys' => $marshaled_items, + ], + ] + ]; + } + + public function compileBatchWriteRequestItems($table_name, $request_items) + { + if (empty($request_items)) { + return []; + } + + $marshaler = $this->marshaler; + $marshaled_items = collect($request_items)->map(function ($request_item) use ($marshaler) { + return collect($request_item)->map(function ($request_body) use ($marshaler) { + $marshaled = []; + foreach ($request_body as $key => $body) { + $marshaled[$key] = $marshaler->marshalItem($body); + } + return $marshaled; + }); + })->toArray(); + + $table_name = $this->tablePrefix . $table_name; + + return [ + 'RequestItems' => [ + $table_name => $marshaled_items, + ], + ]; + } + /** * Compile the Limit attribute. * diff --git a/src/Kitar/Dynamodb/Query/Processor.php b/src/Kitar/Dynamodb/Query/Processor.php index 1a96af9..89bf21e 100644 --- a/src/Kitar/Dynamodb/Query/Processor.php +++ b/src/Kitar/Dynamodb/Query/Processor.php @@ -78,4 +78,29 @@ public function processMultipleItems(Result $awsResponse, $modelClass = null) return $item; }); } + + public function processBatchGetItems(Result $awsResponse, $modelClass = null) + { + $response = $this->unmarshal($awsResponse); + + if (empty($modelClass)) { + return $response; + } + + $items = collect(); + + foreach ($response['Responses'] as $_ => $table_items) { + foreach ($table_items as $item) { + $item = (new $modelClass)->newFromBuilder($item); + $items->push($item); + } + } + + unset($response['Responses']); + + return $items->map(function ($item) use ($response) { + $item->setMeta($response); + return $item; + }); + } } From b835fb0733de1648625ecd502531144f6b2a836a Mon Sep 17 00:00:00 2001 From: Satoshi Kita Date: Wed, 14 Dec 2022 15:48:21 +0900 Subject: [PATCH 2/4] unmarshal batch get results --- src/Kitar/Dynamodb/Query/Processor.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Kitar/Dynamodb/Query/Processor.php b/src/Kitar/Dynamodb/Query/Processor.php index 89bf21e..918c6dd 100644 --- a/src/Kitar/Dynamodb/Query/Processor.php +++ b/src/Kitar/Dynamodb/Query/Processor.php @@ -33,6 +33,14 @@ protected function unmarshal(Result $res) $responseArray['Attributes'] = $this->marshaler->unmarshalItem($responseArray['Attributes']); } + if (! empty($responseArray['Responses'])) { + foreach ($responseArray['Responses'] as &$items) { + foreach ($items as &$item) { + $item = $this->marshaler->unmarshalItem($item); + } + } + } + return $responseArray; } From 7b5a7afb8fc2044f9dcdf2a1813c9e1b40d24134 Mon Sep 17 00:00:00 2001 From: Satoshi Kita Date: Wed, 14 Dec 2022 15:48:40 +0900 Subject: [PATCH 3/4] add tests for batch operations --- tests/Query/BuilderTest.php | 215 ++++++++++++++++++++++++++++++++++ tests/Query/ProcessorTest.php | 50 +++++++- 2 files changed, 264 insertions(+), 1 deletion(-) diff --git a/tests/Query/BuilderTest.php b/tests/Query/BuilderTest.php index b67ed13..9dd786c 100644 --- a/tests/Query/BuilderTest.php +++ b/tests/Query/BuilderTest.php @@ -721,6 +721,221 @@ public function it_can_process_update_item() $this->assertEquals($processor, $query['processor']); } + /** @test */ + public function it_can_process_batch_get_item() + { + $method = 'batchGetItem'; + $params = [ + 'TableName' => 'Thread', + 'RequestItems' => [ + 'Thread' => [ + 'Keys' => [ + [ + 'ForumName' => [ + 'S' => 'Amazon DynamoDB' + ], + 'Subject' => [ + 'S' => 'DynamoDB Thread 1' + ] + ], + [ + 'ForumName' => [ + 'S' => 'Amazon DynamoDB' + ], + 'Subject' => [ + 'S' => 'DynamoDB Thread 2' + ] + ] + ] + ] + ] + ]; + $processor = 'processBatchGetItems'; + + $query = $this->newQuery('Thread') + ->batchGetItem([ + [ + 'ForumName' => 'Amazon DynamoDB', + 'Subject' => 'DynamoDB Thread 1' + ], + [ + 'ForumName' => 'Amazon DynamoDB', + 'Subject' => 'DynamoDB Thread 2' + ] + ]); + + $this->assertEquals($method, $query['method']); + $this->assertEquals($params, $query['params']); + $this->assertEquals($processor, $query['processor']); + } + + /** @test */ + public function it_can_process_batch_put_item() + { + $method = 'batchWriteItem'; + $params = [ + 'TableName' => 'Thread', + 'RequestItems' => [ + 'Thread' => [ + [ + 'PutRequest' => [ + 'Item' => [ + 'ForumName' => [ + 'S' => 'Amazon DynamoDB' + ], + 'Subject' => [ + 'S' => 'DynamoDB Thread 3' + ] + ] + ] + ], + [ + 'PutRequest' => [ + 'Item' => [ + 'ForumName' => [ + 'S' => 'Amazon DynamoDB' + ], + 'Subject' => [ + 'S' => 'DynamoDB Thread 4' + ] + ] + ] + ] + ] + ] + ]; + + $query = $this->newQuery('Thread') + ->batchPutItem([ + [ + 'ForumName' => 'Amazon DynamoDB', + 'Subject' => 'DynamoDB Thread 3' + ], + [ + 'ForumName' => 'Amazon DynamoDB', + 'Subject' => 'DynamoDB Thread 4' + ] + ]); + + $this->assertEquals($method, $query['method']); + $this->assertEquals($params, $query['params']); + $this->assertNull($query['processor']); + } + + /** @test */ + public function it_can_process_batch_delete_item() + { + $method = 'batchWriteItem'; + $params = [ + 'TableName' => 'Thread', + 'RequestItems' => [ + 'Thread' => [ + [ + 'DeleteRequest' => [ + 'Key' => [ + 'ForumName' => [ + 'S' => 'Amazon DynamoDB' + ], + 'Subject' => [ + 'S' => 'DynamoDB Thread 1' + ] + ] + ] + ], + [ + 'DeleteRequest' => [ + 'Key' => [ + 'ForumName' => [ + 'S' => 'Amazon DynamoDB' + ], + 'Subject' => [ + 'S' => 'DynamoDB Thread 2' + ] + ] + ] + ] + ] + ] + ]; + + $query = $this->newQuery('Thread') + ->batchDeleteItem([ + [ + 'ForumName' => 'Amazon DynamoDB', + 'Subject' => 'DynamoDB Thread 1' + ], + [ + 'ForumName' => 'Amazon DynamoDB', + 'Subject' => 'DynamoDB Thread 2' + ] + ]); + + $this->assertEquals($method, $query['method']); + $this->assertEquals($params, $query['params']); + $this->assertNull($query['processor']); + } + + /** @test */ + public function it_can_process_batch_write_item() + { + $method = 'batchWriteItem'; + $params = [ + 'TableName' => 'Thread', + 'RequestItems' => [ + 'Thread' => [ + [ + 'PutRequest' => [ + 'Item' => [ + 'ForumName' => [ + 'S' => 'Amazon DynamoDB' + ], + 'Subject' => [ + 'S' => 'DynamoDB Thread 3' + ] + ] + ] + ], + [ + 'DeleteRequest' => [ + 'Key' => [ + 'ForumName' => [ + 'S' => 'Amazon DynamoDB' + ], + 'Subject' => [ + 'S' => 'DynamoDB Thread 1' + ] + ] + ] + ] + ] + ] + ]; + + $query = $this->newQuery('Thread') + ->batchWriteItem([ + [ + 'PutRequest' => [ + 'Item' => [ + 'ForumName' => 'Amazon DynamoDB', + 'Subject' => 'DynamoDB Thread 3' + ] + ] + ], + [ + 'DeleteRequest' => [ + 'Key' => [ + 'ForumName' => 'Amazon DynamoDB', + 'Subject' => 'DynamoDB Thread 1' + ] + ] + ] + ]); + + $this->assertEquals($method, $query['method']); + $this->assertEquals($params, $query['params']); + $this->assertNull($query['processor']); + } + /** @test */ public function it_can_increment_value_of_attribute() { diff --git a/tests/Query/ProcessorTest.php b/tests/Query/ProcessorTest.php index 67e5f5b..058904b 100644 --- a/tests/Query/ProcessorTest.php +++ b/tests/Query/ProcessorTest.php @@ -23,7 +23,11 @@ class ProcessorTest extends TestCase 'multiple_items_result' => '{"Items":[{"Category":{"S":"Amazon Web Services"},"Name":{"S":"Amazon S3"}},{"Threads":{"N":"2"},"Category":{"S":"Amazon Web Services"},"Messages":{"N":"4"},"Views":{"N":"1000"},"Name":{"S":"Amazon DynamoDB"}}],"Count":2,"ScannedCount":2,"@metadata":{"statusCode":200,"effectiveUri":"https:\/\/dynamodb.ap-northeast-1.amazonaws.com","transferStats":{"http":[[]]}}}', 'multiple_items_processed' => '{"Items":[{"Category":"Amazon Web Services","Name":"Amazon S3"},{"Threads":2,"Category":"Amazon Web Services","Messages":4,"Views":1000,"Name":"Amazon DynamoDB"}],"Count":2,"ScannedCount":2,"@metadata":{"statusCode":200,"effectiveUri":"https:\/\/dynamodb.ap-northeast-1.amazonaws.com","transferStats":{"http":[[]]}}}', 'multiple_items_empty_result' => '{"Items":[],"Count":0,"ScannedCount":2,"@metadata":{"statusCode":200,"effectiveUri":"https:\/\/dynamodb.ap-northeast-1.amazonaws.com","transferStats":{"http":[[]]}}}', - 'multiple_items_empty_processed' => '{"Items":[],"Count":0,"ScannedCount":2,"@metadata":{"statusCode":200,"effectiveUri":"https:\/\/dynamodb.ap-northeast-1.amazonaws.com","transferStats":{"http":[[]]}}}' + 'multiple_items_empty_processed' => '{"Items":[],"Count":0,"ScannedCount":2,"@metadata":{"statusCode":200,"effectiveUri":"https:\/\/dynamodb.ap-northeast-1.amazonaws.com","transferStats":{"http":[[]]}}}', + 'batch_get_items_result' => '{"Responses":{"Thread":[{"Replies":{"N":"0"},"Answered":{"N":"0"},"Views":{"N":"0"},"ForumName":{"S":"Amazon DynamoDB"},"Subject":{"S":"DynamoDB Thread 1"}},{"Replies":{"N":"0"},"Answered":{"N":"0"},"Views":{"N":"0"},"ForumName":{"S":"Amazon DynamoDB"},"Subject":{"S":"DynamoDB Thread 2"}}]},"UnprocessedKeys":[],"@metadata":{"statusCode":200,"effectiveUri":"https:\/\/dynamodb.ap-northeast-1.amazonaws.com","transferStats":{"http":[[]]}}}', + 'batch_get_items_processed' => '{"Responses":{"Thread":[{"Replies":0,"Answered":0,"Views":0,"ForumName":"Amazon DynamoDB","Subject":"DynamoDB Thread 1"},{"Replies":0,"Answered":0,"Views":0,"ForumName":"Amazon DynamoDB","Subject":"DynamoDB Thread 2"}]},"UnprocessedKeys":[],"@metadata":{"statusCode":200,"effectiveUri":"https:\/\/dynamodb.ap-northeast-1.amazonaws.com","transferStats":{"http":[[]]}}}', + 'batch_get_items_empty_result' => '{"Responses":{"Thread":[]},"UnprocessedKeys":[],"@metadata":{"statusCode":200,"effectiveUri":"https:\/\/dynamodb.ap-northeast-1.amazonaws.com","transferStats":{"http":[[]]}}}', + 'batch_get_items_empty_processed' => '{"Responses":{"Thread":[]},"UnprocessedKeys":[],"@metadata":{"statusCode":200,"effectiveUri":"https:\/\/dynamodb.ap-northeast-1.amazonaws.com","transferStats":{"http":[[]]}}}', ]; protected function setUp() :void @@ -79,6 +83,30 @@ public function it_can_process_multiple_items_empty_result() $this->assertEquals($expected, $items); } + /** @test */ + public function it_can_process_batch_get_items_result() + { + $expected = json_decode($this->mocks['batch_get_items_processed'], true); + + $awsResult = new Result(json_decode($this->mocks['batch_get_items_result'], true)); + + $items = $this->processor->processBatchGetItems($awsResult, null); + + $this->assertEquals($expected, $items); + } + + /** @test */ + public function it_can_process_batch_get_items_empty_result() + { + $expected = json_decode($this->mocks['batch_get_items_empty_processed'], true); + + $awsResult = new Result(json_decode($this->mocks['batch_get_items_empty_result'], true)); + + $items = $this->processor->processBatchGetItems($awsResult, null); + + $this->assertEquals($expected, $items); + } + /** @test */ public function it_can_convert_single_result_to_model_instance() { @@ -113,4 +141,24 @@ public function it_can_convert_multiple_results_to_model_instance() ], $item->toArray()); $this->assertEquals(200, $item->meta()['@metadata']['statusCode']); } + + /** @test */ + public function it_can_convert_batch_get_results_to_model_instance() + { + $awsResult = new Result(json_decode($this->mocks['batch_get_items_result'], true)); + + $items = $this->processor->processBatchGetItems($awsResult, User::class); + + $item = $items->first(); + + $this->assertEquals(User::class, get_class($item)); + $this->assertEquals([ + 'Replies' => 0, + 'Answered' => 0, + 'Views' => 0, + 'ForumName' => 'Amazon DynamoDB', + 'Subject' => 'DynamoDB Thread 1', + ], $item->toArray()); + $this->assertEquals(200, $item->meta()['@metadata']['statusCode']); + } } From e29ad0462314baa1948f5fcd7f1bcb5fbfa525cc Mon Sep 17 00:00:00 2001 From: Satoshi Kita Date: Wed, 14 Dec 2022 19:43:04 +0900 Subject: [PATCH 4/4] add document for batch operations --- README.md | 91 +++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 85 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 70e3852..6064e18 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,11 @@ You can find an example implementation in [kitar/simplechat](https://github.com/ * [Using Global Secondary Indexes](#using-global-secondary-indexes) + [index()](#index) * [Atomic Counter](#atomic-counter) + * [Batch Operations](#batch-operations) + + [batchGetItem()](#batchgetitem) + + [batchPutItem()](#batchputitem) + + [batchDeleteItem()](#batchdeleteitem) + + [batchWriteItem()](#batchwriteitem) * [DynamoDB-specific operators for condition() and filter()](#dynamodb-specific-operators-for-condition-and-filter) + [Comparators](#comparators) + [functions](#functions) @@ -197,7 +202,6 @@ class User extends Model implements AuthenticatableContract } ``` -> **Note** > Note that this model is implementing `Illuminate\Contracts\Auth\Authenticatable` and using `Illuminate\Auth\Authenticatable`. This is **optional**, but if we use them, we can use this model with authentication as well. For authentication, please refer to [Authentication section](#authentication-with-model)) for more details. ### Basic Usage @@ -234,7 +238,6 @@ public static function scan($exclusiveStartKey = null, $sort = 'asc', $limit = 5 } ``` -> **Note** > DynamoDB can only handle result set up to 1MB per call, so we have to paginate if there are more results. see [Paginating the Results](#paginating-the-results) for more details. #### Retrieving a model @@ -474,7 +477,6 @@ $response = DB::table('ProductCatalog') ->getItem(['Id' => 101]); ``` -> **Note** > Instead of marshaling manually, pass a plain array. `Kitar\Dynamodb\Query\Grammar` will automatically marshal them before querying. #### putItem() @@ -547,7 +549,6 @@ DB::table('ProductCatalog') ]); ``` -> **Note** > Note that we specify `attribute_not_exists` for the operator of condition. This is DynamoDB-specific operator which called `function`. See [DynamoDB-specific operators for condition() and filter()](#dynamodb-specific-operators-for-condition-and-filter) for more details. OR statements @@ -625,7 +626,6 @@ $response = DB::table('Thread') ->query(); ``` -> **Note** > Note that DynamoDB's `ScanIndexForward` is a feature for `query`. It will not work with `scan`. ### Working with Scans @@ -755,6 +755,86 @@ DB::('Thread')->key([ ]); ``` +### Batch Operations + +Batch operations can get, put or delete multiple items with a single call. There are some DynamoDB limitations (such as items count, payload size, etc), so please check the documentation in advance. ([BatchGetItem](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchGetItem.html), [BatchWriteItem](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchWriteItem.html)) + +#### batchGetItem() + +```php +DB::table('Thread') + ->batchGetItem([ + [ + 'ForumName' => 'Amazon DynamoDB', + 'Subject' => 'DynamoDB Thread 1' + ], + [ + 'ForumName' => 'Amazon DynamoDB', + 'Subject' => 'DynamoDB Thread 2' + ] + ]); +``` + +#### batchPutItem() + +```php +DB::table('Thread') + ->batchPutItem([ + [ + 'ForumName' => 'Amazon DynamoDB', + 'Subject' => 'DynamoDB Thread 3' + ], + [ + 'ForumName' => 'Amazon DynamoDB', + 'Subject' => 'DynamoDB Thread 4' + ] + ]); +``` + +> This is a handy method to batch-put items using `batchWriteItem` + +#### batchDeleteItem() + +```php +DB::table('Thread') + ->batchDeleteItem([ + [ + 'ForumName' => 'Amazon DynamoDB', + 'Subject' => 'DynamoDB Thread 1' + ], + [ + 'ForumName' => 'Amazon DynamoDB', + 'Subject' => 'DynamoDB Thread 2' + ] + ]); +``` + +> This is a handy method to batch-delete items using `batchWriteItem` + +#### batchWriteItem() + +```php +DB::table('Thread') + ->batchWriteItem([ + [ + 'PutRequest' => [ + 'Item' => [ + 'ForumName' => 'Amazon DynamoDB', + 'Subject' => 'DynamoDB Thread 3' + ] + ] + ], + [ + 'DeleteRequest' => [ + 'Key' => [ + 'ForumName' => 'Amazon DynamoDB', + 'Subject' => 'DynamoDB Thread 1' + ] + ] + ] + ]); +``` + ### DynamoDB-specific operators for condition() and filter() For `condition` and `filter` clauses, we can use DynamoDB's comparators and functions. @@ -779,7 +859,6 @@ filter($key, 'begins_with', $value); filter($key, 'contains', $value); ``` -> **Note** > `size` function is not supported at this time. ## Testing