Don't require upstream patches for boto3

The AWS Python SDK, [boto3], has [resource] objects that provide
high-level interfaces to AWS services. The [DynamoDB resource]
greatly simplifies marshalling and unmarshalling data. We rely on
the resource method for [TransactWriteItems] among others that are
absent from boto3. We opened PR
boto/boto3#4010 to add that method.

The resource methods are synthesized at runtime from a data file.
Fortunately, boto3 has a [Loader] mechanism that allows the user to
add extra data files, and the [loader search path] is configurable.

In order to not depend upon our upstream PR for boto3, we distribute
the extra data files and fix up the loader search path by putting it
in a [.pth file] which Python executes automatically during startup.

FIXME: The .pth file method does not currently work when we are doing
editable installs. It _should_ work with editable installs if we factor
the data files and the .pth file to a separate package that we add as
a dependency.

[boto3]: https://github.com/boto/boto3
[resource]: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/resources.html
[DynamoDB resource]: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#resources
[TransactWriteItems]: https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_TransactWriteItems.html
[Loader]: https://botocore.amazonaws.com/v1/documentation/api/latest/reference/loaders.html
[loader search path]: https://botocore.amazonaws.com/v1/documentation/api/latest/reference/loaders.html#the-search-path
[.pth file]: https://docs.python.org/3/library/site.html

Author: lpsinger

README.md

@@ -6,8 +6,3 @@ Use optimistic locking to put DynamoDB records with auto-incrementing attributes
 
 - https://aws.amazon.com/blogs/aws/new-amazon-dynamodb-transactions/
 - https://bitesizedserverless.com/bite/reliable-auto-increments-in-dynamodb/
-
-## FIXME
-
-This package currently depends on code that is in a pull request for boto3 that is not yet merged or released.
-See https://github.com/boto/boto3/pull/4010.

boto3_missing.pth

@@ -0,0 +1 @@
+import boto3_missing; boto3_missing.install()

boto3_missing/__init__.py

@@ -0,0 +1,20 @@
+# Copyright © 2023 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+"""Add missing boto3 SDK data.
+
+See
+https://botocore.amazonaws.com/v1/documentation/api/latest/reference/loaders.html,
+https://github.com/boto/boto3/pull/4010
+"""
+
+from os import environ, pathsep
+
+from . import data
+
+
+def install():
+    new_path = [*data.__path__]
+    if orig_path := environ.get("AWS_DATA_PATH"):
+        new_path.extend(orig_path.split(pathsep))
+    environ["AWS_DATA_PATH"] = pathsep.join(new_path)

boto3_missing/data/dynamodb/2012-08-10/resources-1.sdk-extras.json

@@ -0,0 +1,23 @@
+{
+  "merge": {
+    "service": {
+      "actions": {
+        "GetItem": {
+          "request": { "operation": "GetItem" }
+        },
+        "PutItem": {
+          "request": { "operation": "PutItem" }
+        },
+        "Query": {
+          "request": { "operation": "Query" }
+        },
+        "Scan": {
+          "request": { "operation": "Scan" }
+        },
+        "TransactWriteItems": {
+          "request": { "operation": "TransactWriteItems" }
+        }
+      }
+    }
+  }
+}

dynamodb_autoincrement.py

@@ -9,6 +9,9 @@
 
 from mypy_boto3_dynamodb.service_resource import DynamoDBServiceResource
 
+# FIXME: remove instances of 'type: ignore[attr-defined]' below once
+# boto3-missing becomes unnecessary.
+
 
 PrimitiveDynamoDBValues = Optional[Union[str, int, float, Decimal, bool]]
 DynamoDBValues = Union[
@@ -33,18 +36,6 @@ class BaseDynamoDBAutoIncrement(ABC):
     def next(self, item: DynamoDBItem) -> tuple[Iterable[dict[str, Any]], str]:
         raise NotImplementedError
 
-    def _put_item(self, *, TableName, **kwargs):
-        # FIXME: DynamoDB resource does not have put_item method; emulate it
-        self.dynamodb.Table(TableName).put_item(**kwargs)
-
-    def _get_item(self, *, TableName, **kwargs):
-        # FIXME: DynamoDB resource does not have get_item method; emulate it
-        return self.dynamodb.Table(TableName).get_item(**kwargs)
-
-    def _query(self, *, TableName, **kwargs):
-        # FIXME: DynamoDB resource does not have put_item method; emulate it
-        return self.dynamodb.Table(TableName).query(**kwargs)
-
     def put(self, item: DynamoDBItem):
         TransactionCanceledException = (
             self.dynamodb.meta.client.exceptions.TransactionCanceledException
@@ -53,11 +44,9 @@ def put(self, item: DynamoDBItem):
             puts, next_counter = self.next(item)
             if self.dangerously:
                 for put in puts:
-                    self._put_item(**put)
+                    self.dynamodb.put_item(**put)  # type: ignore[attr-defined]
             else:
                 try:
-                    # FIXME: depends on an unmerged PR for boto3.
-                    # See https://github.com/boto/boto3/pull/4010
                     self.dynamodb.transact_write_items(  # type: ignore[attr-defined]
                         TransactItems=[{"Put": put} for put in puts]
                     )
@@ -69,7 +58,7 @@ def put(self, item: DynamoDBItem):
 class DynamoDBAutoIncrement(BaseDynamoDBAutoIncrement):
     def next(self, item):
         counter = (
-            self._get_item(
+            self.dynamodb.get_item(
                 AttributesToGet=[self.attribute_name],
                 Key=self.counter_table_key,
                 TableName=self.counter_table_name,
@@ -117,7 +106,7 @@ def next(self, item):
 
 class DynamoDBHistoryAutoIncrement(BaseDynamoDBAutoIncrement):
     def list(self) -> list[int]:
-        result = self._query(
+        result = self.dynamodb.query(  # type: ignore[attr-defined]
             TableName=self.table_name,
             ExpressionAttributeNames={
                 **{f"#{i}": key for i, key in enumerate(self.counter_table_key.keys())},
@@ -145,10 +134,10 @@ def get(self, version: Optional[int] = None) -> DynamoDBItem:
                 "TableName": self.table_name,
                 "Key": {**self.counter_table_key, self.attribute_name: version},
             }
-        return self._get_item(**kwargs).get("Item")
+        return self.dynamodb.get_item(**kwargs).get("Item")  # type: ignore[attr-defined]
 
     def next(self, item):
-        existing_item = self._get_item(
+        existing_item = self.dynamodb.get_item(
             TableName=self.counter_table_name,
             Key=self.counter_table_key,
         ).get("Item")

pyproject.toml

@@ -16,7 +16,7 @@ classifiers = [
     "Topic :: Database",
 ]
 dependencies = [
-    "boto3 @ git+https://github.com/lpsinger/boto3@dynamodb-resource-transact-write-items",
+    "boto3",
     "boto3-stubs[dynamodb]",
 ]
 requires-python = ">=3.9"
@@ -41,9 +41,13 @@ exclude_also = [
     "@abstractmethod"
 ]
 
-[tool.setuptools]
-py-modules = [
+[tool.setuptools.packages.find]
+include = [
     "dynamodb_autoincrement",
+    "boto3_missing",
 ]
 
+[tool.setuptools.package-data]
+boto3_missing = ["**/*.json", "../*.pth"]
+
 [tool.setuptools_scm]

test_dynamodb_autoincrement.py

@@ -91,19 +91,19 @@ def test_autoincrement_safely(autoincrement_safely, dynamodb, last_id):
     if last_id is None:
         next_id = 1
     else:
-        dynamodb.Table("autoincrement").put_item(
+        dynamodb.put_item(TableName="autoincrement",
             Item={"tableName": "widgets", "widgetID": last_id}
         )
         next_id = last_id + 1
 
     result = autoincrement_safely.put({"widgetName": "runcible spoon"})
     assert result == next_id
 
-    assert dynamodb.Table("widgets").scan()["Items"] == [
+    assert dynamodb.scan(TableName="widgets")["Items"] == [
         {"widgetID": next_id, "widgetName": "runcible spoon"},
     ]
 
-    assert dynamodb.Table("autoincrement").scan()["Items"] == [
+    assert dynamodb.scan(TableName="autoincrement")["Items"] == [
         {
             "tableName": "widgets",
             "widgetID": next_id,
@@ -152,7 +152,7 @@ def test_autoincrement_dangerously_fails_on_many_parallel_puts(
 @pytest.fixture(params=[None, {"widgetID": 1}, {"widgetID": 1, "version": 1}])
 def initial_item(request, create_tables, dynamodb):
     if request.param is not None:
-        dynamodb.Table("widgets").put_item(Item=request.param)
+        dynamodb.put_item(TableName="widgets", Item=request.param)
     return request.param
 
 
@@ -174,7 +174,7 @@ def test_autoincrement_version(
     )
     assert new_version == 1 + has_initial_item
 
-    history_items = dynamodb.Table("widgetHistory").query(
+    history_items = dynamodb.query(
         TableName="widgetHistory",
         KeyConditionExpression="widgetID = :widgetID",
         ExpressionAttributeValues={