From 0420357339de5c38516ab48797401d51ee05577c Mon Sep 17 00:00:00 2001 From: aemous Date: Fri, 4 Apr 2025 12:07:11 -0400 Subject: [PATCH 1/5] Update JSON formatter to encode raw bytes to Base64 strings. --- .changes/next-release/bugfix-Formatter-33861.json | 5 +++++ awscli/utils.py | 8 ++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 .changes/next-release/bugfix-Formatter-33861.json diff --git a/.changes/next-release/bugfix-Formatter-33861.json b/.changes/next-release/bugfix-Formatter-33861.json new file mode 100644 index 000000000000..a1cd65c63421 --- /dev/null +++ b/.changes/next-release/bugfix-Formatter-33861.json @@ -0,0 +1,5 @@ +{ + "type": "bugfix", + "category": "Formatter", + "description": "Update JSON formatter to encode raw bytes as UTF-8." +} diff --git a/awscli/utils.py b/awscli/utils.py index b35d4b10a54b..fafaafcee088 100644 --- a/awscli/utils.py +++ b/awscli/utils.py @@ -10,6 +10,7 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +import base64 import contextlib import csv import datetime @@ -371,11 +372,14 @@ def operation_uses_document_types(operation_model): def json_encoder(obj): - """JSON encoder that formats datetimes as ISO8601 format.""" + """JSON encoder that formats datetimes as ISO8601 format + and encodes bytes to UTF-8 Base64 string.""" if isinstance(obj, datetime.datetime): return obj.isoformat() + elif isinstance(obj, bytes): + return base64.b64encode(obj).decode("utf-8") else: - return obj + raise TypeError('Encountered unrecognized type in JSON encoder.') @contextlib.contextmanager From b7b5c9387f163255baf3abbe59fab5e71e046107 Mon Sep 17 00:00:00 2001 From: aemous Date: Fri, 11 Apr 2025 12:05:48 -0400 Subject: [PATCH 2/5] Add unit test for JSON formatter raw bytes. --- tests/unit/test_formatter.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/unit/test_formatter.py b/tests/unit/test_formatter.py index db24f1946754..de111a793e41 100644 --- a/tests/unit/test_formatter.py +++ b/tests/unit/test_formatter.py @@ -10,6 +10,8 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +import base64 + import io import os import sys @@ -180,3 +182,22 @@ def test_encoding_override(self, env_vars): '}\n' ).encode() ) + + def test_raw_bytes_get_base64_encoded(self): + raw_bytes = b'foo' + response = {'BinaryValue': raw_bytes} + stdout_b = io.BytesIO() + stdout = io.TextIOWrapper(stdout_b, newline='\n') + + with contextlib.redirect_stdout(stdout): + self.formatter('command-name', response, sys.stdout) + stdout.flush() + + assert ( + stdout_b.getvalue() + == ( + '{\n' + f' "BinaryValue": "{base64.b64encode(raw_bytes).decode("utf-8")}"\n' + '}\n' + ).encode() + ) \ No newline at end of file From aa4c71a595e5506909c72eafb1ff8377fc7568fe Mon Sep 17 00:00:00 2001 From: aemous Date: Fri, 11 Apr 2025 12:07:41 -0400 Subject: [PATCH 3/5] Run linter. --- tests/unit/test_formatter.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unit/test_formatter.py b/tests/unit/test_formatter.py index de111a793e41..02ef6005fdc8 100644 --- a/tests/unit/test_formatter.py +++ b/tests/unit/test_formatter.py @@ -11,7 +11,6 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. import base64 - import io import os import sys @@ -200,4 +199,4 @@ def test_raw_bytes_get_base64_encoded(self): f' "BinaryValue": "{base64.b64encode(raw_bytes).decode("utf-8")}"\n' '}\n' ).encode() - ) \ No newline at end of file + ) From cd62f4350c79ab555867879593a3aa134b47fc2b Mon Sep 17 00:00:00 2001 From: aemous Date: Mon, 14 Apr 2025 10:19:29 -0400 Subject: [PATCH 4/5] Swap test location. --- tests/unit/output/test_json_output.py | 31 +++++++++++++++++++++++++++ tests/unit/test_formatter.py | 22 +------------------ 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/tests/unit/output/test_json_output.py b/tests/unit/output/test_json_output.py index a7626f5cf488..5bb854617601 100644 --- a/tests/unit/output/test_json_output.py +++ b/tests/unit/output/test_json_output.py @@ -11,6 +11,14 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +import base64 + +import sys + +import contextlib + +import io + import platform from botocore.compat import json @@ -134,3 +142,26 @@ def test_fully_buffered_handles_io_error(self): # we still should have called the flush() on the # stream. fake_closed_stream.flush.assert_called_with() + + +class TestBinaryData(unittest.TestCase): + def test_binary_data_gets_base64_encoded(self): + args = mock.Mock(query=None) + raw_bytes = b'foo' + response = {'BinaryValue': raw_bytes} + stdout_b = io.BytesIO() + stdout = io.TextIOWrapper(stdout_b, newline='\n') + formatter = JSONFormatter(args) + + with contextlib.redirect_stdout(stdout): + formatter('command-name', response, sys.stdout) + stdout.flush() + + assert ( + stdout_b.getvalue() + == ( + '{\n' + f' "BinaryValue": "{base64.b64encode(raw_bytes).decode("utf-8")}"\n' + '}\n' + ).encode() + ) \ No newline at end of file diff --git a/tests/unit/test_formatter.py b/tests/unit/test_formatter.py index 02ef6005fdc8..882136bd6515 100644 --- a/tests/unit/test_formatter.py +++ b/tests/unit/test_formatter.py @@ -10,7 +10,6 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -import base64 import io import os import sys @@ -180,23 +179,4 @@ def test_encoding_override(self, env_vars): ' ]\n' '}\n' ).encode() - ) - - def test_raw_bytes_get_base64_encoded(self): - raw_bytes = b'foo' - response = {'BinaryValue': raw_bytes} - stdout_b = io.BytesIO() - stdout = io.TextIOWrapper(stdout_b, newline='\n') - - with contextlib.redirect_stdout(stdout): - self.formatter('command-name', response, sys.stdout) - stdout.flush() - - assert ( - stdout_b.getvalue() - == ( - '{\n' - f' "BinaryValue": "{base64.b64encode(raw_bytes).decode("utf-8")}"\n' - '}\n' - ).encode() - ) + ) \ No newline at end of file From b1f06a16c53157120ada9d7d8ef95e84f8d0a809 Mon Sep 17 00:00:00 2001 From: aemous Date: Mon, 14 Apr 2025 10:20:46 -0400 Subject: [PATCH 5/5] Linter. --- tests/unit/output/test_json_output.py | 20 ++++++++------------ tests/unit/test_formatter.py | 2 +- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/tests/unit/output/test_json_output.py b/tests/unit/output/test_json_output.py index 5bb854617601..317571f1c58d 100644 --- a/tests/unit/output/test_json_output.py +++ b/tests/unit/output/test_json_output.py @@ -12,14 +12,10 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. import base64 - -import sys - import contextlib - import io - import platform +import sys from botocore.compat import json @@ -158,10 +154,10 @@ def test_binary_data_gets_base64_encoded(self): stdout.flush() assert ( - stdout_b.getvalue() - == ( - '{\n' - f' "BinaryValue": "{base64.b64encode(raw_bytes).decode("utf-8")}"\n' - '}\n' - ).encode() - ) \ No newline at end of file + stdout_b.getvalue() + == ( + '{\n' + f' "BinaryValue": "{base64.b64encode(raw_bytes).decode("utf-8")}"\n' + '}\n' + ).encode() + ) diff --git a/tests/unit/test_formatter.py b/tests/unit/test_formatter.py index 882136bd6515..db24f1946754 100644 --- a/tests/unit/test_formatter.py +++ b/tests/unit/test_formatter.py @@ -179,4 +179,4 @@ def test_encoding_override(self, env_vars): ' ]\n' '}\n' ).encode() - ) \ No newline at end of file + )