From 415b622a646089a4566527740cc2eb90c767862b Mon Sep 17 00:00:00 2001 From: Andrzej Dabrowski Date: Fri, 11 Sep 2020 14:08:17 +0200 Subject: [PATCH 1/5] custom none name --- CONTRIBUTING.rst | 1 + djqscsv/djqscsv.py | 12 +++++-- .../djqscsv_tests/tests/test_utilities.py | 33 ++++++++++++++++--- 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 82fc83d..7bd6a0c 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -30,6 +30,7 @@ demo testing to ensure the app behaves as expected, run the following:: $ cd test_app + $ python manage.py migrate $ python manage.py runserver then, visit ``http://localhost:8000/`` in your browser and confirm it produces a valid CSV. diff --git a/djqscsv/djqscsv.py b/djqscsv/djqscsv.py index c530fc1..64f33e8 100644 --- a/djqscsv/djqscsv.py +++ b/djqscsv/djqscsv.py @@ -14,7 +14,7 @@ # the rest will be passed along to the csv writer DJQSCSV_KWARGS = { 'field_header_map', 'field_serializer_map', 'use_verbose_names', - 'field_order'} + 'field_order', 'custom_none_name'} class CSVException(Exception): @@ -78,6 +78,7 @@ def _iter_csv(queryset, file_obj, **kwargs): field_header_map = kwargs.get('field_header_map', {}) field_serializer_map = kwargs.get('field_serializer_map', {}) use_verbose_names = kwargs.get('use_verbose_names', True) + custom_none_name = kwargs.get('custom_none_name', None) field_order = kwargs.get('field_order', None) csv_kwargs = {'encoding': 'utf-8'} @@ -152,7 +153,7 @@ def _iter_csv(queryset, file_obj, **kwargs): yield writer.writerow(merged_header_map) for record in values_qs: - record = _sanitize_record(field_serializer_map, record) + record = _sanitize_record(field_serializer_map, record, custom_none_name) yield writer.writerow(record) @@ -186,7 +187,7 @@ def _validate_and_clean_filename(filename): return filename -def _sanitize_record(field_serializer_map, record): +def _sanitize_record(field_serializer_map, record, custom_none_name): def _serialize_value(value): # provide default serializer for the case when @@ -201,11 +202,16 @@ def _serialize_value(value): if val is not None: serializer = field_serializer_map.get(key, _serialize_value) newval = serializer(val) + # If the user provided None + if newval is None: + newval = custom_none_name # If the user provided serializer did not produce a string, # coerce it to a string if not isinstance(newval, six.text_type): newval = six.text_type(newval) obj[key] = newval + elif custom_none_name: + obj[key] = custom_none_name return obj diff --git a/test_app/djqscsv_tests/tests/test_utilities.py b/test_app/djqscsv_tests/tests/test_utilities.py index 671e9af..4b30cc8 100644 --- a/test_app/djqscsv_tests/tests/test_utilities.py +++ b/test_app/djqscsv_tests/tests/test_utilities.py @@ -47,7 +47,7 @@ class SanitizeUnicodeRecordTests(TestCase): def test_sanitize(self): record = {'name': 'Tenar', 'nickname': u'\ufeffThe White Lady of Gont'} - sanitized = djqscsv._sanitize_record({}, record) + sanitized = djqscsv._sanitize_record({}, record, None) self.assertEqual(sanitized, {'name': 'Tenar', 'nickname': u'\ufeffThe White Lady of Gont'}) @@ -55,11 +55,34 @@ def test_sanitize(self): def test_sanitize_date(self): record = {'name': 'Tenar', 'created': datetime.datetime(1, 1, 1)} - sanitized = djqscsv._sanitize_record({}, record) + sanitized = djqscsv._sanitize_record({}, record, None) self.assertEqual(sanitized, {'name': 'Tenar', 'created': '0001-01-01T00:00:00'}) + def test_sanitize_with_custom_none_name(self): + """ + Ensure we retrieve custom none name if model field is None + """ + record = {'name': 'Tenar', + 'created': None} + custom_name_for_nones = "custom_name_for_nones" + sanitized = djqscsv._sanitize_record({}, record, custom_name_for_nones) + self.assertEqual(sanitized, + {'name': 'Tenar', + 'created': custom_name_for_nones}) + """ + Ensure we retrieve custom none name if user provided None for field value + """ + + record = {'name': 'Tenar', + 'created': datetime.datetime(1, 1, 1)} + serializer = {'created': lambda x: None} + sanitized = djqscsv._sanitize_record(serializer, record, custom_name_for_nones) + self.assertEqual(sanitized, + {'name': 'Tenar', + 'created': custom_name_for_nones}) + def test_sanitize_date_with_non_string_formatter(self): """ This test is only to make sure an edge case provides a sane @@ -68,14 +91,14 @@ def test_sanitize_date_with_non_string_formatter(self): """ record = {'name': 'Tenar'} serializer = {'name': lambda d: len(d)} - sanitized = djqscsv._sanitize_record(serializer, record) + sanitized = djqscsv._sanitize_record(serializer, record, None) self.assertEqual(sanitized, {'name': '5'}) def test_sanitize_date_with_formatter(self): record = {'name': 'Tenar', 'created': datetime.datetime(1973, 5, 13)} serializer = {'created': lambda d: d.strftime('%Y-%m-%d')} - sanitized = djqscsv._sanitize_record(serializer, record) + sanitized = djqscsv._sanitize_record(serializer, record, None) self.assertEqual(sanitized, {'name': 'Tenar', 'created': '1973-05-13'}) @@ -84,7 +107,7 @@ def test_sanitize_date_with_bad_formatter(self): record = {'name': 'Tenar', 'created': datetime.datetime(1973, 5, 13)} with self.assertRaises(AttributeError): - djqscsv._sanitize_record(attrgetter('day'), record) + djqscsv._sanitize_record(attrgetter('day'), record, None) class AppendDatestampTests(TestCase): From 32fad893a0113f7c5074c8480754545daebcd2c8 Mon Sep 17 00:00:00 2001 From: Andrzej Dabrowski Date: Fri, 11 Sep 2020 14:30:22 +0200 Subject: [PATCH 2/5] custom none name --- djqscsv/djqscsv.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/djqscsv/djqscsv.py b/djqscsv/djqscsv.py index 64f33e8..de2f216 100644 --- a/djqscsv/djqscsv.py +++ b/djqscsv/djqscsv.py @@ -153,7 +153,8 @@ def _iter_csv(queryset, file_obj, **kwargs): yield writer.writerow(merged_header_map) for record in values_qs: - record = _sanitize_record(field_serializer_map, record, custom_none_name) + record = _sanitize_record(field_serializer_map, record, + custom_none_name) yield writer.writerow(record) From f8bda1afc21d1db8bfe2e067fafedf5ccad6e592 Mon Sep 17 00:00:00 2001 From: Andrzej Dabrowski Date: Fri, 11 Sep 2020 14:35:53 +0200 Subject: [PATCH 3/5] flake8 --- test_app/djqscsv_tests/tests/test_utilities.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test_app/djqscsv_tests/tests/test_utilities.py b/test_app/djqscsv_tests/tests/test_utilities.py index 4b30cc8..a484066 100644 --- a/test_app/djqscsv_tests/tests/test_utilities.py +++ b/test_app/djqscsv_tests/tests/test_utilities.py @@ -67,7 +67,8 @@ def test_sanitize_with_custom_none_name(self): record = {'name': 'Tenar', 'created': None} custom_name_for_nones = "custom_name_for_nones" - sanitized = djqscsv._sanitize_record({}, record, custom_name_for_nones) + sanitized = djqscsv._sanitize_record({}, record, + custom_name_for_nones) self.assertEqual(sanitized, {'name': 'Tenar', 'created': custom_name_for_nones}) @@ -78,7 +79,8 @@ def test_sanitize_with_custom_none_name(self): record = {'name': 'Tenar', 'created': datetime.datetime(1, 1, 1)} serializer = {'created': lambda x: None} - sanitized = djqscsv._sanitize_record(serializer, record, custom_name_for_nones) + sanitized = djqscsv._sanitize_record(serializer, record, + custom_name_for_nones) self.assertEqual(sanitized, {'name': 'Tenar', 'created': custom_name_for_nones}) From b12cf1a5caec8d4e4cbde62b44ddcf268ac474a9 Mon Sep 17 00:00:00 2001 From: Andrzej Dabrowski Date: Fri, 11 Sep 2020 14:39:37 +0200 Subject: [PATCH 4/5] flake8 --- test_app/djqscsv_tests/tests/test_utilities.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_app/djqscsv_tests/tests/test_utilities.py b/test_app/djqscsv_tests/tests/test_utilities.py index a484066..6e3c364 100644 --- a/test_app/djqscsv_tests/tests/test_utilities.py +++ b/test_app/djqscsv_tests/tests/test_utilities.py @@ -73,7 +73,8 @@ def test_sanitize_with_custom_none_name(self): {'name': 'Tenar', 'created': custom_name_for_nones}) """ - Ensure we retrieve custom none name if user provided None for field value + Ensure we retrieve custom none name if user provided None + for field value """ record = {'name': 'Tenar', From 9435d485cae97bf0d86a97b6b2319644bda7ec83 Mon Sep 17 00:00:00 2001 From: Andrzej Dabrowski Date: Fri, 11 Sep 2020 14:46:36 +0200 Subject: [PATCH 5/5] flake8 --- test_app/djqscsv_tests/tests/test_utilities.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/test_app/djqscsv_tests/tests/test_utilities.py b/test_app/djqscsv_tests/tests/test_utilities.py index 6e3c364..d45d2f6 100644 --- a/test_app/djqscsv_tests/tests/test_utilities.py +++ b/test_app/djqscsv_tests/tests/test_utilities.py @@ -62,7 +62,7 @@ def test_sanitize_date(self): def test_sanitize_with_custom_none_name(self): """ - Ensure we retrieve custom none name if model field is None + Ensure we retrieve custom none name """ record = {'name': 'Tenar', 'created': None} @@ -72,10 +72,6 @@ def test_sanitize_with_custom_none_name(self): self.assertEqual(sanitized, {'name': 'Tenar', 'created': custom_name_for_nones}) - """ - Ensure we retrieve custom none name if user provided None - for field value - """ record = {'name': 'Tenar', 'created': datetime.datetime(1, 1, 1)}