Skip to content

Commit

Permalink
Merge pull request #65 from azavea/topic/support_custom_serialization
Browse files Browse the repository at this point in the history
Topic/support custom serialization
  • Loading branch information
Steve Lamb committed Dec 29, 2014
2 parents 2304dcf + b53bf16 commit 385fdb7
Show file tree
Hide file tree
Showing 4 changed files with 52 additions and 12 deletions.
3 changes: 2 additions & 1 deletion README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,10 @@ views.py::
keyword arguments
-----------------

This module exports two functions that write CSVs, ``render_to_csv_response`` and ``write_csv``. Both of these functions require their own positional arguments. In addition, they both take three optional keyword arguments:
This module exports two functions that write CSVs, ``render_to_csv_response`` and ``write_csv``. Both of these functions require their own positional arguments. In addition, they both take the following optional keyword arguments:

- ``field_header_map`` - (default: ``None``) A dictionary mapping names of model fields to column header names. If specified, the csv writer will use these column headers. Otherwise, it will use defer to other parameters for rendering column names.
- ``field_serializer_map`` - (default: ``{}``) A dictionary mapping names of model fields to functions that serialize them to text. For example, ``{'created': (lambda x: x.strftime('%Y/%m/%d')) }`` will serialize a datetime field called ``created``.
- ``use_verbose_names`` - (default: ``True``) A boolean determining whether to use the django field's ``verbose_name``, or to use it's regular field name as a column header. Note that if a given field is found in the ``field_header_map``, this value will take precendence.
- ``field_order`` - (default: ``None``) A list of fields to determine the sort order. This list need not be complete: any fields not specified will follow those in the list with the order they would have otherwise used.

Expand Down
28 changes: 20 additions & 8 deletions djqscsv/djqscsv.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
import datetime

from django.core.exceptions import ValidationError
from django.templatetags.l10n import localize
from django.utils.text import slugify
from django.http import HttpResponse

Expand All @@ -20,6 +19,7 @@
# Keyword arguments that will be used by this module
# the rest will be passed along to the csv writer
DJQSCSV_KWARGS = {'field_header_map': None,
'field_serializer_map': None,
'use_verbose_names': True,
'field_order': None}

Expand Down Expand Up @@ -59,6 +59,7 @@ def write_csv(queryset, file_obj, **kwargs):

# process keyword arguments to pull out the ones used by this function
field_header_map = kwargs.get('field_header_map', {})
field_serializer_map = kwargs.get('field_serializer_map', {})
use_verbose_names = kwargs.get('use_verbose_names', True)
field_order = kwargs.get('field_order', None)

Expand Down Expand Up @@ -121,7 +122,7 @@ def write_csv(queryset, file_obj, **kwargs):
writer.writerow(merged_header_map)

for record in values_qs:
record = _sanitize_unicode_record(record)
record = _sanitize_unicode_record(field_serializer_map, record)
writer.writerow(record)


Expand Down Expand Up @@ -154,20 +155,31 @@ def _validate_and_clean_filename(filename):
return filename


def _sanitize_unicode_record(record):
def _sanitize_unicode_record(field_serializer_map, record):

def _sanitize_value(value):
def _serialize_value(value):
# provide default serializer for the case when
# non text values get sent without a serializer
if isinstance(value, datetime.datetime):
return value.isoformat()
else:
return unicode(value)

def _sanitize_text(value):
# make sure every text value is of type 'str', coercing unicode
if isinstance(value, unicode):
return value.encode("utf-8")
elif isinstance(value, datetime.datetime):
return value.isoformat().encode("utf-8")
elif isinstance(value, str):
return value
else:
return localize(value)
return str(value).encode("utf-8")

obj = {}
for key, val in six.iteritems(record):
if val is not None:
obj[_sanitize_value(key)] = _sanitize_value(val)
serializer = field_serializer_map.get(key, _serialize_value)
newval = serializer(val)
obj[_sanitize_text(key)] = _sanitize_text(newval)

return obj

Expand Down
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

setup(
name='django-queryset-csv',
version='0.2.10',
version='0.3.0',
description='A simple python module for writing querysets to csv',
long_description=open('README.rst').read(),
author=author,
Expand Down
31 changes: 29 additions & 2 deletions test_app/djqscsv_tests/tests/test_utilities.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,19 +43,46 @@ class SanitizeUnicodeRecordTests(TestCase):
def test_sanitize(self):
record = {'name': 'Tenar',
'nickname': u'\ufeffThe White Lady of Gont'}
sanitized = djqscsv._sanitize_unicode_record(record)
sanitized = djqscsv._sanitize_unicode_record({}, record)
self.assertEqual(sanitized,
{'name': 'Tenar',
'nickname': '\xef\xbb\xbfThe White Lady of Gont'})

def test_sanitize_date(self):
record = {'name': 'Tenar',
'created': datetime.datetime(1, 1, 1)}
sanitized = djqscsv._sanitize_unicode_record(record)
sanitized = djqscsv._sanitize_unicode_record({}, record)
self.assertEqual(sanitized,
{'name': 'Tenar',
'created': '0001-01-01T00:00:00'})

def test_sanitize_date_with_non_string_formatter(self):
"""
This test is only to make sure an edge case provides a sane
default and works as expected. It is not recommended to follow
this practice.
"""
record = {'name': 'Tenar'}
serializer = {'name': lambda d: len(d) }
sanitized = djqscsv._sanitize_unicode_record(serializer, record)
self.assertEqual(sanitized, {'name': '5'})

def test_sanitize_date_with_formatter(self):
record = {'name': 'Tenar',
'created': datetime.datetime(1973, 5, 13)}
serializer = {'created': lambda d: d.strftime('%Y-%m-%d') }
sanitized = djqscsv._sanitize_unicode_record(serializer, record)
self.assertEqual(sanitized,
{'name': 'Tenar',
'created': '1973-05-13'})

def test_sanitize_date_with_bad_formatter(self):
record = {'name': 'Tenar',
'created': datetime.datetime(1973, 5, 13)}
formatter = lambda d: d.day
with self.assertRaises(AttributeError):
djqscsv._sanitize_unicode_record(formatter, record)


class AppendDatestampTests(TestCase):

Expand Down

0 comments on commit 385fdb7

Please sign in to comment.