Merge pull request #46 from artoonie/support-dominion-v517

Support dominion v517

rcvformats/conversions/dominion_xlsx.py

@@ -31,13 +31,6 @@ class RowConstants():
         """
         Data for the row number that various items are on
         """
-        # Define constants
-        # the -12 is because New Mexico files have 12 rows of headers,
-        # so the first number is the actual row number, then we subtract num_header_rows,
-        # just for readability.
-        SEAT_TITLE_NUM_ROWS_AFTER_HEADER = 12 - 12
-        ROUND_LABELS_NUM_ROWS_AFTER_HEADER = 32 - 12
-        FIRST_CANDIDATE_NUM_ROWS_AFTER_HEADER = 34 - 12
         ROW_AFTER_INACTIVE_FOR_THRESHOLD = 1
 
         def __init__(self):
@@ -56,14 +49,28 @@ def __init__(self):
             # Row that has the number of inactive ("non transferrable") ballots at each round
             self.inactive_ballots = None
 
-        def find_rows_before_summary_table(self, sheet):
+        def find_rows_before_summary_table(self, workbook):
             """
             Fills in values for rows before the summary table
             """
-            offset = self._count_num_header_rows(sheet)
-            self.round_label = self.ROUND_LABELS_NUM_ROWS_AFTER_HEADER + offset
-            self.first_candidate = self.FIRST_CANDIDATE_NUM_ROWS_AFTER_HEADER + offset
-            self.seat_title = self.SEAT_TITLE_NUM_ROWS_AFTER_HEADER + offset
+            # Define constants
+            # the -12 is because New Mexico files have 12 rows of headers,
+            # so the first number is the actual row number, then we subtract num_header_rows,
+            # just for readability.
+            seat_title_num_rows_after_header = 12 - 12
+            round_labels_num_rows_after_header = 32 - 12
+            first_candidate_num_rows_after_header = 34 - 12
+
+            offset = self._count_num_header_rows(workbook[workbook.sheetnames[0]])
+            self.seat_title = seat_title_num_rows_after_header + offset
+
+            # v5.17+: first candidate is on the second sheet at a fixed position
+            if len(workbook.sheetnames) != 1:
+                self.first_candidate = 7
+                self.round_label = 5
+            else:
+                self.first_candidate = first_candidate_num_rows_after_header + offset
+                self.round_label = round_labels_num_rows_after_header + offset
 
         def find_rows_after_summary_table(self, sheet, num_candidates):
             """
@@ -136,22 +143,32 @@ def __init__(self):
 
     def _convert_file_object_to_ut(self, file_object):
         workbook = load_workbook(file_object)  # note: somehow, readonly is 2x slower
-        self.sheet = workbook.active
 
-        self.row_constants.find_rows_before_summary_table(self.sheet)
+        # Round-by-round results are always on the last sheet.
+        # On Dominion >v5.17, they go in the second sheet; otherwise, they're on the
+        # first and only sheet.
+        config_sheet = workbook[workbook.sheetnames[0]]
+        if len(workbook.sheetnames) == 1:
+            round_by_round_sheet = config_sheet
+        else:
+            round_by_round_sheet = workbook[workbook.sheetnames[1]]
+        self.sheet = round_by_round_sheet
+        self.row_constants.find_rows_before_summary_table(workbook)
         self.candidates = self._parse_candidates()
         self.row_constants.find_rows_after_summary_table(self.sheet, len(self.candidates))
-
         self.data_per_round = self._parse_rounds()
+        results = self._get_vote_counts_per_candidate()
 
+        # Config headers are always on the first sheet.
+        self.sheet = workbook[workbook.sheetnames[0]]
         config = self._parse_config()
-        results = self._get_vote_counts_per_candidate()
+
         urcvt_data = {'config': config, 'results': results}
 
+        self.postprocess_remove_last_round_elimination(urcvt_data)
+        self._postprocess_set_threshold_from_spreadsheet(urcvt_data, round_by_round_sheet)
         workbook.close()
 
-        self.postprocess_remove_last_round_elimination(urcvt_data)
-        self._postprocess_set_threshold_from_spreadsheet(urcvt_data)
         return urcvt_data
 
     def _parse_config(self):
@@ -315,13 +332,13 @@ def _get_vote_counts_per_candidate(self):
                 'tallyResults': tally_results})
         return results
 
-    def _postprocess_set_threshold_from_spreadsheet(self, data):
+    def _postprocess_set_threshold_from_spreadsheet(self, data, sheet):
         """
         The threshold is always listed on the table of per-round info
         We don't guess here - if we can't find it, we leave it blank.
         """
         last_round_col = self.data_per_round[-1].column
         maybe_threshold_row = self.row_constants.maybe_threshold
         if maybe_threshold_row is not None:
-            threshold = self.sheet.cell(maybe_threshold_row, last_round_col).value
+            threshold = sheet.cell(maybe_threshold_row, last_round_col).value
             data['config']['threshold'] = threshold

rcvformats/conversions/ut_without_transfers.py

@@ -50,7 +50,13 @@ def _get_elected_names(cls, rounds, round_i):
     def _convert_tally_string_to_decimal(cls, rounds):
         for round_data in rounds:
             for person in round_data['tally']:
-                round_data['tally'][person] = float(round_data['tally'][person])
+                tally_non_float = round_data['tally'][person]
+                if tally_non_float is None:
+                    if person == "Inactive Ballots":
+                        tally_non_float = 0
+                    else:
+                        raise ValueError("Must have values for every candidate")
+                round_data['tally'][person] = float(tally_non_float)
 
     def _fill_in_tallyresults(self, rounds):
         """ Fill out rounds['tallyResults'] based on rounds['tally'] """

rcvformats/test/testconversions.py

@@ -96,14 +96,22 @@ def test_electionbuddy_conversion_accurate():
     _assert_conversion_correct(file_in, file_out, converter)
 
 
-def test_dominion_conversion_accurate():
+def test_dominion_old_conversion_accurate():
     """ Converts dominion_xlsx XLSX to the standard format """
     file_in = 'testdata/inputs/dominion_xlsx/las-cruces-mayor.xlsx'
     file_out = 'testdata/conversions/from-dominion.json'
     converter = dominion_xlsx.DominionXlsxConverter()
     _assert_conversion_correct(file_in, file_out, converter)
 
 
+def test_dominion_v5_17_conversion_accurate():
+    """ Converts dominion_xlsx XLSX to the standard format """
+    file_in = 'testdata/inputs/dominion_xlsx/v5_17_multi.xlsx'
+    file_out = 'testdata/conversions/from-dominion-v5-17.json'
+    converter = dominion_xlsx.DominionXlsxConverter()
+    _assert_conversion_correct(file_in, file_out, converter)
+
+
 def test_dominion_txt():
     """ Converts dominion_txt TXT file to the standard format """
     file_in = 'testdata/inputs/dominion.txt'

requirements-test.txt

@@ -1,3 +1,3 @@
 nose-py3==1.6.2
 pylint==2.17.4
-autopep8==1.5.7
+autopep8==2.0.2

requirements.txt

@@ -1,3 +1,3 @@
-jsonschema==3.2.0
-openpyxl==3.0.9
+jsonschema==4.17.3
+openpyxl==3.1.2
 defusedxml==0.7.1

testdata/conversions/from-dominion-v5-17.json

@@ -0,0 +1,109 @@
+{
+  "config": {
+    "date": "2023-11-07",
+    "contest": "Best Ice Cream Flavor",
+    "office": "Best Ice Cream Flavor",
+    "threshold": 145
+  },
+  "results": [
+    {
+      "round": 1,
+      "tally": {
+        "Vanilla": 90,
+        "Chocolate": 72,
+        "Strawberry": 72,
+        "Neapolitan": 0,
+        "Rainbow Sherbet": 18,
+        "Chocolate Chip": 36,
+        "Butter Pecan": 0,
+        "Write-in": 0,
+        "Inactive Ballots": 0
+      },
+      "tallyResults": [
+        {
+          "eliminated": "Neapolitan"
+        }
+      ]
+    },
+    {
+      "round": 2,
+      "tally": {
+        "Vanilla": 90,
+        "Chocolate": 72,
+        "Strawberry": 72,
+        "Rainbow Sherbet": 18,
+        "Chocolate Chip": 36,
+        "Butter Pecan": 0,
+        "Write-in": 0,
+        "Inactive Ballots": 0
+      },
+      "tallyResults": [
+        {
+          "eliminated": "Butter Pecan"
+        }
+      ]
+    },
+    {
+      "round": 3,
+      "tally": {
+        "Vanilla": 90,
+        "Chocolate": 72,
+        "Strawberry": 72,
+        "Rainbow Sherbet": 18,
+        "Chocolate Chip": 36,
+        "Write-in": 0,
+        "Inactive Ballots": 0
+      },
+      "tallyResults": [
+        {
+          "eliminated": "Rainbow Sherbet"
+        }
+      ]
+    },
+    {
+      "round": 4,
+      "tally": {
+        "Vanilla": 90,
+        "Chocolate": 72,
+        "Strawberry": 72,
+        "Chocolate Chip": 54,
+        "Write-in": 0,
+        "Inactive Ballots": 0
+      },
+      "tallyResults": [
+        {
+          "eliminated": "Chocolate Chip"
+        }
+      ]
+    },
+    {
+      "round": 5,
+      "tally": {
+        "Vanilla": 126,
+        "Chocolate": 72,
+        "Strawberry": 90,
+        "Write-in": 0,
+        "Inactive Ballots": 0
+      },
+      "tallyResults": [
+        {
+          "eliminated": "Chocolate"
+        }
+      ]
+    },
+    {
+      "round": 6,
+      "tally": {
+        "Vanilla": 126,
+        "Strawberry": 162,
+        "Write-in": 0,
+        "Inactive Ballots": 0
+      },
+      "tallyResults": [
+        {
+          "elected": "Strawberry"
+        }
+      ]
+    }
+  ]
+}