From d93252d2a5975a931baedea9f365ca2403f4be98 Mon Sep 17 00:00:00 2001 From: Robert Date: Mon, 11 Nov 2024 15:04:45 -0500 Subject: [PATCH] mibios/query: Small speed-up for iterate() when using FKCache and values_list() FKCache now makes a list per row () instead of a tuple) and the PK gets removed via list.remove() (previously by slicing (which makes a new tuple.) The boost is up to 10% on iterate() diminishing to 4% when behind Value2CSVGenerator. --- mibios/query.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/mibios/query.py b/mibios/query.py index ec8a95ad..95cf312d 100644 --- a/mibios/query.py +++ b/mibios/query.py @@ -429,10 +429,10 @@ def get_map_value_row_fn(self, queryset): list_of_caches.append(self.get(fk_field, None)) def map_row_fn(row): - return tuple(( + return [ val if (cache is None or val is None) else cache[val] for cache, val in zip(list_of_caches, row) - )) + ] return map_row_fn @@ -584,12 +584,18 @@ def __init__(self, queryset, chunk_size, cache): self.pk_pos = pk_pos self.hide_pk = hide_pk + @staticmethod + def _rm_pk(row): + del row[0] # PK is always first elem if we have to remove it + return row + def _iter(self): qs = self.queryset cache = self.cache chunk_size = self.chunk_size pk_pos = self.pk_pos hide_pk = self.hide_pk + rm_pk = self._rm_pk last_pk = 0 while True: @@ -600,13 +606,15 @@ def _iter(self): chunk = cache.update_values_list(chunk) # For non-empty chunk get last PK before they are removed. Must - # also avoid negative indexing in case chunk is queryset, so - # calculate last row via length. + # also avoid negative indexing in no-cache case where chunk is + # queryset, so calculate last row via length. if chunk_length := len(chunk): last_pk = chunk[chunk_length - 1][pk_pos] if hide_pk: - chunk = ((i[slice(1, None)] for i in chunk)) + # if we get here, then pk_pos is always 0 + # chunk = ((i[slice(1, None)] for i in chunk)) + chunk = ((rm_pk(row) for row in chunk)) yield from chunk