Skip to content

Commit

Permalink
mibios/query: Fix iterate() over values_list w/o PKs and no cache.
Browse files Browse the repository at this point in the history
Added back code to remove PK from tuple row, needed in the no-cache
case.
  • Loading branch information
robert102 committed Nov 21, 2024
1 parent b58accf commit 74cefe1
Showing 1 changed file with 12 additions and 10 deletions.
22 changes: 12 additions & 10 deletions mibios/query.py
Original file line number Diff line number Diff line change
Expand Up @@ -474,7 +474,7 @@ class BaseIterable:
Alternative iterable implementation for large table data export.
To be used by our Queryset.iterate(). These iterators are to be used only
once.
once. Results are always ordered by primary key.
"""
DEFAULT_CHUNK_SIZE = 20000

Expand All @@ -497,7 +497,7 @@ def debug(self):
def __iter__(self):
# Attach the iterator as attribute to allow easier introspection.
# Allow this to happen only once per object lifetime to reduce possible
# confusing.
# confusion.
if self._it is None:
self._it = self._iter()
else:
Expand Down Expand Up @@ -586,6 +586,7 @@ def __init__(self, queryset, chunk_size, cache):

@staticmethod
def _rm_pk(row):
""" helper to remove PK from a list row """
del row[0] # PK is always first elem if we have to remove it
return row

Expand All @@ -601,20 +602,21 @@ def _iter(self):
while True:
chunk = qs.filter(pk__gt=last_pk)[:chunk_size]

if cache:
# chunk is replaced, type is list now
chunk = cache.update_values_list(chunk)

# For non-empty chunk get last PK before they are removed. Must
# also avoid negative indexing in no-cache case where chunk is
# queryset, so calculate last row via length.
if chunk_length := len(chunk):
last_pk = chunk[chunk_length - 1][pk_pos]

if hide_pk:
# if we get here, then pk_pos is always 0
# chunk = ((i[slice(1, None)] for i in chunk))
chunk = ((rm_pk(row) for row in chunk))
if cache:
# chunk is replaced, type is list now (was tuple)
chunk = cache.update_values_list(chunk)
if hide_pk:
# rm PK from list
chunk = ((rm_pk(row) for row in chunk))
elif hide_pk:
# rm PK from tuple (get new tuple via slicing)
chunk = ((row[slice(1, None)] for row in chunk))

yield from chunk

Expand Down

0 comments on commit 74cefe1

Please sign in to comment.