Skip to content

Commit

Permalink
fix: update progress tracking in LLM record processing
Browse files Browse the repository at this point in the history
  • Loading branch information
DerKernigeFeuerpfeil committed Jan 28, 2025
1 parent 24d83d8 commit 92a5912
Showing 1 changed file with 6 additions and 1 deletion.
7 changes: 6 additions & 1 deletion run_ac.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,8 @@ def parse_data_to_record_dict(record_chunk):
idx = 0
progress_size = 100
amount = len(record_dict_list)
__print_progress(0.0)
current_progress = 0.0
__print_progress(current_progress)

async def process_llm_record_batch(record_dict_batch: list):
"""Process a batch of record_dicts, writes results into shared var calculated_attribute_by_record_id."""
Expand All @@ -148,6 +149,10 @@ async def process_llm_record_batch(record_dict_batch: list):
)
calculated_attribute_by_record_id[record_dict["id"]] = attr_value

global current_progress
current_progress = current_progress + len(record_dict_batch) / amount
__print_progress(round(current_progress, 2))

async def process_async_llm_calls(record_dict_list):
batch_size = max(
len(record_dict_list) // int(attribute_calculators.NUM_WORKERS_A2VYBG), 1
Expand Down

0 comments on commit 92a5912

Please sign in to comment.