Skip to content

Commit

Permalink
Fix benchmark memory issues
Browse files Browse the repository at this point in the history
  • Loading branch information
strtgbb committed Oct 24, 2024
1 parent 50a1165 commit 671afe0
Showing 1 changed file with 6 additions and 2 deletions.
8 changes: 6 additions & 2 deletions ontime_benchmark/tests/benchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,11 @@ def insert_ontime_data(self, from_year, to_year, table_name, node=None):
node = self.context.node

node.query(
f"INSERT INTO {table_name} SELECT * FROM s3('https://clickhouse-public-datasets.s3.amazonaws.com/ontime/csv_by_year/{{{from_year}..{to_year}}}.csv.gz', CSVWithNames) SETTINGS max_insert_threads = 20, receive_timeout=600, max_memory_usage=0;",
f"INSERT INTO {table_name} "
f"SELECT * FROM s3('https://clickhouse-public-datasets.s3.amazonaws.com/ontime/csv_by_year/{{{from_year}..{to_year}}}.csv.gz', CSVWithNames) "
"SETTINGS receive_timeout=600, "
"max_insert_threads=10, " # This affects memory more than it affects performance
"max_memory_usage=29500000000;", # Runners have about this much available memory
timeout=1200,
)

Expand Down Expand Up @@ -186,7 +190,7 @@ def run_benchmark(num):
start_year = 2007
end_year = 2012

for retry in retries(timeout=60, delay=0.1):
for retry in retries(timeout=60, delay=1):
with retry:
Step(
name=f"insert data from {start_year} to {end_year} into ontime table",
Expand Down

0 comments on commit 671afe0

Please sign in to comment.