Skip to content

Commit

Permalink
alter stress: try to avoid running out of host disk space
Browse files Browse the repository at this point in the history
  • Loading branch information
strtgbb committed May 22, 2024
1 parent 6b912d6 commit 040ec0c
Show file tree
Hide file tree
Showing 2 changed files with 13 additions and 5 deletions.
8 changes: 4 additions & 4 deletions alter/stress/tests/actions.py
Original file line number Diff line number Diff line change
Expand Up @@ -503,12 +503,12 @@ def clear_random_column(self):
@TestStep
@Retry(timeout=step_retry_timeout, delay=step_retry_delay)
@Name("delete row")
def delete_random_rows(self):
def delete_random_rows(self, table_name=None):
"""Delete a few rows at random."""
table_name = get_random_table_name()
table_name = table_name or get_random_table_name()
node = get_random_node_for_table(table_name=table_name)
column_name = get_random_column_name(node=node, table_name=table_name)
divisor = random.choice([5, 11, 17, 23])
divisor = random.choice([2, 3, 5])
remainder = random.randint(0, divisor - 1)

By(
Expand All @@ -534,7 +534,7 @@ def delete_random_rows_lightweight(self):
table_name = get_random_table_name()
node = get_random_node_for_table(table_name=table_name)
column_name = get_random_column_name(node=node, table_name=table_name)
divisor = random.choice([5, 11, 17, 23])
divisor = random.choice([2, 3, 5])
remainder = random.randint(0, divisor - 1)

with By(f"delete rows from {table_name} with {node.name}"):
Expand Down
10 changes: 9 additions & 1 deletion alter/stress/tests/stress_alter.py
Original file line number Diff line number Diff line change
Expand Up @@ -265,7 +265,7 @@ def alter_combinations(
join()

finally:
with Then("I make sure that the replicas are consistent", flags=TE):
with Finally("I make sure that the replicas are consistent", flags=TE):
if kill_stuck_mutations:
with By("killing any failing mutations"):
for node in self.context.ch_nodes:
Expand All @@ -285,6 +285,14 @@ def alter_combinations(
restore_consistent_structure=enforce_table_structure
)

with And("I make sure that there is still free disk space on the host"):
r = self.context.cluster.command(None, "df -h .")
if "100%" in r.output:
with When("I drop rows to free up space"):
for table_name in self.context.table_names:
delete_random_rows(table_name=table_name)
delete_random_rows(table_name=table_name)

note(f"Average time per test combination {(time.time()-t)/(i+1):.1f}s")

finally:
Expand Down

0 comments on commit 040ec0c

Please sign in to comment.