Skip to content

Commit

Permalink
Merge branch 'ko3n1g/ci/retry-wait' into 'main'
Browse files Browse the repository at this point in the history
revert: Try/catch

See merge request ADLR/megatron-lm!2288
  • Loading branch information
ko3n1g committed Oct 31, 2024
2 parents c3eb3be + 7d43d84 commit d546182
Showing 1 changed file with 1 addition and 13 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -91,19 +91,7 @@ def launch_and_wait_for_completion(
flush=True,
)

n_attempt = 0
while n_attempt < 10:
try:
pipeline.wait(max_wait_time=60 * 60 * 24 * 7)
except requests.exceptions.ConnectionError:
n_attempt += 1
print(f"Connection error, try again (attempt {n_attempt})")
time.sleep(60)
except Exception as e:
raise e

if pipeline.get_status() == PipelineStatus.SUCCESS:
break
pipeline.wait(max_wait_time=60 * 60 * 24 * 7)

print(f"Pipeline terminated; status: {pipeline.get_status()}")
return pipeline
Expand Down

0 comments on commit d546182

Please sign in to comment.