[release] Allow release test to specify num_retries (ray-project#57578)

So it can be configured for expensive tests to not retry

---------

Signed-off-by: kevin <[email protected]>
Co-authored-by: Lonnie Liu <[email protected]>

Author: 2 people

release/ray_release/buildkite/step.py

@@ -141,6 +141,10 @@ def get_step(
     if smoke_test:
         cmd += ["--smoke-test"]
 
+    num_retries = test.get("run", {}).get("num_retries")
+    if num_retries:
+        step["retry"]["automatic"][0]["limit"] = num_retries
+
     step["plugins"][0][DOCKER_PLUGIN_KEY]["command"] = cmd
 
     env_to_use = test.get("env", DEFAULT_ENVIRONMENT)

release/ray_release/schema.json

@@ -159,6 +159,9 @@
 				},
 				"artifact_path": {
 					"type": "string"
+				},
+				"num_retries": {
+					"type": "integer"
 				}
 			},
 			"required": [

release/ray_release/tests/test_step.py

@@ -21,6 +21,11 @@ def _stub_test(val: dict) -> Test:
             "cluster": {
                 "byod": {},
             },
+            "run": {
+                "script": "python test.py",
+                "timeout": 100,
+                "num_retries": 3,
+            },
         }
     )
     test.update(val)
@@ -32,6 +37,7 @@ def test_get_step(mock):
     with patch.dict("os.environ", {"RAYCI_BUILD_ID": "a1b2c3d4"}):
         step = get_step(_stub_test({}), run_id=2)
     assert step["label"] == "test (None) (2)"
+    assert step["retry"]["automatic"][0]["limit"] == 3
 
 
 @patch("ray_release.test.Test.update_from_s3", return_value=None)

release/release_tests.yaml

@@ -91,6 +91,7 @@
 
   run:
     timeout: 1800
+    num_retries: 3
     script: python hello_world.py
 
   variations: