Merge pull request #379 from roboflow/feature/add-gpu-test-machine

Added the github runner instead of the custom runner

.github/workflows/test.nvidia_t4.yml

@@ -6,7 +6,7 @@ on:
 jobs:
   build:
     if: ${{ !github.event.act }}
-    runs-on: [self-hosted, t4-gpu, gcp]
+    runs-on: Roboflow-GPU-VM-Runner
 
     steps:
       - name: Set up QEMU
@@ -25,8 +25,8 @@ jobs:
 
       - name: 🦾 Install dependencies
         run: |
-          python -m pip install --upgrade pip
-          python -m pip install -r requirements/requirements.test.integration.txt
+          python3 -m pip install --upgrade pip
+          python3 -m pip install -r requirements/requirements.test.integration.txt
       - name: 🔨 Build and Push Test Docker - GPU
         run: |
           docker pull roboflow/roboflow-inference-server-gpu:test
@@ -37,7 +37,7 @@ jobs:
           PORT=9101 INFERENCE_SERVER_REPO=roboflow-inference-server-gpu make start_test_docker_gpu
       - name: 🧪 Regression Tests - GPU
         run: |
-          MINIMUM_FPS=25 FUNCTIONAL=true PORT=9101 API_KEY=${{ secrets.API_KEY }} asl_instance_segmentation_API_KEY=${{ secrets.ASL_INSTANCE_SEGMENTATION_API_KEY }} asl_poly_instance_seg_API_KEY=${{ secrets.ASL_POLY_INSTANCE_SEG_API_KEY }} bccd_favz3_API_KEY=${{ secrets.BCCD_FAVZ3_API_KEY }} bccd_i4nym_API_KEY=${{ secrets.BCCD_I4NYM_API_KEY }} cats_and_dogs_smnpl_API_KEY=${{ secrets.CATS_AND_DOGS_SMNPL_API_KEY }} coins_xaz9i_API_KEY=${{ secrets.COINS_XAZ9I_API_KEY }} melee_API_KEY=${{ secrets.MELEE_API_KEY }} yolonas_test_API_KEY=${{ secrets.YOLONAS_TEST_API_KEY }} python -m pytest tests/inference/integration_tests/
+          MINIMUM_FPS=25 FUNCTIONAL=true PORT=9101 API_KEY=${{ secrets.API_KEY }} asl_instance_segmentation_API_KEY=${{ secrets.ASL_INSTANCE_SEGMENTATION_API_KEY }} asl_poly_instance_seg_API_KEY=${{ secrets.ASL_POLY_INSTANCE_SEG_API_KEY }} bccd_favz3_API_KEY=${{ secrets.BCCD_FAVZ3_API_KEY }} bccd_i4nym_API_KEY=${{ secrets.BCCD_I4NYM_API_KEY }} cats_and_dogs_smnpl_API_KEY=${{ secrets.CATS_AND_DOGS_SMNPL_API_KEY }} coins_xaz9i_API_KEY=${{ secrets.COINS_XAZ9I_API_KEY }} melee_API_KEY=${{ secrets.MELEE_API_KEY }} yolonas_test_API_KEY=${{ secrets.YOLONAS_TEST_API_KEY }} python3 -m pytest tests/inference/integration_tests/
       - name: 🧹 Cleanup Test Docker - GPU
         run: make stop_test_docker
         if: success() || failure()
@@ -51,7 +51,7 @@ jobs:
           PORT=9101 INFERENCE_SERVER_REPO=roboflow-inference-server-gpu-parallel make start_test_docker_gpu
       - name: 🧪 Regression Tests - Parallel GPU
         run: |
-          SKIP_VISUALISATION_TESTS=true FUNCTIONAL=true PORT=9101 API_KEY=${{ secrets.API_KEY }} asl_instance_segmentation_API_KEY=${{ secrets.ASL_INSTANCE_SEGMENTATION_API_KEY }} asl_poly_instance_seg_API_KEY=${{ secrets.ASL_POLY_INSTANCE_SEG_API_KEY }} bccd_favz3_API_KEY=${{ secrets.BCCD_FAVZ3_API_KEY }} bccd_i4nym_API_KEY=${{ secrets.BCCD_I4NYM_API_KEY }} cats_and_dogs_smnpl_API_KEY=${{ secrets.CATS_AND_DOGS_SMNPL_API_KEY }} coins_xaz9i_API_KEY=${{ secrets.COINS_XAZ9I_API_KEY }} melee_API_KEY=${{ secrets.MELEE_API_KEY }} yolonas_test_API_KEY=${{ secrets.YOLONAS_TEST_API_KEY }} python -m pytest tests/inference/integration_tests/regression_test.py tests/inference/integration_tests/batch_regression_test.py
+          IS_PARALLEL_SERVER=true SKIP_VISUALISATION_TESTS=true FUNCTIONAL=true PORT=9101 API_KEY=${{ secrets.API_KEY }} asl_instance_segmentation_API_KEY=${{ secrets.ASL_INSTANCE_SEGMENTATION_API_KEY }} asl_poly_instance_seg_API_KEY=${{ secrets.ASL_POLY_INSTANCE_SEG_API_KEY }} bccd_favz3_API_KEY=${{ secrets.BCCD_FAVZ3_API_KEY }} bccd_i4nym_API_KEY=${{ secrets.BCCD_I4NYM_API_KEY }} cats_and_dogs_smnpl_API_KEY=${{ secrets.CATS_AND_DOGS_SMNPL_API_KEY }} coins_xaz9i_API_KEY=${{ secrets.COINS_XAZ9I_API_KEY }} melee_API_KEY=${{ secrets.MELEE_API_KEY }} yolonas_test_API_KEY=${{ secrets.YOLONAS_TEST_API_KEY }} python3 -m pytest tests/inference/integration_tests/regression_test.py tests/inference/integration_tests/batch_regression_test.py
       - name: 🧹 Cleanup Test Docker - Parallel GPU
         run: make stop_test_docker
         if: success() || failure()

tests/inference/integration_tests/batch_regression_test.py

@@ -14,7 +14,6 @@
 PIXEL_TOLERANCE = 2
 CONFIDENCE_TOLERANCE = 0.005
 TIME_TOLERANCE = 0.75
-
 api_key = os.environ.get("API_KEY")
 port = os.environ.get("PORT", 9001)
 base_url = os.environ.get("BASE_URL", "http://localhost")
@@ -198,7 +197,10 @@ def infer_request_with_base64_image(
 ]
 
 DETECTION_TEST_PARAMS = []
+is_parallel_server = bool_env(os.getenv("IS_PARALLEL_SERVER", False))
 for test in TESTS:
+    if test["description"] == "YOLACT Instance Segmentation" and is_parallel_server:
+        continue # Skip YOLACT tests for parallel server
     if "expected_response" in test:
         for res_func in INFER_RESPONSE_FUNCTIONS:
             DETECTION_TEST_PARAMS.append((test, res_func))

tests/inference/integration_tests/regression_test.py

@@ -15,7 +15,6 @@
 PIXEL_TOLERANCE = 2
 CONFIDENCE_TOLERANCE = 0.005
 TIME_TOLERANCE = 0.75
-
 api_key = os.environ.get("API_KEY")
 port = os.environ.get("PORT", 9001)
 base_url = os.environ.get("BASE_URL", "http://localhost")
@@ -404,8 +403,11 @@ def compare_detection_response(
 ]
 
 SKIP_YOLOV8_TEST = bool_env(os.getenv("SKIP_YOLOV8_TEST", False))
+is_parallel_server = bool_env(os.getenv("IS_PARALLEL_SERVER", False))
 DETECTION_TEST_PARAMS = []
 for test in TESTS:
+    if test["description"] == "YOLACT Instance Segmentation" and is_parallel_server:
+        continue # Skip YOLACT tests for parallel server
     if "expected_response" in test:
         if not SKIP_YOLOV8_TEST or "YOLOv8" not in test["description"]:
             for res_func in INFER_RESPONSE_FUNCTIONS: