diff --git a/FINALS_README.md b/FINALS_README.md index b453f41..c0c8d39 100644 --- a/FINALS_README.md +++ b/FINALS_README.md @@ -6,7 +6,7 @@ 1. [Additional Directories](#additional-directories) 2. [Simulator](#simulator) 2. [Instructions](#instructions) - 1. [Local Developement Environment Setup](#local-developement-environment-setup) + 1. [Local Development Environment Setup](#local-development-environment-setup) 2. [Artifacts Submission](#artifacts-submission) 3. [Docker + Artifact Registry Recap](#docker-and-artifact-registry-recap) 4. [Docker Compose](#docker-compose) @@ -44,7 +44,7 @@ The simulator is written in JavaScript and uses `3js` to render the 3D environme ## Instructions -### Local Developement Environment Setup +### Local Development Environment Setup From here on, you will need to have a local development environment set up for your own testing. This local development environment should have: @@ -105,7 +105,7 @@ When your model is pushed successfully, you should be able to see it under your ### Docker Compose -To run all the containers, we will be using [Docker Compose](https://docs.docker.com/compose/), a tool that lets you easily configure and run multi-container applications. +To run all the containers, we will be using [Docker Compose](https://docs.docker.com/compose/), a tool that lets you easily configure and run multi-container applications. Note that you will likely have to install it if it is not already installed (e.g. it may not be installed on the GCP instance); see [the Docker Compose plugin installation instructions](https://docs.docker.com/compose/install/#scenario-two-install-the-compose-plugin). ```bash # start all the services @@ -181,3 +181,32 @@ pip install robomaster ``` This will create a new Python 3.8 environment `robo` and configure it to use `x86` packages using Rosetta instead of native ones for `arm64`. + +### Loading Models Offline + +Some HuggingFace models, such as the DETR model referenced in the training curriculum, encounter some issues when trying to load from a finetuned model offline, as they rely on some backbone weights from HuggingFace Hub not stored in the model when downloaded using the `save_pretrained()` function. + +One possible solution to this problem is to run a script to load the models once when building the Dockerfile. This will cache whatever backbone weights are necessary, so subsequent calls of the model will just use the cached version instead of trying to access the HuggingFace Hub (which won't be possible without internet access). + +Here's an example of what such a script, `save_cache.py`, would look like: + +```python +from transformers import ( + AutoImageProcessor, + AutoModelForObjectDetection +) + +detr_checkpoint = "models/detr_model.pth" + +detr_model = AutoModelForObjectDetection.from_pretrained(detr_checkpoint) +detr_processor = AutoImageProcessor.from_pretrained(detr_checkpoint) +``` + +Then in your Dockerfile, add these lines after `pip install -r requirements.txt` and copying over necessary files (especially the model file). + +```dockerfile +COPY save_cache.py save_cache.py +RUN python3 save_cache.py +``` + +This will copy and run `save_cache.py`, which will save the model backbone weights to the `.cache` directory on the image, allowing it to be loaded offline by HuggingFace Transformers later on. diff --git a/main/requirements.txt b/main/requirements.txt index abb2632..32c50eb 100644 --- a/main/requirements.txt +++ b/main/requirements.txt @@ -1,3 +1,3 @@ websockets -requests -asyncio \ No newline at end of file +asyncio +httpx diff --git a/main/src/auto_manager.py b/main/src/auto_manager.py index ee59a8e..5738bad 100644 --- a/main/src/auto_manager.py +++ b/main/src/auto_manager.py @@ -1,7 +1,6 @@ from random import randint from typing import Dict, List from finals_manager import FinalsManager -import requests class AutoManager(FinalsManager): @@ -9,11 +8,11 @@ def __init__(self, local_ip: str): super().__init__() self.local_ip = local_ip - def run_asr(self, audio_bytes: bytes) -> str: + async def run_asr(self, audio_bytes: bytes) -> str: print("Running ASR") return "asr" - def run_nlp(self, transcript: str) -> Dict[str, str]: + async def run_nlp(self, transcript: str) -> Dict[str, str]: print("Running NLP") return { "target": "airplane", @@ -21,21 +20,21 @@ def run_nlp(self, transcript: str) -> Dict[str, str]: "tool": "surface-to-air missiles", } - def run_vlm(self, image_bytes: bytes, caption: str) -> List[int]: + async def run_vlm(self, image_bytes: bytes, caption: str) -> List[int]: print("Running VLM") return [0, 0, 0, 0] - def send_heading(self, heading: str) -> bytes: + async def send_heading(self, heading: str) -> bytes: assert heading.isdigit(), "The heading string contains non-digit characters" print(f"Sending cannon heading {heading}") - results = requests.post( + results = await self.async_post( f"http://{self.local_ip}:5003/send_heading", json={"heading": heading} ) # snapshot of image return results.content - def reset_cannon(self): + async def reset_cannon(self): print("Resetting cannon to original position") - results = requests.post(f"http://{self.local_ip}:5003/reset_cannon") + results = await self.async_post(f"http://{self.local_ip}:5003/reset_cannon") print(results.text) return results.json() diff --git a/main/src/finals_manager.py b/main/src/finals_manager.py index f463fc1..0fd8f48 100644 --- a/main/src/finals_manager.py +++ b/main/src/finals_manager.py @@ -1,33 +1,42 @@ import json import websockets from abc import ABC, abstractmethod -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional +import httpx class FinalsManager(ABC): def __init__(self): print("initializing participant finals server manager") + self.client = httpx.AsyncClient() - def send_result( + async def exit(self): + await self.client.aclose() + + async def async_post(self, endpoint: str, json: Optional[Dict] = None): + return await self.client.post(endpoint, json=json) + + async def send_result( self, websocket: websockets.WebSocketClientProtocol, data: Dict[str, Any] ): - return websocket.send(json.dumps(data)) + return await websocket.send(json.dumps(data)) @abstractmethod - def run_asr(self, audio_bytes: bytes) -> str: + async def run_asr(self, audio_bytes: bytes) -> str: raise NotImplemented @abstractmethod - def run_nlp(self, transcript: str) -> Dict[str, str]: + async def run_nlp(self, transcript: str) -> Dict[str, str]: raise NotImplemented @abstractmethod - def send_heading(self, heading: str) -> bytes: + async def send_heading(self, heading: str) -> bytes: raise NotImplemented @abstractmethod - def reset_cannon(self) -> None: + async def reset_cannon(self) -> None: raise NotImplemented - def run_vlm(self, image_bytes: bytes, caption: str) -> List[int]: + @abstractmethod + async def run_vlm(self, image_bytes: bytes, caption: str) -> List[int]: raise NotImplemented diff --git a/main/src/mock_manager.py b/main/src/mock_manager.py index f3a992d..6691281 100644 --- a/main/src/mock_manager.py +++ b/main/src/mock_manager.py @@ -1,4 +1,4 @@ -from time import sleep +from asyncio import sleep from typing import Dict, List from finals_manager import FinalsManager @@ -7,11 +7,11 @@ class MockManager(FinalsManager): def __init__(self): super().__init__() - def run_asr(self, audio_bytes: bytes) -> str: + async def run_asr(self, audio_bytes: bytes) -> str: print("Running ASR") return "asr" - def run_nlp(self, transcript: str) -> Dict[str, str]: + async def run_nlp(self, transcript: str) -> Dict[str, str]: print("Running NLP") return { "target": "airplane", @@ -19,15 +19,15 @@ def run_nlp(self, transcript: str) -> Dict[str, str]: "tool": "surface-to-air missiles", } - def run_vlm(self, image_bytes: bytes, caption: str) -> List[int]: + async def run_vlm(self, image_bytes: bytes, caption: str) -> List[int]: print("Running VLM") return [0, 0, 0, 0] - def send_heading(self, heading: str) -> bytes: + async def send_heading(self, heading: str) -> bytes: print(f"Sending cannon heading {heading}") - sleep(1) + await sleep(1) return bytes() - def reset_cannon(self) -> None: + async def reset_cannon(self) -> None: print("Resetting cannon to original position") return {} diff --git a/main/src/models_manager.py b/main/src/models_manager.py index 0266211..b8844f8 100644 --- a/main/src/models_manager.py +++ b/main/src/models_manager.py @@ -1,7 +1,6 @@ from base64 import b64encode from typing import Dict, List from finals_manager import FinalsManager -import requests class ModelsManager(FinalsManager): @@ -9,40 +8,40 @@ def __init__(self, local_ip: str): super().__init__() self.local_ip = local_ip - def run_asr(self, audio_bytes: bytes) -> str: + async def run_asr(self, audio_bytes: bytes) -> str: print("Running ASR") audio_str = b64encode(audio_bytes).decode("ascii") - results = requests.post( + results = await self.async_post( f"http://{self.local_ip}:5001/stt", json={"instances": [{"b64": audio_str}]} ) return results.json()["predictions"][0] - def run_nlp(self, transcript: str) -> Dict[str, str]: + async def run_nlp(self, transcript: str) -> Dict[str, str]: print("Running NLP") - results = requests.post( + results = await self.async_post( f"http://{self.local_ip}:5002/extract", json={"instances": [{"transcript": transcript}]}, ) return results.json()["predictions"][0] - def send_heading(self, heading: str) -> bytes: + async def send_heading(self, heading: str) -> bytes: assert heading.isdigit(), "The heading string contains non-digit characters" print(f"Sending cannon heading {heading}") - results = requests.post( + results = await self.async_post( f"http://{self.local_ip}:5003/send_heading", json={"heading": heading} ) # snapshot of image return results.content - def reset_cannon(self): + async def reset_cannon(self): print("Resetting cannon to original position") - results = requests.post(f"http://{self.local_ip}:5003/reset_cannon") + results = await self.async_post(f"http://{self.local_ip}:5003/reset_cannon") return results.json() - def run_vlm(self, image_bytes: bytes, caption: str) -> List[int]: + async def run_vlm(self, image_bytes: bytes, caption: str) -> List[int]: print("Running VLM") image_str = b64encode(image_bytes).decode("ascii") - results = requests.post( + results = await self.async_post( f"http://{self.local_ip}:5004/identify", json={"instances": [{"b64": image_str, "caption": caption}]}, ) diff --git a/main/src/participant_server.py b/main/src/participant_server.py index 73453c9..beea3c7 100644 --- a/main/src/participant_server.py +++ b/main/src/participant_server.py @@ -29,7 +29,7 @@ async def server(): print(f"connecting to competition server {SERVER_IP} at port {SERVER_PORT}") try: while True: - # should be receiving either audio bytes for asr, or "done!" message + # should be receiving either audio bytes for asr, or done/healthcheck json socket_input = await websocket.recv() if type(socket_input) is str: # handle either done or healthcheck @@ -42,28 +42,28 @@ async def server(): continue print(f"run {index}") # ASR - transcript = manager.run_asr(socket_input) + transcript = await manager.run_asr(socket_input) print(transcript) # NLP - qa_ans = manager.run_nlp(transcript) + qa_ans = await manager.run_nlp(transcript) print(qa_ans) query = qa_ans["target"] # autonomy try: - image = manager.send_heading(qa_ans["heading"]) + image = await manager.send_heading(qa_ans["heading"]) except AssertionError as e: # if heading is wrong, get image of scene at default heading 000 print(e) - image = manager.send_heading("000") + image = await manager.send_heading("000") # VLM - vlm_results = manager.run_vlm(image, query) + vlm_results = await manager.run_vlm(image, query) print(vlm_results) # submit results and reset await manager.send_result( websocket, {"asr": transcript, "nlp": qa_ans, "vlm": vlm_results}, ) - manager.reset_cannon() + await manager.reset_cannon() print(f"done run {index}") index += 1 except websockets.ConnectionClosed: diff --git a/simulator/data/audio/audio_0_target_0.wav b/simulator/data/audio/audio_0_target_0.wav index 7223924..9556b2b 100644 Binary files a/simulator/data/audio/audio_0_target_0.wav and b/simulator/data/audio/audio_0_target_0.wav differ diff --git a/simulator/data/audio/audio_1_target_0.wav b/simulator/data/audio/audio_1_target_0.wav index 6c6869a..a333700 100644 Binary files a/simulator/data/audio/audio_1_target_0.wav and b/simulator/data/audio/audio_1_target_0.wav differ diff --git a/simulator/data/audio/audio_2_target_0.wav b/simulator/data/audio/audio_2_target_0.wav index 86ba5d8..6dd7f06 100644 Binary files a/simulator/data/audio/audio_2_target_0.wav and b/simulator/data/audio/audio_2_target_0.wav differ diff --git a/simulator/data/audio/audio_3_target_0.wav b/simulator/data/audio/audio_3_target_0.wav index e5c73e7..b1e07ee 100644 Binary files a/simulator/data/audio/audio_3_target_0.wav and b/simulator/data/audio/audio_3_target_0.wav differ diff --git a/simulator/data/audio/audio_4_target_0.wav b/simulator/data/audio/audio_4_target_0.wav index 2da1757..5c60d91 100644 Binary files a/simulator/data/audio/audio_4_target_0.wav and b/simulator/data/audio/audio_4_target_0.wav differ diff --git a/simulator/data/cubemaps/cubemap_0_negx.jpg b/simulator/data/cubemaps/cubemap_0_negx.jpg index 37c3cb2..c9e8f59 100644 Binary files a/simulator/data/cubemaps/cubemap_0_negx.jpg and b/simulator/data/cubemaps/cubemap_0_negx.jpg differ diff --git a/simulator/data/cubemaps/cubemap_0_negy.jpg b/simulator/data/cubemaps/cubemap_0_negy.jpg index 37d1a52..d3db2f8 100644 Binary files a/simulator/data/cubemaps/cubemap_0_negy.jpg and b/simulator/data/cubemaps/cubemap_0_negy.jpg differ diff --git a/simulator/data/cubemaps/cubemap_0_negz.jpg b/simulator/data/cubemaps/cubemap_0_negz.jpg index fd4cc66..60fd5fc 100644 Binary files a/simulator/data/cubemaps/cubemap_0_negz.jpg and b/simulator/data/cubemaps/cubemap_0_negz.jpg differ diff --git a/simulator/data/cubemaps/cubemap_0_posx.jpg b/simulator/data/cubemaps/cubemap_0_posx.jpg index b36819e..fe5c518 100644 Binary files a/simulator/data/cubemaps/cubemap_0_posx.jpg and b/simulator/data/cubemaps/cubemap_0_posx.jpg differ diff --git a/simulator/data/cubemaps/cubemap_0_posy.jpg b/simulator/data/cubemaps/cubemap_0_posy.jpg index 76e2c2e..4af3e2d 100644 Binary files a/simulator/data/cubemaps/cubemap_0_posy.jpg and b/simulator/data/cubemaps/cubemap_0_posy.jpg differ diff --git a/simulator/data/cubemaps/cubemap_0_posz.jpg b/simulator/data/cubemaps/cubemap_0_posz.jpg index 9b51d44..b373851 100644 Binary files a/simulator/data/cubemaps/cubemap_0_posz.jpg and b/simulator/data/cubemaps/cubemap_0_posz.jpg differ diff --git a/simulator/data/cubemaps/cubemap_1_negx.jpg b/simulator/data/cubemaps/cubemap_1_negx.jpg index 1e93869..0bc199c 100644 Binary files a/simulator/data/cubemaps/cubemap_1_negx.jpg and b/simulator/data/cubemaps/cubemap_1_negx.jpg differ diff --git a/simulator/data/cubemaps/cubemap_1_negy.jpg b/simulator/data/cubemaps/cubemap_1_negy.jpg index a41d558..71aaa52 100644 Binary files a/simulator/data/cubemaps/cubemap_1_negy.jpg and b/simulator/data/cubemaps/cubemap_1_negy.jpg differ diff --git a/simulator/data/cubemaps/cubemap_1_negz.jpg b/simulator/data/cubemaps/cubemap_1_negz.jpg index 26a58fd..48b3da4 100644 Binary files a/simulator/data/cubemaps/cubemap_1_negz.jpg and b/simulator/data/cubemaps/cubemap_1_negz.jpg differ diff --git a/simulator/data/cubemaps/cubemap_1_posx.jpg b/simulator/data/cubemaps/cubemap_1_posx.jpg index ddcb288..ec15369 100644 Binary files a/simulator/data/cubemaps/cubemap_1_posx.jpg and b/simulator/data/cubemaps/cubemap_1_posx.jpg differ diff --git a/simulator/data/cubemaps/cubemap_1_posy.jpg b/simulator/data/cubemaps/cubemap_1_posy.jpg index a1910be..a96966f 100644 Binary files a/simulator/data/cubemaps/cubemap_1_posy.jpg and b/simulator/data/cubemaps/cubemap_1_posy.jpg differ diff --git a/simulator/data/cubemaps/cubemap_1_posz.jpg b/simulator/data/cubemaps/cubemap_1_posz.jpg index 50f7e2c..ae6a8bd 100644 Binary files a/simulator/data/cubemaps/cubemap_1_posz.jpg and b/simulator/data/cubemaps/cubemap_1_posz.jpg differ diff --git a/simulator/data/cubemaps/cubemap_2_negx.jpg b/simulator/data/cubemaps/cubemap_2_negx.jpg index 901bae9..850eab0 100644 Binary files a/simulator/data/cubemaps/cubemap_2_negx.jpg and b/simulator/data/cubemaps/cubemap_2_negx.jpg differ diff --git a/simulator/data/cubemaps/cubemap_2_negy.jpg b/simulator/data/cubemaps/cubemap_2_negy.jpg index cc7abf4..509a1dd 100644 Binary files a/simulator/data/cubemaps/cubemap_2_negy.jpg and b/simulator/data/cubemaps/cubemap_2_negy.jpg differ diff --git a/simulator/data/cubemaps/cubemap_2_negz.jpg b/simulator/data/cubemaps/cubemap_2_negz.jpg index b783cd1..5246822 100644 Binary files a/simulator/data/cubemaps/cubemap_2_negz.jpg and b/simulator/data/cubemaps/cubemap_2_negz.jpg differ diff --git a/simulator/data/cubemaps/cubemap_2_posx.jpg b/simulator/data/cubemaps/cubemap_2_posx.jpg index 014c3ce..4ab7f82 100644 Binary files a/simulator/data/cubemaps/cubemap_2_posx.jpg and b/simulator/data/cubemaps/cubemap_2_posx.jpg differ diff --git a/simulator/data/cubemaps/cubemap_2_posy.jpg b/simulator/data/cubemaps/cubemap_2_posy.jpg index 00f91d1..7684782 100644 Binary files a/simulator/data/cubemaps/cubemap_2_posy.jpg and b/simulator/data/cubemaps/cubemap_2_posy.jpg differ diff --git a/simulator/data/cubemaps/cubemap_2_posz.jpg b/simulator/data/cubemaps/cubemap_2_posz.jpg index 424ef33..e08301c 100644 Binary files a/simulator/data/cubemaps/cubemap_2_posz.jpg and b/simulator/data/cubemaps/cubemap_2_posz.jpg differ diff --git a/simulator/data/cubemaps/cubemap_3_negx.jpg b/simulator/data/cubemaps/cubemap_3_negx.jpg index aa4af06..eb2500a 100644 Binary files a/simulator/data/cubemaps/cubemap_3_negx.jpg and b/simulator/data/cubemaps/cubemap_3_negx.jpg differ diff --git a/simulator/data/cubemaps/cubemap_3_negy.jpg b/simulator/data/cubemaps/cubemap_3_negy.jpg index a4c5892..20d1db4 100644 Binary files a/simulator/data/cubemaps/cubemap_3_negy.jpg and b/simulator/data/cubemaps/cubemap_3_negy.jpg differ diff --git a/simulator/data/cubemaps/cubemap_3_negz.jpg b/simulator/data/cubemaps/cubemap_3_negz.jpg index 4350a8d..57673d3 100644 Binary files a/simulator/data/cubemaps/cubemap_3_negz.jpg and b/simulator/data/cubemaps/cubemap_3_negz.jpg differ diff --git a/simulator/data/cubemaps/cubemap_3_posx.jpg b/simulator/data/cubemaps/cubemap_3_posx.jpg index 38ea36b..b73330c 100644 Binary files a/simulator/data/cubemaps/cubemap_3_posx.jpg and b/simulator/data/cubemaps/cubemap_3_posx.jpg differ diff --git a/simulator/data/cubemaps/cubemap_3_posy.jpg b/simulator/data/cubemaps/cubemap_3_posy.jpg index 9c1d812..3395106 100644 Binary files a/simulator/data/cubemaps/cubemap_3_posy.jpg and b/simulator/data/cubemaps/cubemap_3_posy.jpg differ diff --git a/simulator/data/cubemaps/cubemap_3_posz.jpg b/simulator/data/cubemaps/cubemap_3_posz.jpg index ea4a1eb..3fc42a3 100644 Binary files a/simulator/data/cubemaps/cubemap_3_posz.jpg and b/simulator/data/cubemaps/cubemap_3_posz.jpg differ diff --git a/simulator/data/cubemaps/cubemap_4_negx.jpg b/simulator/data/cubemaps/cubemap_4_negx.jpg index 8aeece3..5fd4c48 100644 Binary files a/simulator/data/cubemaps/cubemap_4_negx.jpg and b/simulator/data/cubemaps/cubemap_4_negx.jpg differ diff --git a/simulator/data/cubemaps/cubemap_4_negy.jpg b/simulator/data/cubemaps/cubemap_4_negy.jpg index 1e00277..ab76b42 100644 Binary files a/simulator/data/cubemaps/cubemap_4_negy.jpg and b/simulator/data/cubemaps/cubemap_4_negy.jpg differ diff --git a/simulator/data/cubemaps/cubemap_4_negz.jpg b/simulator/data/cubemaps/cubemap_4_negz.jpg index 5d29350..c07f34e 100644 Binary files a/simulator/data/cubemaps/cubemap_4_negz.jpg and b/simulator/data/cubemaps/cubemap_4_negz.jpg differ diff --git a/simulator/data/cubemaps/cubemap_4_posx.jpg b/simulator/data/cubemaps/cubemap_4_posx.jpg index 7e8f789..2cfa9a2 100644 Binary files a/simulator/data/cubemaps/cubemap_4_posx.jpg and b/simulator/data/cubemaps/cubemap_4_posx.jpg differ diff --git a/simulator/data/cubemaps/cubemap_4_posy.jpg b/simulator/data/cubemaps/cubemap_4_posy.jpg index 0406ff1..ea9d09f 100644 Binary files a/simulator/data/cubemaps/cubemap_4_posy.jpg and b/simulator/data/cubemaps/cubemap_4_posy.jpg differ diff --git a/simulator/data/cubemaps/cubemap_4_posz.jpg b/simulator/data/cubemaps/cubemap_4_posz.jpg index 11abe18..c8226bd 100644 Binary files a/simulator/data/cubemaps/cubemap_4_posz.jpg and b/simulator/data/cubemaps/cubemap_4_posz.jpg differ diff --git a/simulator/data/data.json b/simulator/data/data.json index 14ed4be..04da509 100644 --- a/simulator/data/data.json +++ b/simulator/data/data.json @@ -1,6 +1,6 @@ [ { - "audio": "simulator/data/audio/audio_0_target_0.wav", + "audio": "audio_0_target_0.wav", "files": [ "cubemap_0_posx.jpg", "cubemap_0_negx.jpg", @@ -11,104 +11,114 @@ ], "corners": [ { - "x": -50, - "y": 45.1171875, - "z": 12.6953125 + "x": -21.533203125, + "y": 29.638671875, + "z": -50 }, { - "x": -50, - "y": 44.82421875, - "z": 20.21484375 + "x": -24.267578125, + "y": 29.443359375, + "z": -50 }, { - "x": -50, - "y": 44.775390625, - "z": 20.263671875 + "x": -26.7578125, + "y": 28.90625, + "z": -50 }, { - "x": -50, - "y": 42.96875, - "z": 20.80078125 + "x": -26.904296875, + "y": 28.857421875, + "z": -50 }, { - "x": -50, - "y": 42.87109375, - "z": 20.703125 + "x": -27.099609375, + "y": 28.759765625, + "z": -50 }, { - "x": -50, - "y": 42.28515625, - "z": 19.873046875 + "x": -27.1484375, + "y": 28.7109375, + "z": -50 }, { - "x": -50, - "y": 42.236328125, - "z": 19.580078125 + "x": -28.564453125, + "y": 26.318359375, + "z": -50 }, { - "x": -50, - "y": 42.08984375, - "z": 18.65234375 + "x": -28.564453125, + "y": 26.26953125, + "z": -50 }, { - "x": -50, - "y": 41.943359375, - "z": 15.478515625 + "x": -28.466796875, + "y": 25.68359375, + "z": -50 }, { - "x": -50, - "y": 41.943359375, - "z": 15.33203125 + "x": -28.41796875, + "y": 25.68359375, + "z": -50 }, { - "x": -50, - "y": 41.9921875, - "z": 13.8671875 + "x": -23.583984375, + "y": 25.732421875, + "z": -50 }, { - "x": -50, - "y": 42.1875, - "z": 13.623046875 + "x": -22.900390625, + "y": 25.87890625, + "z": -50 }, { - "x": -50, - "y": 44.384765625, - "z": 11.669921875 + "x": -20.1171875, + "y": 27.587890625, + "z": -50 }, { - "x": -50, - "y": 44.482421875, - "z": 11.62109375 + "x": -18.896484375, + "y": 28.564453125, + "z": -50 }, { - "x": -50, - "y": 44.775390625, - "z": 11.62109375 + "x": -18.84765625, + "y": 28.61328125, + "z": -50 }, { - "x": -50, - "y": 44.82421875, - "z": 11.669921875 + "x": -18.84765625, + "y": 28.662109375, + "z": -50 + }, + { + "x": -18.896484375, + "y": 29.00390625, + "z": -50 }, { - "x": -50, - "y": 45.068359375, - "z": 12.109375 + "x": -19.287109375, + "y": 29.19921875, + "z": -50 + }, + { + "x": -19.53125, + "y": 29.296875, + "z": -50 }, { - "x": -50, - "y": 45.1171875, - "z": 12.353515625 + "x": -21.337890625, + "y": 29.638671875, + "z": -50 } ], "truth": { - "heading": "090", - "tool": "EMP", - "target": "white and blue commercial aircraft" + "heading": "350", + "tool": "anti-air artillery", + "target": "blue and grey camouflage fighter jet" } }, { - "audio": "simulator/data/audio/audio_1_target_0.wav", + "audio": "audio_1_target_0.wav", "files": [ "cubemap_1_posx.jpg", "cubemap_1_negx.jpg", @@ -119,94 +129,89 @@ ], "corners": [ { - "x": -50, - "y": 15.72265625, - "z": -21.19140625 - }, - { - "x": -50, - "y": 18.505859375, - "z": -18.505859375 + "x": 24.658203125, + "y": 27.099609375, + "z": 50 }, { - "x": -50, - "y": 18.65234375, - "z": -18.212890625 + "x": 22.021484375, + "y": 28.271484375, + "z": 50 }, { - "x": -50, - "y": 18.701171875, - "z": -18.06640625 + "x": 21.044921875, + "y": 28.90625, + "z": 50 }, { - "x": -50, - "y": 18.701171875, - "z": -17.96875 + "x": 20.947265625, + "y": 29.00390625, + "z": 50 }, { - "x": -50, - "y": 18.65234375, - "z": -16.455078125 + "x": 20.947265625, + "y": 29.150390625, + "z": 50 }, { - "x": -50, - "y": 18.603515625, - "z": -16.357421875 + "x": 21.044921875, + "y": 29.248046875, + "z": 50 }, { - "x": -50, - "y": 18.5546875, - "z": -16.30859375 + "x": 21.240234375, + "y": 29.345703125, + "z": 50 }, { - "x": -50, - "y": 16.455078125, - "z": -15.8203125 + "x": 21.435546875, + "y": 29.39453125, + "z": 50 }, { - "x": -50, - "y": 15.185546875, - "z": -16.9921875 + "x": 27.5390625, + "y": 30.46875, + "z": 50 }, { - "x": -50, - "y": 15.0390625, - "z": -17.236328125 + "x": 27.734375, + "y": 30.46875, + "z": 50 }, { - "x": -50, - "y": 14.794921875, - "z": -17.67578125 + "x": 28.076171875, + "y": 29.052734375, + "z": 50 }, { - "x": -50, - "y": 14.794921875, - "z": -17.7734375 + "x": 28.02734375, + "y": 28.955078125, + "z": 50 }, { - "x": -50, - "y": 15.478515625, - "z": -20.849609375 + "x": 27.83203125, + "y": 28.61328125, + "z": 50 }, { - "x": -50, - "y": 15.576171875, - "z": -21.142578125 + "x": 27.5390625, + "y": 28.22265625, + "z": 50 }, { - "x": -50, - "y": 15.673828125, - "z": -21.19140625 + "x": 26.953125, + "y": 27.978515625, + "z": 50 } ], "truth": { - "heading": "090", - "tool": "interceptor jets", - "target": "blue and yellow fighter jet" + "heading": "185", + "tool": "surface-to-air missiles", + "target": "white and blue commercial aircraft" } }, { - "audio": "simulator/data/audio/audio_2_target_0.wav", + "audio": "audio_2_target_0.wav", "files": [ "cubemap_2_posx.jpg", "cubemap_2_negx.jpg", @@ -217,89 +222,84 @@ ], "corners": [ { - "x": 50, - "y": 25.830078125, - "z": -23.681640625 - }, - { - "x": 50, - "y": 25.634765625, - "z": -23.876953125 + "x": -8.49609375, + "y": 26.904296875, + "z": 50 }, { - "x": 50, - "y": 25.48828125, - "z": -23.73046875 + "x": -5.2734375, + "y": 26.171875, + "z": 50 }, { - "x": 50, - "y": 22.36328125, - "z": -18.84765625 + "x": -4.98046875, + "y": 26.07421875, + "z": 50 }, { - "x": 50, - "y": 22.36328125, - "z": -18.65234375 + "x": -4.638671875, + "y": 25.927734375, + "z": 50 }, { - "x": 50, - "y": 22.412109375, - "z": -16.11328125 + "x": -4.443359375, + "y": 25.830078125, + "z": 50 }, { - "x": 50, - "y": 22.705078125, - "z": -14.501953125 + "x": -4.296875, + "y": 25.732421875, + "z": 50 }, { - "x": 50, - "y": 22.75390625, - "z": -14.404296875 + "x": -4.296875, + "y": 25.68359375, + "z": 50 }, { - "x": 50, - "y": 22.900390625, - "z": -14.2578125 + "x": -4.58984375, + "y": 25.537109375, + "z": 50 }, { - "x": 50, - "y": 26.953125, - "z": -10.3515625 + "x": -8.740234375, + "y": 23.779296875, + "z": 50 }, { - "x": 50, - "y": 27.05078125, - "z": -10.3515625 + "x": -9.033203125, + "y": 23.92578125, + "z": 50 }, { - "x": 50, - "y": 27.197265625, - "z": -10.498046875 + "x": -11.767578125, + "y": 25.390625, + "z": 50 }, { - "x": 50, - "y": 27.587890625, - "z": -14.16015625 + "x": -11.81640625, + "y": 25.537109375, + "z": 50 }, { - "x": 50, - "y": 27.587890625, - "z": -14.404296875 + "x": -11.572265625, + "y": 26.318359375, + "z": 50 }, { - "x": 50, - "y": 27.5390625, - "z": -14.697265625 + "x": -8.544921875, + "y": 26.904296875, + "z": 50 } ], "truth": { - "heading": "270", - "tool": "surface-to-air missiles", - "target": "green and brown camouflage helicopter" + "heading": "185", + "tool": "interceptor jets", + "target": "white, blue, and red commercial aircraft" } }, { - "audio": "simulator/data/audio/audio_3_target_0.wav", + "audio": "audio_3_target_0.wav", "files": [ "cubemap_3_posx.jpg", "cubemap_3_negx.jpg", @@ -310,104 +310,84 @@ ], "corners": [ { - "x": 21.97265625, - "y": 26.07421875, - "z": -50 - }, - { - "x": 16.11328125, - "y": 25.341796875, - "z": -50 - }, - { - "x": 16.11328125, - "y": 25.09765625, - "z": -50 - }, - { - "x": 16.162109375, - "y": 23.53515625, - "z": -50 - }, - { - "x": 16.259765625, - "y": 21.337890625, - "z": -50 + "x": -27.294921875, + "y": 20.849609375, + "z": 50 }, { - "x": 16.30859375, - "y": 21.240234375, - "z": -50 + "x": -23.6328125, + "y": 18.994140625, + "z": 50 }, { - "x": 16.455078125, - "y": 21.142578125, - "z": -50 + "x": -23.583984375, + "y": 18.75, + "z": 50 }, { - "x": 16.796875, - "y": 20.947265625, - "z": -50 + "x": -23.583984375, + "y": 18.5546875, + "z": 50 }, { - "x": 17.08984375, - "y": 20.8984375, - "z": -50 + "x": -23.6328125, + "y": 18.505859375, + "z": 50 }, { - "x": 17.28515625, - "y": 20.8984375, - "z": -50 + "x": -25.09765625, + "y": 17.919921875, + "z": 50 }, { - "x": 22.412109375, - "y": 21.142578125, - "z": -50 + "x": -25.244140625, + "y": 17.919921875, + "z": 50 }, { - "x": 25.48828125, - "y": 22.412109375, - "z": -50 + "x": -27.099609375, + "y": 18.1640625, + "z": 50 }, { - "x": 25.5859375, - "y": 22.509765625, - "z": -50 + "x": -29.248046875, + "y": 19.189453125, + "z": 50 }, { - "x": 26.3671875, - "y": 23.4375, - "z": -50 + "x": -29.345703125, + "y": 19.23828125, + "z": 50 }, { - "x": 26.46484375, - "y": 25.439453125, - "z": -50 + "x": -29.443359375, + "y": 19.3359375, + "z": 50 }, { - "x": 26.46484375, - "y": 25.68359375, - "z": -50 + "x": -29.4921875, + "y": 19.482421875, + "z": 50 }, { - "x": 26.416015625, - "y": 25.732421875, - "z": -50 + "x": -29.4921875, + "y": 19.7265625, + "z": 50 }, { - "x": 22.412109375, - "y": 26.07421875, - "z": -50 + "x": -27.392578125, + "y": 20.849609375, + "z": 50 } ], "truth": { - "heading": "360", - "tool": "electromagnetic pulse", - "target": "white and red helicopter" + "heading": "170", + "tool": "interceptor jets", + "target": "silver commercial aircraft" } }, { - "audio": "simulator/data/audio/audio_4_target_0.wav", + "audio": "audio_4_target_0.wav", "files": [ "cubemap_4_posx.jpg", "cubemap_4_negx.jpg", @@ -418,85 +398,100 @@ ], "corners": [ { - "x": 34.521484375, - "y": 26.85546875, + "x": -36.5234375, + "y": 9.27734375, "z": 50 }, { - "x": 35.9375, - "y": 23.388671875, + "x": -33.88671875, + "y": 8.49609375, "z": 50 }, { - "x": 35.9375, - "y": 23.291015625, + "x": -33.837890625, + "y": 8.3984375, "z": 50 }, { - "x": 35.7421875, - "y": 23.046875, + "x": -33.88671875, + "y": 8.30078125, "z": 50 }, { - "x": 35.693359375, - "y": 22.998046875, + "x": -34.86328125, + "y": 6.4453125, "z": 50 }, { - "x": 35.64453125, - "y": 22.998046875, + "x": -34.9609375, + "y": 6.34765625, "z": 50 }, { - "x": 34.27734375, - "y": 23.486328125, + "x": -35.15625, + "y": 6.201171875, "z": 50 }, { - "x": 34.130859375, - "y": 23.6328125, + "x": -35.25390625, + "y": 6.15234375, "z": 50 }, { - "x": 33.0078125, - "y": 24.90234375, + "x": -35.3515625, + "y": 6.15234375, "z": 50 }, { - "x": 32.080078125, - "y": 27.05078125, + "x": -37.5, + "y": 6.4453125, "z": 50 }, { - "x": 32.080078125, - "y": 27.099609375, + "x": -37.548828125, + "y": 6.494140625, "z": 50 }, { - "x": 32.177734375, - "y": 27.294921875, + "x": -37.841796875, + "y": 6.93359375, "z": 50 }, { - "x": 32.275390625, - "y": 27.44140625, + "x": -38.134765625, + "y": 7.51953125, "z": 50 }, { - "x": 32.32421875, - "y": 27.44140625, + "x": -38.0859375, + "y": 7.91015625, "z": 50 }, { - "x": 34.47265625, - "y": 26.904296875, + "x": -37.060546875, + "y": 9.130859375, + "z": 50 + }, + { + "x": -37.01171875, + "y": 9.1796875, + "z": 50 + }, + { + "x": -36.81640625, + "y": 9.228515625, + "z": 50 + }, + { + "x": -36.572265625, + "y": 9.27734375, "z": 50 } ], "truth": { - "heading": "180", - "tool": "electromagnetic pulse", - "target": "silver commercial aircraft" + "heading": "185", + "tool": "machine gun", + "target": "grey fighter plane" } } ] \ No newline at end of file diff --git a/test_competition_server.py b/test_competition_server.py index fa0d32d..3beb355 100644 --- a/test_competition_server.py +++ b/test_competition_server.py @@ -152,7 +152,7 @@ async def team_endpoint(websocket: WebSocket, team_name: str): results["elapsed"] = elapsed results["nlp_score"] = nlp_eval([case["truth"]], [results["nlp"]]) results["vlm_score"] = vlm_eval([results["bbox"]], [results["vlm"]]) - results["perf_score"] = 1 - min(10, elapsed) / 10 + results["perf_score"] = 1 - min(30, elapsed) / 30 results["score"] = ( 0.45 * results["nlp_score"] + 0.45 * results["vlm_score"]