diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..7efa857 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,19 @@ +name: Build +on: + pull_request: + push: + branches: + - main + +jobs: + publish: + name: Build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Build + run: ./scripts/build.sh + + - name: Run Test + run: docker run nuclio_lighting_flash:latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..a62592d --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,42 @@ +name: Tag & Release + +on: + workflow_dispatch: + inputs: + TAG_NAME: + description: 'Tag name. eg) 1.0.0 or 1.0.0-rc.1' + required: true + +jobs: + release: + name: Release ${{ github.event.inputs.TAG_NAME }} + runs-on: ubuntu-latest + steps: + - name: Docker Login + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Checkout this repository + uses: actions/checkout@v2.3.4 + with: + path: ${{ github.repository }} + + - name: Bump version + env: + VERSION: ${{ github.event.inputs.TAG_NAME }} + DIRECTORY: + run: | + docker build . -t nuclio_lighting_flash + docker tag nuclio_lighting_flash ghcr.io/greenroom-robotics/nuclio_lighting_flash:latest + docker tag nuclio_lighting_flash ghcr.io/greenroom-robotics/nuclio_lighting_flash:${{github.event.inputs.TAG_NAME}} + docker push nuclio_lighting_flash ghcr.io/greenroom-robotics/nuclio_lighting_flash:latest + docker push nuclio_lighting_flash ghcr.io/greenroom-robotics/nuclio_lighting_flash:${{github.event.inputs.TAG_NAME}} + + - name: Create a release + uses: softprops/action-gh-release@v1 + with: + tag_name: ${{ github.event.inputs.TAG_NAME }} + generate_release_notes: true diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..95a100c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,17 @@ +FROM pytorchlightning/pytorch_lightning:1.6.4-py3.9-torch1.9 + +COPY ./nuclio_lighting_flash /opt/nuclio + +# Remote nvidia lists as they have a borked GPG +RUN rm /etc/apt/sources.list.d/nvidia-ml.list /etc/apt/sources.list.d/cuda.list + +# Install opencv deps +RUN apt-get update +RUN apt-get install ffmpeg libsm6 libxext6 -y + +# Install lighting flash and it's deps +RUN pip install lightning-flash icevision 'lightning-flash[image]' + +WORKDIR /opt/nuclio + +CMD python ./test_flash_model_handler.py \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..38714cc --- /dev/null +++ b/README.md @@ -0,0 +1,36 @@ +# Nuclio Lighting Flash + +This repo builds a base image "GreenroomRobotics/nuclio_lighting_flash" which can be used to run flash ObjectDetection models in Nuclio / CVAT + + +## Usage + +* Use [lightningflash-efficientdet-d0](./example/lightningflash-efficientdet-d0) as a reference example +* Modify/create your own as you see fit +* Deploy it: +```bash +nuctl deploy --project-name cvat \ + --path example/lightningflash-efficientdet-d0 \ + --platform local +``` + + +## Development + +### Get started + +In order to develop you'll want a nuclio instance running on your local machine... + +* `docker-compose up` to start nuclio. +* `./scripts/build.sh` to build `ghcr.io/greenroom-robotics/nuclio_lighting_flash:latest` +* Deploy the example to your nuclio instance: + +```bash +nuctl deploy --project-name cvat \ + --path example/lightningflash-efficientdet-d0 \ + --platform local +``` + +### Release a version + +* Run the [Release](./.github/workflows/release.yml) workflow on github \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..4f851a7 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,14 @@ +version: '3.3' +services: + nuclio: + container_name: nuclio + image: quay.io/nuclio/dashboard:1.8.15-amd64 + restart: always + volumes: + - /tmp:/tmp + - /var/run/docker.sock:/var/run/docker.sock + environment: + NUCLIO_CHECK_FUNCTION_CONTAINERS_HEALTHINESS: 'true' + NUCLIO_DASHBOARD_DEFAULT_FUNCTION_MOUNT_MODE: 'volume' + ports: + - '8070:8070' diff --git a/example/lightningflash-efficientdet-d0/function.yaml b/example/lightningflash-efficientdet-d0/function.yaml new file mode 100644 index 0000000..b0d9646 --- /dev/null +++ b/example/lightningflash-efficientdet-d0/function.yaml @@ -0,0 +1,117 @@ +metadata: + name: lightningflash-efficientdet-d0 + namespace: cvat + annotations: + name: Flash - EfficientDet d0 + type: detector + framework: pytorch + head: efficientdet # any flash ObjectDetector head + backbone: d0 # any flash ObjectDetector backbone + # checkpoint_path: # path to a model checkpoint + spec: | + [ + { "id": 1, "name": "person" }, + { "id": 2, "name": "bicycle" }, + { "id": 3, "name": "car" }, + { "id": 4, "name": "motorcycle" }, + { "id": 5, "name": "airplane" }, + { "id": 6, "name": "bus" }, + { "id": 7, "name": "train" }, + { "id": 8, "name": "truck" }, + { "id": 9, "name": "boat" }, + { "id":10, "name": "traffic_light" }, + { "id":11, "name": "fire_hydrant" }, + { "id":13, "name": "stop_sign" }, + { "id":14, "name": "parking_meter" }, + { "id":15, "name": "bench" }, + { "id":16, "name": "bird" }, + { "id":17, "name": "cat" }, + { "id":18, "name": "dog" }, + { "id":19, "name": "horse" }, + { "id":20, "name": "sheep" }, + { "id":21, "name": "cow" }, + { "id":22, "name": "elephant" }, + { "id":23, "name": "bear" }, + { "id":24, "name": "zebra" }, + { "id":25, "name": "giraffe" }, + { "id":27, "name": "backpack" }, + { "id":28, "name": "umbrella" }, + { "id":31, "name": "handbag" }, + { "id":32, "name": "tie" }, + { "id":33, "name": "suitcase" }, + { "id":34, "name": "frisbee" }, + { "id":35, "name": "skis" }, + { "id":36, "name": "snowboard" }, + { "id":37, "name": "sports_ball" }, + { "id":38, "name": "kite" }, + { "id":39, "name": "baseball_bat" }, + { "id":40, "name": "baseball_glove" }, + { "id":41, "name": "skateboard" }, + { "id":42, "name": "surfboard" }, + { "id":43, "name": "tennis_racket" }, + { "id":44, "name": "bottle" }, + { "id":46, "name": "wine_glass" }, + { "id":47, "name": "cup" }, + { "id":48, "name": "fork" }, + { "id":49, "name": "knife" }, + { "id":50, "name": "spoon" }, + { "id":51, "name": "bowl" }, + { "id":52, "name": "banana" }, + { "id":53, "name": "apple" }, + { "id":54, "name": "sandwich" }, + { "id":55, "name": "orange" }, + { "id":56, "name": "broccoli" }, + { "id":57, "name": "carrot" }, + { "id":58, "name": "hot_dog" }, + { "id":59, "name": "pizza" }, + { "id":60, "name": "donut" }, + { "id":61, "name": "cake" }, + { "id":62, "name": "chair" }, + { "id":63, "name": "couch" }, + { "id":64, "name": "potted_plant" }, + { "id":65, "name": "bed" }, + { "id":67, "name": "dining_table" }, + { "id":70, "name": "toilet" }, + { "id":72, "name": "tv" }, + { "id":73, "name": "laptop" }, + { "id":74, "name": "mouse" }, + { "id":75, "name": "remote" }, + { "id":76, "name": "keyboard" }, + { "id":77, "name": "cell_phone" }, + { "id":78, "name": "microwave" }, + { "id":79, "name": "oven" }, + { "id":80, "name": "toaster" }, + { "id":81, "name": "sink" }, + { "id":83, "name": "refrigerator" }, + { "id":84, "name": "book" }, + { "id":85, "name": "clock" }, + { "id":86, "name": "vase" }, + { "id":87, "name": "scissors" }, + { "id":88, "name": "teddy_bear" }, + { "id":89, "name": "hair_drier" }, + { "id":90, "name": "toothbrush" } + ] + +spec: + description: "head: efficientdet, backbone: d0" + runtime: 'python:3.6' + handler: main:handler + eventTimeout: 30s + build: + image: cvat/nuclio_lighting_flash + baseImage: ghcr.io/greenroom-robotics/nuclio_lighting_flash:latest + + triggers: + myHttpTrigger: + maxWorkers: 2 + kind: 'http' + workerAvailabilityTimeoutMilliseconds: 10000 + attributes: + maxRequestBodySize: 33554432 # 32MB + + platform: + attributes: + restartPolicy: + name: always + maximumRetryCount: 3 + mountMode: volume diff --git a/example/lightningflash-from-checkpoint/function.yaml b/example/lightningflash-from-checkpoint/function.yaml new file mode 100644 index 0000000..5689af0 --- /dev/null +++ b/example/lightningflash-from-checkpoint/function.yaml @@ -0,0 +1,117 @@ +metadata: + name: lightningflash-from-checkpoint + namespace: cvat + annotations: + name: Flash - From Checkout + type: detector + framework: pytorch + # head: efficientdet # any flash ObjectDetector head + # backbone: d0 # any flash ObjectDetector backbone + checkpoint_path: http://some-path-to-a-model.pt # path to a model checkpoint + spec: | + [ + { "id": 1, "name": "person" }, + { "id": 2, "name": "bicycle" }, + { "id": 3, "name": "car" }, + { "id": 4, "name": "motorcycle" }, + { "id": 5, "name": "airplane" }, + { "id": 6, "name": "bus" }, + { "id": 7, "name": "train" }, + { "id": 8, "name": "truck" }, + { "id": 9, "name": "boat" }, + { "id":10, "name": "traffic_light" }, + { "id":11, "name": "fire_hydrant" }, + { "id":13, "name": "stop_sign" }, + { "id":14, "name": "parking_meter" }, + { "id":15, "name": "bench" }, + { "id":16, "name": "bird" }, + { "id":17, "name": "cat" }, + { "id":18, "name": "dog" }, + { "id":19, "name": "horse" }, + { "id":20, "name": "sheep" }, + { "id":21, "name": "cow" }, + { "id":22, "name": "elephant" }, + { "id":23, "name": "bear" }, + { "id":24, "name": "zebra" }, + { "id":25, "name": "giraffe" }, + { "id":27, "name": "backpack" }, + { "id":28, "name": "umbrella" }, + { "id":31, "name": "handbag" }, + { "id":32, "name": "tie" }, + { "id":33, "name": "suitcase" }, + { "id":34, "name": "frisbee" }, + { "id":35, "name": "skis" }, + { "id":36, "name": "snowboard" }, + { "id":37, "name": "sports_ball" }, + { "id":38, "name": "kite" }, + { "id":39, "name": "baseball_bat" }, + { "id":40, "name": "baseball_glove" }, + { "id":41, "name": "skateboard" }, + { "id":42, "name": "surfboard" }, + { "id":43, "name": "tennis_racket" }, + { "id":44, "name": "bottle" }, + { "id":46, "name": "wine_glass" }, + { "id":47, "name": "cup" }, + { "id":48, "name": "fork" }, + { "id":49, "name": "knife" }, + { "id":50, "name": "spoon" }, + { "id":51, "name": "bowl" }, + { "id":52, "name": "banana" }, + { "id":53, "name": "apple" }, + { "id":54, "name": "sandwich" }, + { "id":55, "name": "orange" }, + { "id":56, "name": "broccoli" }, + { "id":57, "name": "carrot" }, + { "id":58, "name": "hot_dog" }, + { "id":59, "name": "pizza" }, + { "id":60, "name": "donut" }, + { "id":61, "name": "cake" }, + { "id":62, "name": "chair" }, + { "id":63, "name": "couch" }, + { "id":64, "name": "potted_plant" }, + { "id":65, "name": "bed" }, + { "id":67, "name": "dining_table" }, + { "id":70, "name": "toilet" }, + { "id":72, "name": "tv" }, + { "id":73, "name": "laptop" }, + { "id":74, "name": "mouse" }, + { "id":75, "name": "remote" }, + { "id":76, "name": "keyboard" }, + { "id":77, "name": "cell_phone" }, + { "id":78, "name": "microwave" }, + { "id":79, "name": "oven" }, + { "id":80, "name": "toaster" }, + { "id":81, "name": "sink" }, + { "id":83, "name": "refrigerator" }, + { "id":84, "name": "book" }, + { "id":85, "name": "clock" }, + { "id":86, "name": "vase" }, + { "id":87, "name": "scissors" }, + { "id":88, "name": "teddy_bear" }, + { "id":89, "name": "hair_drier" }, + { "id":90, "name": "toothbrush" } + ] + +spec: + description: "object detector from checkpoint" + runtime: 'python:3.6' + handler: main:handler + eventTimeout: 30s + build: + image: cvat/nuclio_lighting_flash + baseImage: ghcr.io/greenroom-robotics/nuclio_lighting_flash:latest + + triggers: + myHttpTrigger: + maxWorkers: 2 + kind: 'http' + workerAvailabilityTimeoutMilliseconds: 10000 + attributes: + maxRequestBodySize: 33554432 # 32MB + + platform: + attributes: + restartPolicy: + name: always + maximumRetryCount: 3 + mountMode: volume diff --git a/nuclio_lighting_flash/fixtures/giraffe.jpg b/nuclio_lighting_flash/fixtures/giraffe.jpg new file mode 100644 index 0000000..c819e3a Binary files /dev/null and b/nuclio_lighting_flash/fixtures/giraffe.jpg differ diff --git a/nuclio_lighting_flash/flash_model_handler.py b/nuclio_lighting_flash/flash_model_handler.py new file mode 100644 index 0000000..b9253bc --- /dev/null +++ b/nuclio_lighting_flash/flash_model_handler.py @@ -0,0 +1,42 @@ +from flash.image import ObjectDetector, ObjectDetectionData +from flash.core.trainer import Trainer +from flash.core.utilities.stages import RunningStage +from PIL.Image import Image + +from nuclio_detection_labels_output import NuclioDetectionLabelsOutput + + +class MockTrainer(Trainer): + def __init__(self): + super().__init__() + self.state.stage = RunningStage.PREDICTING # type: ignore + + +class FlashModelHandler: + def __init__( + self, + model: ObjectDetector, + image_size = 1024, + labels = {} + ): + self.image_size = image_size + self.labels = labels + self.model = model + self.trainer = MockTrainer() + self.model.eval() + + def infer(self, image: Image, threshold: float = 0.0): + path = "/tmp/image.jpg" + image.save(path) + + datamodule = ObjectDetectionData.from_files( + predict_files=[path], + transform_kwargs={"image_size": self.image_size}, + batch_size=1, + ) + predictions = self.trainer.predict( + self.model, + datamodule=datamodule, + output=NuclioDetectionLabelsOutput(threshold=threshold, labels=self.labels), + ) + return predictions \ No newline at end of file diff --git a/nuclio_lighting_flash/flash_model_handler_in_memory.py b/nuclio_lighting_flash/flash_model_handler_in_memory.py new file mode 100644 index 0000000..f119627 --- /dev/null +++ b/nuclio_lighting_flash/flash_model_handler_in_memory.py @@ -0,0 +1,64 @@ +from typing import Union +import torch +from torchvision import transforms +from flash.image import ObjectDetector +from flash.core.data.io.output import Output +from flash.core.data.io.input import DataKeys +from flash.core.data.data_module import DataModule +from flash.core.data.io.input import DataKeys +from flash.core.data.io.output_transform import OutputTransform +from flash.core.trainer import Trainer +from flash.core.utilities.stages import RunningStage +from flash.core.integrations.icevision.adapter import to_icevision_record + +# WORK IN PROGRESS + +IMAGE_SIZE = 1024 + + +class MockTrainer(Trainer): + def __init__(self): + super().__init__() + self.state.stage = RunningStage.PREDICTING # type: ignore + + +class FlashModelHandler: + def __init__(self, model_path: str, output: Union[str, Output] = Output()): + self.model = ObjectDetector( + head="efficientdet", backbone="d0", num_classes=91, image_size=IMAGE_SIZE + ) + self.output_transform_final = Output() + self.trainer = MockTrainer() + self.data_module = DataModule(batch_size=1) + self.data_module.trainer = self.trainer # type: ignore + self.model.eval() + + self.output_transform = OutputTransform() + + def infer(self, image): + image_input = {} + image_input[DataKeys.INPUT] = image + inputs = { + DataKeys.INPUT: [ + [transforms.ToTensor()(image).unsqueeze_(0)], + [to_icevision_record(image_input)], + ], + DataKeys.METADATA: {"size": [224, 224]}, + } + + with torch.no_grad(): + inputs = self.model.transfer_batch_to_device(inputs, self.model.device, 0) # type: ignore + inputs = self.data_module.on_after_batch_transfer(inputs, 0) + preds = self.model.predict_step(inputs, 0) + print("================") + print(preds) + # preds = self.output_transform(preds) + # preds = preds[0][DataKeys.PREDS] + # preds_output = self.output_transform_final(preds) + + # return { + # "confidence": 1, + # "label": preds_output, + # "points": [0, 0, 10, 10], + # "type": "rectangle", + # } diff --git a/nuclio_lighting_flash/main.py b/nuclio_lighting_flash/main.py new file mode 100644 index 0000000..8dd6cf7 --- /dev/null +++ b/nuclio_lighting_flash/main.py @@ -0,0 +1,55 @@ +from __future__ import annotations +import json +import base64 +from PIL import Image +import io +import yaml + +from flash.image import ObjectDetector +from flash_model_handler import FlashModelHandler + + +def init_context(context): + context.logger.info("Init context... 0%") + + # Read labels + with open("/opt/nuclio/function.yaml", "rb") as function_file: + functionconfig = yaml.safe_load(function_file) + annotations = labels_spec = functionconfig["metadata"]["annotations"] + + labels_spec = annotations["spec"] + labels = {item["id"]: item["name"] for item in json.loads(labels_spec)} + + # Read the DL model + # Either "checkpoint_path" or "head" and "backbone" should be specified + model = ( + ObjectDetector.load_from_checkpoint(annotations["checkpoint_path"]) + if "checkpoint_path" in annotations + else ObjectDetector( + head=annotations["head"], + backbone=annotations["backbone"], + num_classes=len(labels), + image_size=1024, + ) + ) + model_handler = FlashModelHandler(model=model, image_size=1024, labels=labels) + context.user_data.model = model_handler + + context.logger.info("Init context...100%") + + +def handler(context, event): + context.logger.info("Run Lighting Flash model") + data = event.body + buf = io.BytesIO(base64.b64decode(data["image"])) + threshold = float(data.get("threshold", 0.5)) + image = Image.open(buf) + + results = context.user_data.model.infer(image, threshold) + + return context.Response( + body=json.dumps(results), + headers={}, + content_type="application/json", + status_code=200, + ) diff --git a/nuclio_lighting_flash/nuclio_detection_labels_output.py b/nuclio_lighting_flash/nuclio_detection_labels_output.py new file mode 100644 index 0000000..f9a96cb --- /dev/null +++ b/nuclio_lighting_flash/nuclio_detection_labels_output.py @@ -0,0 +1,63 @@ +from typing import Any, Dict, List, Optional, Union + +from flash.core.data.io.input import DataKeys +from flash.core.data.io.output import Output +from flash.core.model import Task +from flash.core.registry import FlashRegistry +from flash.core.utilities.imports import _FIFTYONE_AVAILABLE, lazy_import, requires +from flash.core.utilities.providers import _FIFTYONE + +class NuclioDetectionLabelsOutput(Output): + """A :class:`.Output` which converts model outputs to Nuclio detection format. + + Args: + labels: A list of labels, assumed to map the class index to the label for that class. + threshold: a score threshold to apply to candidate detections. + """ + + def __init__( + self, + labels: Optional[List[str]] = None, + threshold: Optional[float] = None, + ): + super().__init__() + self._labels = labels + self.threshold = threshold + + @classmethod + def from_task(cls, task: Task, **kwargs) -> Output: + return cls(labels=getattr(task, "labels", None)) + + def transform(self, sample: Dict[str, Any]) -> List[Dict[str, Any]]: + detections = [] + + preds = sample[DataKeys.PREDS] + + for bbox, label, score in zip(preds["bboxes"], preds["labels"], preds["scores"]): + confidence = score.tolist() + + if self.threshold is not None and confidence < self.threshold: + continue + + box = [ + bbox["xmin"], + bbox["ymin"], + bbox["xmin"] + bbox["width"], + bbox["ymin"] + bbox["height"], + ] + + label = label.item() + if self._labels is not None: + label = self._labels[label] + else: + label = str(int(label)) + + detections.append( + { + "confidence": confidence, + "label": label, + "points": box, + "type": "rectangle", + } + ) + return detections diff --git a/nuclio_lighting_flash/test_flash_model_handler.py b/nuclio_lighting_flash/test_flash_model_handler.py new file mode 100644 index 0000000..8f27fca --- /dev/null +++ b/nuclio_lighting_flash/test_flash_model_handler.py @@ -0,0 +1,23 @@ +from PIL import Image +from flash.image import ObjectDetector +import os + +from flash_model_handler import FlashModelHandler + +model = ObjectDetector( + head="efficientdet", + backbone="d0", + num_classes=91, + image_size=1024 +) +model_handler = FlashModelHandler( + model=model, + image_size=1024, + labels={ + 25: "giraffe" + } +) +image = Image.open(os.path.join(os.getcwd(), "./fixtures/giraffe.jpg")) + +result = model_handler.infer(image, 0) +print(result) diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100755 index 0000000..d81c6c3 --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1 @@ +docker build . -t ghcr.io/greenroom-robotics/nuclio_lighting_flash \ No newline at end of file