Sort object detection results and reduce code duplication (#1173)

* Sort object detection results and reduce code dup. * Filter objdet results by ratio and area * Address code review --------- Co-authored-by: Mohammad Durrani <[email protected]>
PhotonVision · Jan 23, 2024 · eca3cea · eca3cea
1 parent cbbfbda
commit eca3cea
Show file tree

Hide file tree

Showing 12 changed files with 239 additions and 93 deletions.
diff --git a/photon-client/src/components/dashboard/tabs/ObjectDetectionTab.vue b/photon-client/src/components/dashboard/tabs/ObjectDetectionTab.vue
@@ -11,6 +11,16 @@ const currentPipelineSettings = computed<ActivePipelineSettings>(
   () => useCameraSettingsStore().currentPipelineSettings
 );
 
+// TODO fix pv-range-slider so that store access doesn't need to be deferred
+const contourArea = computed<[number, number]>({
+  get: () => Object.values(useCameraSettingsStore().currentPipelineSettings.contourArea) as [number, number],
+  set: (v) => (useCameraSettingsStore().currentPipelineSettings.contourArea = v)
+});
+const contourRatio = computed<[number, number]>({
+  get: () => Object.values(useCameraSettingsStore().currentPipelineSettings.contourRatio) as [number, number],
+  set: (v) => (useCameraSettingsStore().currentPipelineSettings.contourRatio = v)
+});
+
 const interactiveCols = computed(() =>
   (getCurrentInstance()?.proxy.$vuetify.breakpoint.mdAndDown || false) &&
   (!useStateStore().sidebarFolded || useCameraSettingsStore().isDriverMode)
@@ -32,5 +42,42 @@ const interactiveCols = computed(() =>
       :step="0.01"
       @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ confidence: value }, false)"
     />
+    <pv-range-slider
+      v-model="contourArea"
+      label="Area"
+      :min="0"
+      :max="100"
+      :slider-cols="interactiveCols"
+      :step="0.01"
+      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ contourArea: value }, false)"
+    />
+    <pv-range-slider
+      v-model="contourRatio"
+      label="Ratio (W/H)"
+      tooltip="Min and max ratio between the width and height of a contour's bounding rectangle"
+      :min="0"
+      :max="100"
+      :slider-cols="interactiveCols"
+      :step="0.01"
+      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ contourRatio: value }, false)"
+    />
+    <pv-select
+      v-model="useCameraSettingsStore().currentPipelineSettings.contourTargetOrientation"
+      label="Target Orientation"
+      tooltip="Used to determine how to calculate target landmarks, as well as aspect ratio"
+      :items="['Portrait', 'Landscape']"
+      :select-cols="interactiveCols"
+      @input="
+        (value) => useCameraSettingsStore().changeCurrentPipelineSetting({ contourTargetOrientation: value }, false)
+      "
+    />
+    <pv-select
+      v-model="currentPipelineSettings.contourSortMode"
+      label="Target Sort"
+      tooltip="Chooses the sorting mode used to determine the 'best' targets to provide to user code"
+      :select-cols="interactiveCols"
+      :items="['Largest', 'Smallest', 'Highest', 'Lowest', 'Rightmost', 'Leftmost', 'Centermost']"
+      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ contourSortMode: value }, false)"
+    />
   </div>
 </template>
diff --git a/photon-core/src/main/java/org/photonvision/vision/opencv/Contour.java b/photon-core/src/main/java/org/photonvision/vision/opencv/Contour.java
@@ -47,6 +47,16 @@ public Contour(MatOfPoint mat) {
         this.mat = mat;
     }
 
+    public Contour(Rect2d box) {
+        // no easy way to convert a Rect2d to Mat, diy it. Order is tl tr br bl
+        this.mat =
+                new MatOfPoint(
+                        box.tl(),
+                        new Point(box.x + box.width, box.y),
+                        box.br(),
+                        new Point(box.x, box.y + box.height));
+    }
+
     public MatOfPoint2f getMat2f() {
         if (mat2f == null) {
             mat2f = new MatOfPoint2f(mat.toArray());

diff --git a/photon-core/src/main/java/org/photonvision/vision/opencv/ContourSortMode.java b/photon-core/src/main/java/org/photonvision/vision/opencv/ContourSortMode.java
@@ -25,15 +25,15 @@ public enum ContourSortMode {
             Comparator.comparingDouble(PotentialTarget::getArea)
                     .reversed()), // reversed so that zero index has the largest size
     Smallest(Largest.getComparator().reversed()),
-    Highest(Comparator.comparingDouble(rect -> rect.getMinAreaRect().center.y)),
+    Highest(Comparator.comparingDouble(tgt -> tgt.getMinAreaRect().center.y)),
     Lowest(Highest.getComparator().reversed()),
-    Leftmost(Comparator.comparingDouble(target -> target.getMinAreaRect().center.x * -1)),
+    Leftmost(Comparator.comparingDouble(tgt -> tgt.getMinAreaRect().center.x * -1)),
     Rightmost(Leftmost.getComparator().reversed()),
     Centermost(
             Comparator.comparingDouble(
-                    rect ->
-                            (Math.pow(rect.getMinAreaRect().center.y, 2)
-                                    + Math.pow(rect.getMinAreaRect().center.x, 2))));
+                    tgt ->
+                            (Math.pow(tgt.getMinAreaRect().center.y, 2)
+                                    + Math.pow(tgt.getMinAreaRect().center.x, 2))));
 
     private final Comparator<PotentialTarget> m_comparator;
 

diff --git a/photon-core/src/main/java/org/photonvision/vision/pipe/impl/FilterObjectDetectionsPipe.java b/photon-core/src/main/java/org/photonvision/vision/pipe/impl/FilterObjectDetectionsPipe.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.vision.pipe.impl;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.photonvision.common.util.numbers.DoubleCouple;
+import org.photonvision.vision.frame.FrameStaticProperties;
+import org.photonvision.vision.pipe.CVPipe;
+
+public class FilterObjectDetectionsPipe
+        extends CVPipe<
+                List<NeuralNetworkPipeResult>,
+                List<NeuralNetworkPipeResult>,
+                FilterObjectDetectionsPipe.FilterContoursParams> {
+    List<NeuralNetworkPipeResult> m_filteredContours = new ArrayList<>();
+
+    @Override
+    protected List<NeuralNetworkPipeResult> process(List<NeuralNetworkPipeResult> in) {
+        m_filteredContours.clear();
+        for (var contour : in) {
+            filterContour(contour);
+        }
+
+        return m_filteredContours;
+    }
+
+    private void filterContour(NeuralNetworkPipeResult contour) {
+        var boc = contour.box;
+
+        // Area filtering
+        double areaPercentage = boc.area() / params.getFrameStaticProperties().imageArea * 100.0;
+        double minAreaPercentage = params.getArea().getFirst();
+        double maxAreaPercentage = params.getArea().getSecond();
+        if (areaPercentage < minAreaPercentage || areaPercentage > maxAreaPercentage) return;
+
+        // Aspect ratio filtering; much simpler since always axis-aligned
+        double aspectRatio = boc.width / boc.height;
+        if (aspectRatio < params.getRatio().getFirst() || aspectRatio > params.getRatio().getSecond())
+            return;
+
+        m_filteredContours.add(contour);
+    }
+
+    public static class FilterContoursParams {
+        private final DoubleCouple m_area;
+        private final DoubleCouple m_ratio;
+        private final FrameStaticProperties m_frameStaticProperties;
+        public final boolean isLandscape;
+
+        public FilterContoursParams(
+                DoubleCouple area,
+                DoubleCouple ratio,
+                FrameStaticProperties camProperties,
+                boolean isLandscape) {
+            this.m_area = area;
+            this.m_ratio = ratio;
+            this.m_frameStaticProperties = camProperties;
+            this.isLandscape = isLandscape;
+        }
+
+        public DoubleCouple getArea() {
+            return m_area;
+        }
+
+        public DoubleCouple getRatio() {
+            return m_ratio;
+        }
+
+        public FrameStaticProperties getFrameStaticProperties() {
+            return m_frameStaticProperties;
+        }
+    }
+}
diff --git a/photon-core/src/main/java/org/photonvision/vision/pipe/impl/SortContoursPipe.java b/photon-core/src/main/java/org/photonvision/vision/pipe/impl/SortContoursPipe.java
@@ -42,6 +42,7 @@ protected List<PotentialTarget> process(List<PotentialTarget> in) {
             if (params.getSortMode() != ContourSortMode.Centermost) {
                 m_sortedContours.sort(params.getSortMode().getComparator());
             } else {
+                // we need knowledge of camera properties to calculate this distance -- do it ourselves
                 m_sortedContours.sort(Comparator.comparingDouble(this::calcSquareCenterDistance));
             }
         }
@@ -50,10 +51,10 @@ protected List<PotentialTarget> process(List<PotentialTarget> in) {
                 m_sortedContours.subList(0, Math.min(in.size(), params.getMaxTargets())));
     }
 
-    private double calcSquareCenterDistance(PotentialTarget rect) {
+    private double calcSquareCenterDistance(PotentialTarget tgt) {
         return Math.sqrt(
-                Math.pow(params.getCamProperties().centerX - rect.getMinAreaRect().center.x, 2)
-                        + Math.pow(params.getCamProperties().centerY - rect.getMinAreaRect().center.y, 2));
+                Math.pow(params.getCamProperties().centerX - tgt.getMinAreaRect().center.x, 2)
+                        + Math.pow(params.getCamProperties().centerY - tgt.getMinAreaRect().center.y, 2));
     }
 
     public static class SortContoursParams {

diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/CVPipeline.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/CVPipeline.java
@@ -26,6 +26,8 @@
 
 public abstract class CVPipeline<R extends CVPipelineResult, S extends CVPipelineSettings>
         implements Releasable {
+    static final int MAX_MULTI_TARGET_RESULTS = 10;
+
     protected S settings;
     protected FrameStaticProperties frameStaticProperties;
     protected QuirkyCamera cameraQuirks;

diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/ColoredShapePipeline.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/ColoredShapePipeline.java
@@ -109,7 +109,7 @@ protected void setPipeParamsImpl() {
         SortContoursPipe.SortContoursParams sortContoursParams =
                 new SortContoursPipe.SortContoursParams(
                         settings.contourSortMode,
-                        settings.outputShowMultipleTargets ? 5 : 1,
+                        settings.outputShowMultipleTargets ? MAX_MULTI_TARGET_RESULTS : 1,
                         frameStaticProperties); // TODO don't hardcode?
         sortContoursPipe.setParams(sortContoursParams);
 

diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/ObjectDetectionPipeline.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/ObjectDetectionPipeline.java
@@ -17,21 +17,26 @@
 
 package org.photonvision.vision.pipeline;
 
-import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Collectors;
 import org.photonvision.vision.frame.Frame;
 import org.photonvision.vision.frame.FrameThresholdType;
+import org.photonvision.vision.opencv.DualOffsetValues;
 import org.photonvision.vision.pipe.CVPipe.CVPipeResult;
 import org.photonvision.vision.pipe.impl.*;
 import org.photonvision.vision.pipe.impl.RknnDetectionPipe.RknnDetectionPipeParams;
 import org.photonvision.vision.pipeline.result.CVPipelineResult;
+import org.photonvision.vision.target.PotentialTarget;
+import org.photonvision.vision.target.TargetOrientation;
 import org.photonvision.vision.target.TrackedTarget;
-import org.photonvision.vision.target.TrackedTarget.TargetCalculationParameters;
 
 public class ObjectDetectionPipeline
         extends CVPipeline<CVPipelineResult, ObjectDetectionPipelineSettings> {
     private final CalculateFPSPipe calculateFPSPipe = new CalculateFPSPipe();
     private final RknnDetectionPipe rknnPipe = new RknnDetectionPipe();
+    private final SortContoursPipe sortContoursPipe = new SortContoursPipe();
+    private final Collect2dTargetsPipe collect2dTargetsPipe = new Collect2dTargetsPipe();
+    private final FilterObjectDetectionsPipe filterContoursPipe = new FilterObjectDetectionsPipe();
 
     private static final FrameThresholdType PROCESSING_TYPE = FrameThresholdType.NONE;
 
@@ -52,6 +57,38 @@ protected void setPipeParamsImpl() {
         params.confidence = settings.confidence;
         params.nms = settings.nms;
         rknnPipe.setParams(params);
+
+        DualOffsetValues dualOffsetValues =
+                new DualOffsetValues(
+                        settings.offsetDualPointA,
+                        settings.offsetDualPointAArea,
+                        settings.offsetDualPointB,
+                        settings.offsetDualPointBArea);
+
+        SortContoursPipe.SortContoursParams sortContoursParams =
+                new SortContoursPipe.SortContoursParams(
+                        settings.contourSortMode,
+                        settings.outputShowMultipleTargets ? MAX_MULTI_TARGET_RESULTS : 1,
+                        frameStaticProperties);
+        sortContoursPipe.setParams(sortContoursParams);
+
+        var filterContoursParams =
+                new FilterObjectDetectionsPipe.FilterContoursParams(
+                        settings.contourArea,
+                        settings.contourRatio,
+                        frameStaticProperties,
+                        settings.contourTargetOrientation == TargetOrientation.Landscape);
+        filterContoursPipe.setParams(filterContoursParams);
+
+        Collect2dTargetsPipe.Collect2dTargetsParams collect2dTargetsParams =
+                new Collect2dTargetsPipe.Collect2dTargetsParams(
+                        settings.offsetRobotOffsetMode,
+                        settings.offsetSinglePoint,
+                        dualOffsetValues,
+                        settings.contourTargetOffsetPointEdge,
+                        settings.contourTargetOrientation,
+                        frameStaticProperties);
+        collect2dTargetsPipe.setParams(collect2dTargetsParams);
     }
 
     @Override
@@ -60,31 +97,35 @@ protected CVPipelineResult process(Frame input_frame, ObjectDetectionPipelineSet
 
         // ***************** change based on backend ***********************
 
-        CVPipeResult<List<NeuralNetworkPipeResult>> ret = rknnPipe.run(input_frame.colorImage);
-        sumPipeNanosElapsed += ret.nanosElapsed;
+        CVPipeResult<List<NeuralNetworkPipeResult>> rknnResult = rknnPipe.run(input_frame.colorImage);
+        sumPipeNanosElapsed += rknnResult.nanosElapsed;
         List<NeuralNetworkPipeResult> targetList;
 
-        targetList = ret.output;
         var names = rknnPipe.getClassNames();
 
         input_frame.colorImage.getMat().copyTo(input_frame.processedImage.getMat());
 
         // ***************** change based on backend ***********************
 
-        List<TrackedTarget> targets = new ArrayList<>();
+        var filterContoursResult = filterContoursPipe.run(rknnResult.output);
+        sumPipeNanosElapsed += filterContoursResult.nanosElapsed;
+
+        CVPipeResult<List<PotentialTarget>> sortContoursResult =
+                sortContoursPipe.run(
+                        filterContoursResult.output.stream()
+                                .map(shape -> new PotentialTarget(shape))
+                                .collect(Collectors.toList()));
+        sumPipeNanosElapsed += sortContoursResult.nanosElapsed;
 
-        for (var t : targetList) {
-            targets.add(
-                    new TrackedTarget(
-                            t,
-                            new TargetCalculationParameters(
-                                    false, null, null, null, null, frameStaticProperties)));
-        }
+        CVPipeResult<List<TrackedTarget>> collect2dTargetsResult =
+                collect2dTargetsPipe.run(sortContoursResult.output);
+        sumPipeNanosElapsed += collect2dTargetsResult.nanosElapsed;
 
         var fpsResult = calculateFPSPipe.run(null);
         var fps = fpsResult.output;
 
-        return new CVPipelineResult(sumPipeNanosElapsed, fps, targets, input_frame, names);
+        return new CVPipelineResult(
+                sumPipeNanosElapsed, fps, collect2dTargetsResult.output, input_frame, names);
     }
 
     @Override

diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/ReflectivePipeline.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/ReflectivePipeline.java
@@ -64,29 +64,6 @@ protected void setPipeParamsImpl() {
                         settings.offsetDualPointB,
                         settings.offsetDualPointBArea);
 
-        // var rotateImageParams = new
-        // RotateImagePipe.RotateImageParams(settings.inputImageRotationMode);
-        // rotateImagePipe.setParams(rotateImageParams);
-
-        // if (cameraQuirks.hasQuirk(CameraQuirk.PiCam) && LibCameraJNI.isSupported()) {
-        //     LibCameraJNI.setThresholds(
-        //             settings.hsvHue.getFirst() / 180d,
-        //             settings.hsvSaturation.getFirst() / 255d,
-        //             settings.hsvValue.getFirst() / 255d,
-        //             settings.hsvHue.getSecond() / 180d,
-        //             settings.hsvSaturation.getSecond() / 255d,
-        //             settings.hsvValue.getSecond() / 255d);
-        //     //     LibCameraJNI.setInvertHue(settings.hueInverted);
-        //     LibCameraJNI.setRotation(settings.inputImageRotationMode.value);
-        //     //     LibCameraJNI.setShouldCopyColor(settings.inputShouldShow);
-        // } else {
-        //     var hsvParams =
-        //             new HSVPipe.HSVParams(
-        //                     settings.hsvHue, settings.hsvSaturation, settings.hsvValue,
-        // settings.hueInverted);
-        //     hsvPipe.setParams(hsvParams);
-        // }
-
         var findContoursParams = new FindContoursPipe.FindContoursParams();
         findContoursPipe.setParams(findContoursParams);
 
@@ -113,7 +90,7 @@ protected void setPipeParamsImpl() {
         var sortContoursParams =
                 new SortContoursPipe.SortContoursParams(
                         settings.contourSortMode,
-                        settings.outputShowMultipleTargets ? 8 : 1, // TODO don't hardcode?
+                        settings.outputShowMultipleTargets ? MAX_MULTI_TARGET_RESULTS : 1,
                         frameStaticProperties);
         sortContoursPipe.setParams(sortContoursParams);
 

diff --git a/photon-core/src/main/java/org/photonvision/vision/processes/VisionRunner.java b/photon-core/src/main/java/org/photonvision/vision/processes/VisionRunner.java
@@ -98,8 +98,7 @@ private void update() {
                 var pipelineResult = pipeline.run(frame, cameraQuirks);
                 pipelineResultConsumer.accept(pipelineResult);
             } catch (Exception ex) {
-                logger.error("Exception on loop " + loopCount);
-                ex.printStackTrace();
+                logger.error("Exception on loop " + loopCount, ex);
             }
 
             loopCount++;