Skip to content

Commit 7b19a95

Browse files
JuniormunkJuniormunk
authored
Camera Lost Stream (#1341)
* Fix no stream on camera unplug. * Spotless remove datarate * Make Static Frames Class * lint and format
1 parent db531f1 commit 7b19a95

File tree

4 files changed

+149
-124
lines changed

4 files changed

+149
-124
lines changed

‎photon-core/src/main/java/org/photonvision/vision/frame/StaticFrames.java

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
/*
2+
* Copyright (C) Photon Vision.
3+
*
4+
* This program is free software: you can redistribute it and/or modify
5+
* it under the terms of the GNU General Public License as published by
6+
* the Free Software Foundation, either version 3 of the License, or
7+
* (at your option) any later version.
8+
*
9+
* This program is distributed in the hope that it will be useful,
10+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
* GNU General Public License for more details.
13+
*
14+
* You should have received a copy of the GNU General Public License
15+
* along with this program. If not, see <https://www.gnu.org/licenses/>.
16+
*/
17+
18+
package org.photonvision.vision.frame;
19+
20+
import java.awt.Color;
21+
import org.opencv.core.CvType;
22+
import org.opencv.core.Mat;
23+
import org.opencv.core.Point;
24+
import org.opencv.core.Rect;
25+
import org.opencv.imgproc.Imgproc;
26+
import org.photonvision.common.util.ColorHelper;
27+
28+
public class StaticFrames {
29+
public static final Mat LOST_MAT = new Mat(60, 15 * 7, CvType.CV_8UC3);
30+
public static final Mat EMPTY_MAT = new Mat(60, 15 * 7, CvType.CV_8UC3);
31+
32+
static {
33+
EMPTY_MAT.setTo(ColorHelper.colorToScalar(Color.BLACK));
34+
var col = 0;
35+
Imgproc.rectangle(
36+
EMPTY_MAT,
37+
new Rect(col, 0, 15, EMPTY_MAT.height()),
38+
ColorHelper.colorToScalar(new Color(0xa2a2a2)),
39+
-1);
40+
col += 15;
41+
Imgproc.rectangle(
42+
EMPTY_MAT,
43+
new Rect(col, 0, 15, EMPTY_MAT.height()),
44+
ColorHelper.colorToScalar(new Color(0xa2a300)),
45+
-1);
46+
col += 15;
47+
Imgproc.rectangle(
48+
EMPTY_MAT,
49+
new Rect(col, 0, 15, EMPTY_MAT.height()),
50+
ColorHelper.colorToScalar(new Color(0x00a3a2)),
51+
-1);
52+
col += 15;
53+
Imgproc.rectangle(
54+
EMPTY_MAT,
55+
new Rect(col, 0, 15, EMPTY_MAT.height()),
56+
ColorHelper.colorToScalar(new Color(0x00a200)),
57+
-1);
58+
col += 15;
59+
Imgproc.rectangle(
60+
EMPTY_MAT,
61+
new Rect(col, 0, 15, EMPTY_MAT.height()),
62+
ColorHelper.colorToScalar(new Color(0x440045)),
63+
-1);
64+
col += 15;
65+
Imgproc.rectangle(
66+
EMPTY_MAT,
67+
new Rect(col, 0, 15, EMPTY_MAT.height()),
68+
ColorHelper.colorToScalar(new Color(0x0000a2)),
69+
-1);
70+
col += 15;
71+
Imgproc.rectangle(
72+
EMPTY_MAT,
73+
new Rect(col, 0, 15, EMPTY_MAT.height()),
74+
ColorHelper.colorToScalar(new Color(0)),
75+
-1);
76+
Imgproc.rectangle(
77+
EMPTY_MAT,
78+
new Rect(0, 50, EMPTY_MAT.width(), 10),
79+
ColorHelper.colorToScalar(new Color(0)),
80+
-1);
81+
Imgproc.rectangle(
82+
EMPTY_MAT, new Rect(15, 50, 30, 10), ColorHelper.colorToScalar(Color.WHITE), -1);
83+
84+
EMPTY_MAT.copyTo(LOST_MAT);
85+
86+
Imgproc.putText(
87+
EMPTY_MAT, "Stream", new Point(14, 20), 0, 0.6, ColorHelper.colorToScalar(Color.white), 2);
88+
Imgproc.putText(
89+
EMPTY_MAT,
90+
"Disabled",
91+
new Point(14, 45),
92+
0,
93+
0.6,
94+
ColorHelper.colorToScalar(Color.white),
95+
2);
96+
97+
Imgproc.putText(
98+
EMPTY_MAT, "Stream", new Point(14, 20), 0, 0.6, ColorHelper.colorToScalar(Color.RED), 1);
99+
Imgproc.putText(
100+
EMPTY_MAT, "Disabled", new Point(14, 45), 0, 0.6, ColorHelper.colorToScalar(Color.RED), 1);
101+
102+
Imgproc.putText(
103+
LOST_MAT, "Camera", new Point(14, 20), 0, 0.6, ColorHelper.colorToScalar(Color.white), 2);
104+
Imgproc.putText(
105+
LOST_MAT, "Lost", new Point(14, 45), 0, 0.6, ColorHelper.colorToScalar(Color.white), 2);
106+
Imgproc.putText(
107+
LOST_MAT, "Camera", new Point(14, 20), 0, 0.6, ColorHelper.colorToScalar(Color.RED), 1);
108+
Imgproc.putText(
109+
LOST_MAT, "Lost", new Point(14, 45), 0, 0.6, ColorHelper.colorToScalar(Color.RED), 1);
110+
}
111+
112+
public StaticFrames() {}
113+
}

‎photon-core/src/main/java/org/photonvision/vision/frame/consumer/FileSaveFrameConsumer.java

Lines changed: 25 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.photonvision.common.dataflow.networktables.NetworkTablesManager;
3030
import org.photonvision.common.logging.LogGroup;
3131
import org.photonvision.common.logging.Logger;
32+
import org.photonvision.vision.frame.StaticFrames;
3233
import org.photonvision.vision.opencv.CVMat;
3334

3435
public class FileSaveFrameConsumer implements Consumer<CVMat> {
@@ -64,36 +65,38 @@ public FileSaveFrameConsumer(String camNickname, String cameraUniqueName, String
6465
}
6566

6667
public void accept(CVMat image) {
67-
if (image != null && image.getMat() != null && !image.getMat().empty()) {
68-
long currentCount = saveFrameEntry.get();
68+
long currentCount = saveFrameEntry.get();
6969

70-
// Await save request
71-
if (currentCount == -1) return;
70+
// Await save request
71+
if (currentCount == -1) return;
7272

73-
// The requested count is greater than the actual count
74-
if (savedImagesCount < currentCount) {
75-
Date now = new Date();
73+
// The requested count is greater than the actual count
74+
if (savedImagesCount < currentCount) {
75+
Date now = new Date();
7676

77-
String fileName =
78-
cameraNickname + "_" + streamType + "_" + df.format(now) + "T" + tf.format(now);
77+
String fileName =
78+
cameraNickname + "_" + streamType + "_" + df.format(now) + "T" + tf.format(now);
7979

80-
// Check if the Unique Camera directory exists and create it if it doesn't
81-
String cameraPath = FILE_PATH + File.separator + this.cameraUniqueName;
82-
var cameraDir = new File(cameraPath);
83-
if (!cameraDir.exists()) {
84-
cameraDir.mkdir();
85-
}
80+
// Check if the Unique Camera directory exists and create it if it doesn't
81+
String cameraPath = FILE_PATH + File.separator + this.cameraUniqueName;
82+
var cameraDir = new File(cameraPath);
83+
if (!cameraDir.exists()) {
84+
cameraDir.mkdir();
85+
}
8686

87-
String saveFilePath = cameraPath + File.separator + fileName + FILE_EXTENSION;
87+
String saveFilePath = cameraPath + File.separator + fileName + FILE_EXTENSION;
8888

89+
if (image == null || image.getMat() == null || image.getMat().empty()) {
90+
Imgcodecs.imwrite(saveFilePath, StaticFrames.LOST_MAT);
91+
} else {
8992
Imgcodecs.imwrite(saveFilePath, image.getMat());
90-
91-
savedImagesCount++;
92-
logger.info("Saved new image at " + saveFilePath);
93-
} else if (savedImagesCount > currentCount) {
94-
// Reset local value with NT value in case of de-sync
95-
savedImagesCount = currentCount;
9693
}
94+
95+
savedImagesCount++;
96+
logger.info("Saved new image at " + saveFilePath);
97+
} else if (savedImagesCount > currentCount) {
98+
// Reset local value with NT value in case of de-sync
99+
savedImagesCount = currentCount;
97100
}
98101
}
99102

‎photon-core/src/main/java/org/photonvision/vision/frame/consumer/MJPGFrameConsumer.java

Lines changed: 8 additions & 102 deletions
Original file line numberDiff line numberDiff line change
@@ -21,102 +21,22 @@
2121
import edu.wpi.first.networktables.NetworkTable;
2222
import edu.wpi.first.networktables.NetworkTableInstance;
2323
import edu.wpi.first.util.PixelFormat;
24-
import java.awt.*;
2524
import java.util.ArrayList;
26-
import org.opencv.core.CvType;
27-
import org.opencv.core.Mat;
28-
import org.opencv.core.Point;
29-
import org.opencv.core.Rect;
30-
import org.opencv.imgproc.Imgproc;
31-
import org.photonvision.common.util.ColorHelper;
3225
import org.photonvision.common.util.math.MathUtils;
26+
import org.photonvision.vision.frame.StaticFrames;
3327
import org.photonvision.vision.opencv.CVMat;
3428

3529
public class MJPGFrameConsumer implements AutoCloseable {
3630
private static final double MAX_FRAMERATE = -1;
3731
private static final long MAX_FRAME_PERIOD_NS = Math.round(1e9 / MAX_FRAMERATE);
38-
private long lastFrameTimeNs;
39-
40-
private static final Mat EMPTY_MAT = new Mat(60, 15 * 7, CvType.CV_8UC3);
41-
private static final double EMPTY_FRAMERATE = 2;
42-
private static final long EMPTY_FRAME_PERIOD_NS = Math.round(1e9 / EMPTY_FRAMERATE);
43-
private long lastEmptyTimeNs;
44-
45-
static {
46-
EMPTY_MAT.setTo(ColorHelper.colorToScalar(Color.BLACK));
47-
var col = 0;
48-
Imgproc.rectangle(
49-
EMPTY_MAT,
50-
new Rect(col, 0, 15, EMPTY_MAT.height()),
51-
ColorHelper.colorToScalar(new Color(0xa2a2a2)),
52-
-1);
53-
col += 15;
54-
Imgproc.rectangle(
55-
EMPTY_MAT,
56-
new Rect(col, 0, 15, EMPTY_MAT.height()),
57-
ColorHelper.colorToScalar(new Color(0xa2a300)),
58-
-1);
59-
col += 15;
60-
Imgproc.rectangle(
61-
EMPTY_MAT,
62-
new Rect(col, 0, 15, EMPTY_MAT.height()),
63-
ColorHelper.colorToScalar(new Color(0x00a3a2)),
64-
-1);
65-
col += 15;
66-
Imgproc.rectangle(
67-
EMPTY_MAT,
68-
new Rect(col, 0, 15, EMPTY_MAT.height()),
69-
ColorHelper.colorToScalar(new Color(0x00a200)),
70-
-1);
71-
col += 15;
72-
Imgproc.rectangle(
73-
EMPTY_MAT,
74-
new Rect(col, 0, 15, EMPTY_MAT.height()),
75-
ColorHelper.colorToScalar(new Color(0x440045)),
76-
-1);
77-
col += 15;
78-
Imgproc.rectangle(
79-
EMPTY_MAT,
80-
new Rect(col, 0, 15, EMPTY_MAT.height()),
81-
ColorHelper.colorToScalar(new Color(0x0000a2)),
82-
-1);
83-
col += 15;
84-
Imgproc.rectangle(
85-
EMPTY_MAT,
86-
new Rect(col, 0, 15, EMPTY_MAT.height()),
87-
ColorHelper.colorToScalar(new Color(0)),
88-
-1);
89-
Imgproc.rectangle(
90-
EMPTY_MAT,
91-
new Rect(0, 50, EMPTY_MAT.width(), 10),
92-
ColorHelper.colorToScalar(new Color(0)),
93-
-1);
94-
Imgproc.rectangle(
95-
EMPTY_MAT, new Rect(15, 50, 30, 10), ColorHelper.colorToScalar(Color.WHITE), -1);
96-
97-
Imgproc.putText(
98-
EMPTY_MAT, "Stream", new Point(14, 20), 0, 0.6, ColorHelper.colorToScalar(Color.white), 2);
99-
Imgproc.putText(
100-
EMPTY_MAT,
101-
"Disabled",
102-
new Point(14, 45),
103-
0,
104-
0.6,
105-
ColorHelper.colorToScalar(Color.white),
106-
2);
107-
Imgproc.putText(
108-
EMPTY_MAT, "Stream", new Point(14, 20), 0, 0.6, ColorHelper.colorToScalar(Color.RED), 1);
109-
Imgproc.putText(
110-
EMPTY_MAT, "Disabled", new Point(14, 45), 0, 0.6, ColorHelper.colorToScalar(Color.RED), 1);
111-
}
11232

33+
private long lastFrameTimeNs;
11334
private CvSource cvSource;
11435
private MjpegServer mjpegServer;
11536

11637
private VideoListener listener;
11738

11839
private final NetworkTable table;
119-
boolean isDisabled = false;
12040

12141
public MJPGFrameConsumer(String sourceName, int width, int height, int port) {
12242
this.cvSource = new CvSource(sourceName, PixelFormat.kMJPEG, width, height, 30);
@@ -174,29 +94,15 @@ public MJPGFrameConsumer(String name, int port) {
17494
}
17595

17696
public void accept(CVMat image) {
177-
if (image != null && !image.getMat().empty()) {
178-
long now = MathUtils.wpiNanoTime();
179-
if (now - lastFrameTimeNs > MAX_FRAME_PERIOD_NS) {
180-
lastFrameTimeNs = now;
181-
cvSource.putFrame(image.getMat());
182-
}
183-
184-
// Make sure our disabled framerate limiting doesn't get confused
185-
isDisabled = false;
186-
lastEmptyTimeNs = 0;
187-
}
188-
}
97+
long now = MathUtils.wpiNanoTime();
18998

190-
public void disabledTick() {
191-
if (!isDisabled) {
192-
cvSource.setVideoMode(PixelFormat.kMJPEG, EMPTY_MAT.width(), EMPTY_MAT.height(), 0);
193-
isDisabled = true;
99+
if (image == null || image.getMat() == null || image.getMat().empty()) {
100+
image.copyFrom(StaticFrames.LOST_MAT);
194101
}
195102

196-
long now = MathUtils.wpiNanoTime();
197-
if (now - lastEmptyTimeNs > EMPTY_FRAME_PERIOD_NS) {
198-
lastEmptyTimeNs = now;
199-
cvSource.putFrame(EMPTY_MAT);
103+
if (now - lastFrameTimeNs > MAX_FRAME_PERIOD_NS) {
104+
lastFrameTimeNs = now;
105+
cvSource.putFrame(image.getMat());
200106
}
201107
}
202108

‎photon-core/src/main/java/org/photonvision/vision/processes/VisionRunner.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.photonvision.common.logging.LogGroup;
2323
import org.photonvision.common.logging.Logger;
2424
import org.photonvision.vision.camera.QuirkyCamera;
25+
import org.photonvision.vision.frame.Frame;
2526
import org.photonvision.vision.frame.FrameProvider;
2627
import org.photonvision.vision.pipe.impl.HSVPipe;
2728
import org.photonvision.vision.pipeline.AdvancedPipelineSettings;
@@ -95,6 +96,8 @@ private void update() {
9596
// Frame empty -- no point in trying to do anything more?
9697
if (frame.processedImage.getMat().empty() && frame.colorImage.getMat().empty()) {
9798
// give up without increasing loop count
99+
// Still feed with blank frames just dont run any pipelines
100+
pipelineResultConsumer.accept(new CVPipelineResult(0l, 0, 0, null, new Frame()));
98101
continue;
99102
}
100103

0 commit comments

Comments
 (0)