Only 8-bit single-channel or 3-channel (with BGR channel order) images are supported. If the - * format, depth or channel order is different, use Mat.convertTo() and/or cvtColor() to convert - * it first. + *
The image format is guessed from the number of channels. The channel mapping is as follows. + * 1: kGray 2: kYUYV 3: BGR 4: BGRA Any other channel numbers will throw an error. If your image + * is an in alternate format, use the overload that takes a PixelFormat. * - * @param image OpenCV image + * @param image OpenCV Image */ public void putFrame(Mat image) { - // We only support 8-bit images, convert if necessary + // We only support 8 bit channels boolean cleanupRequired = false; Mat finalImage; if (image.depth() == 0) { @@ -64,22 +64,64 @@ public void putFrame(Mat image) { int channels = finalImage.channels(); PixelFormat format; if (channels == 1) { + // 1 channel is assumed Graysacle format = PixelFormat.kGray; + } else if (channels == 2) { + // 2 channels is assumed YUYV + format = PixelFormat.kYUYV; } else if (channels == 3) { + // 3 channels is assumed BGR format = PixelFormat.kBGR; + } else if (channels == 4) { + // 4 channels is assumed BGRA + format = PixelFormat.kBGRA; } else { - throw new VideoException("Unsupported image type"); + throw new VideoException("Unable to get pixel format for " + channels + " channels"); + } + + putFrame(finalImage, format, true); + } finally { + if (cleanupRequired) { + finalImage.release(); } - // TODO old code supported BGRA, but the only way I can support that is slow. - // Update cscore to support BGRA for raw frames + } + } + + /** + * Put an OpenCV image and notify sinks. + * + *
The format of the Mat must match the PixelFormat. You will corrupt memory if they dont. With
+ * skipVerification false, we will verify the number of channels matches the pixel format. If
+ * skipVerification is true, this step is skipped and is passed straight through.
+ *
+ * @param image OpenCV image
+ * @param format The pixel format of the image
+ * @param skipVerification skip verifying pixel format
+ */
+ public void putFrame(Mat image, PixelFormat format, boolean skipVerification) {
+ // We only support 8-bit images, convert if necessary
+ boolean cleanupRequired = false;
+ Mat finalImage;
+ if (image.depth() == 0) {
+ finalImage = image;
+ } else {
+ finalImage = new Mat();
+ image.convertTo(finalImage, 0);
+ cleanupRequired = true;
+ }
+ try {
+ if (!skipVerification) {
+ verifyFormat(finalImage, format);
+ }
+ long step = image.step1() * image.elemSize1();
CameraServerJNI.putRawSourceFrameData(
m_handle,
finalImage.dataAddr(),
- (int) finalImage.total() * channels,
+ (int) finalImage.total() * finalImage.channels(),
finalImage.width(),
finalImage.height(),
- image.width(),
+ (int) step,
format.getValue());
} finally {
@@ -88,4 +130,52 @@ public void putFrame(Mat image) {
}
}
}
+
+ private void verifyFormat(Mat image, PixelFormat pixelFormat) {
+ int channels = image.channels();
+ switch (pixelFormat) {
+ case kBGR:
+ if (channels == 3) {
+ return;
+ }
+ break;
+ case kBGRA:
+ if (channels == 4) {
+ return;
+ }
+ break;
+ case kGray:
+ if (channels == 1) {
+ return;
+ }
+ break;
+ case kRGB565:
+ if (channels == 2) {
+ return;
+ }
+ break;
+ case kUYVY:
+ if (channels == 2) {
+ return;
+ }
+ break;
+ case kY16:
+ if (channels == 2) {
+ return;
+ }
+ break;
+ case kYUYV:
+ if (channels == 2) {
+ return;
+ }
+ break;
+ case kMJPEG:
+ if (channels == 1) {
+ return;
+ }
+ break;
+ default:
+ break;
+ }
+ }
}
diff --git a/cscore/src/main/native/cpp/Frame.cpp b/cscore/src/main/native/cpp/Frame.cpp
index 1a30393e595..af108f46a72 100644
--- a/cscore/src/main/native/cpp/Frame.cpp
+++ b/cscore/src/main/native/cpp/Frame.cpp
@@ -314,6 +314,54 @@ Image* Frame::ConvertImpl(Image* image, VideoMode::PixelFormat pixelFormat,
}
}
break;
+ case VideoMode::kBGRA:
+ // If source is RGB565, YUYV, UYVY, Gray or Y16, need to convert to BGR
+ // first
+ if (cur->pixelFormat == VideoMode::kRGB565) {
+ // Check to see if BGR version already exists...
+ if (Image* newImage =
+ GetExistingImage(cur->width, cur->height, VideoMode::kBGR)) {
+ cur = newImage;
+ } else {
+ cur = ConvertRGB565ToBGR(cur);
+ }
+ } else if (cur->pixelFormat == VideoMode::kYUYV) {
+ // Check to see if BGR version already exists...
+ if (Image* newImage =
+ GetExistingImage(cur->width, cur->height, VideoMode::kBGR)) {
+ cur = newImage;
+ } else {
+ cur = ConvertYUYVToBGR(cur);
+ }
+ } else if (cur->pixelFormat == VideoMode::kUYVY) {
+ // Check to see if BGR version already exists...
+ if (Image* newImage =
+ GetExistingImage(cur->width, cur->height, VideoMode::kBGR)) {
+ cur = newImage;
+ } else {
+ cur = ConvertUYVYToBGR(cur);
+ }
+ } else if (cur->pixelFormat == VideoMode::kGray) {
+ // Check to see if BGR version already exists...
+ if (Image* newImage =
+ GetExistingImage(cur->width, cur->height, VideoMode::kBGR)) {
+ cur = newImage;
+ } else {
+ cur = ConvertGrayToBGR(cur);
+ }
+ } else if (cur->pixelFormat == VideoMode::kY16) {
+ // Check to see if BGR version already exists...
+ if (Image* newImage =
+ GetExistingImage(cur->width, cur->height, VideoMode::kBGR)) {
+ cur = newImage;
+ } else if (Image* newImage = GetExistingImage(cur->width, cur->height,
+ VideoMode::kGray)) {
+ cur = ConvertGrayToBGR(newImage);
+ } else {
+ cur = ConvertGrayToBGR(ConvertY16ToGray(cur));
+ }
+ }
+ return ConvertBGRToBGRA(cur);
case VideoMode::kYUYV:
case VideoMode::kUYVY:
default:
@@ -664,6 +712,28 @@ Image* Frame::ConvertY16ToGray(Image* image) {
return rv;
}
+Image* Frame::ConvertBGRToBGRA(Image* image) {
+ if (!image || image->pixelFormat != VideoMode::kBGR) {
+ return nullptr;
+ }
+
+ // Allocate a RGB565 image
+ auto newImage =
+ m_impl->source.AllocImage(VideoMode::kBGRA, image->width, image->height,
+ image->width * image->height * 4);
+
+ // Convert
+ cv::cvtColor(image->AsMat(), newImage->AsMat(), cv::COLOR_BGR2BGRA);
+
+ // Save the result
+ Image* rv = newImage.release();
+ if (m_impl) {
+ std::scoped_lock lock(m_impl->mutex);
+ m_impl->images.push_back(rv);
+ }
+ return rv;
+}
+
Image* Frame::GetImageImpl(int width, int height,
VideoMode::PixelFormat pixelFormat,
int requiredJpegQuality, int defaultJpegQuality) {
@@ -727,3 +797,16 @@ void Frame::ReleaseFrame() {
m_impl->source.ReleaseFrameImpl(std::unique_ptr Only 8-bit single-channel or 3-channel (with BGR channel order) images
- * are supported. If the format, depth or channel order is different, use
- * cv::Mat::convertTo() and/or cv::cvtColor() to convert it first.
+ *
+ * The image format is guessed from the number of channels. The channel
+ * mapping is as follows. 1: kGray 2: kYUYV 3: BGR 4: BGRA Any other channel
+ * numbers will throw an error. If your image is an in alternate format, use
+ * the overload that takes a PixelFormat.
*
- * @param image OpenCV image
+ * @param image OpenCV Image
*/
void PutFrame(cv::Mat& image);
+
+ /**
+ * Put an OpenCV image and notify sinks.
+ *
+ *
+ * The format of the Mat must match the PixelFormat. You will corrupt memory
+ * if they dont. With skipVerification false, we will verify the number of
+ * channels matches the pixel format. If skipVerification is true, this step
+ * is skipped and is passed straight through.
+ *
+ * @param image OpenCV image
+ * @param pixelFormat The pixel format of the image
+ * @param skipVerification skip verifying pixel format
+ */
+ void PutFrame(cv::Mat& image, VideoMode::PixelFormat pixelFormat,
+ bool skipVerification);
+
+ private:
+ static bool VerifyFormat(cv::Mat& image, VideoMode::PixelFormat pixelFormat);
};
/**
@@ -131,6 +152,8 @@ class CvSink : public ImageSink {
uint64_t GrabFrameNoTimeoutDirect(cv::Mat& image);
private:
+ constexpr int GetCvFormat(WPI_PixelFormat pixelFormat);
+
wpi::RawFrame rawFrame;
VideoMode::PixelFormat pixelFormat;
};
@@ -145,6 +168,56 @@ inline CvSource::CvSource(std::string_view name, VideoMode::PixelFormat format,
&m_status);
}
+inline bool CvSource::VerifyFormat(cv::Mat& image,
+ VideoMode::PixelFormat pixelFormat) {
+ int channels = image.channels();
+ switch (pixelFormat) {
+ case VideoMode::PixelFormat::kBGR:
+ if (channels == 3) {
+ return true;
+ }
+ break;
+ case VideoMode::PixelFormat::kBGRA:
+ if (channels == 4) {
+ return true;
+ }
+ break;
+ case VideoMode::PixelFormat::kGray:
+ if (channels == 1) {
+ return true;
+ }
+ break;
+ case VideoMode::PixelFormat::kRGB565:
+ if (channels == 2) {
+ return true;
+ }
+ break;
+ case VideoMode::PixelFormat::kUYVY:
+ if (channels == 2) {
+ return true;
+ }
+ break;
+ case VideoMode::PixelFormat::kY16:
+ if (channels == 2) {
+ return true;
+ }
+ break;
+ case VideoMode::PixelFormat::kYUYV:
+ if (channels == 2) {
+ return true;
+ }
+ break;
+ case VideoMode::PixelFormat::kMJPEG:
+ if (channels == 1) {
+ return true;
+ }
+ break;
+ default:
+ break;
+ }
+ return false;
+}
+
inline void CvSource::PutFrame(cv::Mat& image) {
// We only support 8-bit images; convert if necessary.
cv::Mat finalImage;
@@ -155,27 +228,54 @@ inline void CvSource::PutFrame(cv::Mat& image) {
}
int channels = finalImage.channels();
- WPI_PixelFormat format;
+ VideoMode::PixelFormat format;
if (channels == 1) {
- format = WPI_PIXFMT_GRAY;
+ // 1 channel is assumed Graysacle
+ format = VideoMode::PixelFormat::kGray;
+ } else if (channels == 2) {
+ // 2 channels is assumed YUYV
+ format = VideoMode::PixelFormat::kYUYV;
} else if (channels == 3) {
- format = WPI_PIXFMT_BGR;
+ // 3 channels is assumed BGR
+ format = VideoMode::PixelFormat::kBGR;
+ } else if (channels == 4) {
+ // 4 channels is assumed BGRA
+ format = VideoMode::PixelFormat::kBGRA;
} else {
// TODO Error
return;
}
- // TODO old code supported BGRA, but the only way I can support that is slow.
- // Update cscore to support BGRA for raw frames
+
+ PutFrame(finalImage, format, true);
+}
+
+inline void CvSource::PutFrame(cv::Mat& image,
+ VideoMode::PixelFormat pixelFormat,
+ bool skipVerification) {
+ // We only support 8-bit images; convert if necessary.
+ cv::Mat finalImage;
+ if (image.depth() == CV_8U) {
+ finalImage = image;
+ } else {
+ image.convertTo(finalImage, CV_8U);
+ }
+
+ if (!skipVerification) {
+ if (!VerifyFormat(finalImage, pixelFormat)) {
+ // TODO Error
+ return;
+ }
+ }
WPI_RawFrame frame; // use WPI_Frame because we don't want the destructor
frame.data = finalImage.data;
frame.freeFunc = nullptr;
frame.freeCbData = nullptr;
- frame.size = finalImage.total() * channels;
+ frame.size = finalImage.total() * finalImage.channels();
frame.width = finalImage.cols;
frame.height = finalImage.rows;
- frame.stride = finalImage.cols;
- frame.pixelFormat = format;
+ frame.stride = finalImage.step;
+ frame.pixelFormat = pixelFormat;
m_status = 0;
PutSourceFrame(m_handle, frame, &m_status);
}
@@ -209,16 +309,21 @@ inline uint64_t CvSink::GrabFrameNoTimeout(cv::Mat& image) {
return retVal;
}
-inline constexpr int GetCvFormat(WPI_PixelFormat pixelFormat) {
+inline constexpr int CvSink::GetCvFormat(WPI_PixelFormat pixelFormat) {
int type = 0;
switch (pixelFormat) {
case WPI_PIXFMT_YUYV:
case WPI_PIXFMT_RGB565:
+ case WPI_PIXFMT_Y16:
+ case WPI_PIXFMT_UYVY:
type = CV_8UC2;
break;
case WPI_PIXFMT_BGR:
type = CV_8UC3;
break;
+ case WPI_PIXFMT_BGRA:
+ type = CV_8UC4;
+ break;
case WPI_PIXFMT_GRAY:
case WPI_PIXFMT_MJPEG:
default:
@@ -231,31 +336,33 @@ inline constexpr int GetCvFormat(WPI_PixelFormat pixelFormat) {
inline uint64_t CvSink::GrabFrameDirect(cv::Mat& image, double timeout) {
rawFrame.height = 0;
rawFrame.width = 0;
+ rawFrame.stride = 0;
rawFrame.pixelFormat = pixelFormat;
- m_status = GrabSinkFrameTimeout(m_handle, rawFrame, timeout, &m_status);
- if (m_status <= 0) {
- return m_status;
+ auto timestamp = GrabSinkFrameTimeout(m_handle, rawFrame, timeout, &m_status);
+ if (m_status != CS_OK) {
+ return 0;
}
image =
cv::Mat{rawFrame.height, rawFrame.width,
GetCvFormat(static_cast