From 9ed0631ec962f5ab56541f2a1aebfef6fe107ec3 Mon Sep 17 00:00:00 2001 From: Thad House Date: Mon, 12 Feb 2024 23:42:17 -0800 Subject: [PATCH] [cscore] Add BGRA support (#6365) --- .../java/edu/wpi/first/cscore/CvSink.java | 5 + .../java/edu/wpi/first/cscore/CvSource.java | 112 +++++++++++-- cscore/src/main/native/cpp/Frame.cpp | 83 ++++++++++ cscore/src/main/native/cpp/Frame.h | 5 + cscore/src/main/native/cpp/Image.h | 5 + .../src/main/native/cpp/MjpegServerImpl.cpp | 3 + cscore/src/main/native/cpp/RawSourceImpl.cpp | 29 +--- cscore/src/main/native/cpp/SourceImpl.cpp | 14 ++ .../main/native/cpp/jni/CameraServerJNI.cpp | 1 - cscore/src/main/native/include/cscore_cpp.h | 3 +- cscore/src/main/native/include/cscore_cv.h | 155 +++++++++++++++--- .../src/main/native/linux/UsbCameraImpl.cpp | 4 + .../main/native/objcpp/UsbCameraDelegate.mm | 15 +- .../src/main/native/objcpp/UsbCameraImpl.mm | 1 - .../src/main/native/windows/UsbCameraImpl.cpp | 60 +------ .../java/edu/wpi/first/util/PixelFormat.java | 4 +- .../src/main/native/include/wpi/RawFrame.h | 1 + 17 files changed, 371 insertions(+), 129 deletions(-) diff --git a/cscore/src/main/java/edu/wpi/first/cscore/CvSink.java b/cscore/src/main/java/edu/wpi/first/cscore/CvSink.java index 922bd38abbd..a6db5978da9 100644 --- a/cscore/src/main/java/edu/wpi/first/cscore/CvSink.java +++ b/cscore/src/main/java/edu/wpi/first/cscore/CvSink.java @@ -36,11 +36,16 @@ private int getCVFormat(PixelFormat pixelFormat) { switch (pixelFormat) { case kYUYV: case kRGB565: + case kY16: + case kUYVY: type = CvType.CV_8UC2; break; case kBGR: type = CvType.CV_8UC3; break; + case kBGRA: + type = CvType.CV_8UC4; + break; case kGray: case kMJPEG: default: diff --git a/cscore/src/main/java/edu/wpi/first/cscore/CvSource.java b/cscore/src/main/java/edu/wpi/first/cscore/CvSource.java index 1fcf5ab21fe..f3bd147274e 100644 --- a/cscore/src/main/java/edu/wpi/first/cscore/CvSource.java +++ b/cscore/src/main/java/edu/wpi/first/cscore/CvSource.java @@ -40,16 +40,16 @@ public CvSource(String name, PixelFormat pixelFormat, int width, int height, int } /** - * Put an OpenCV image and notify sinks. + * Put an OpenCV image and notify sinks * - *

Only 8-bit single-channel or 3-channel (with BGR channel order) images are supported. If the - * format, depth or channel order is different, use Mat.convertTo() and/or cvtColor() to convert - * it first. + *

The image format is guessed from the number of channels. The channel mapping is as follows. + * 1: kGray 2: kYUYV 3: BGR 4: BGRA Any other channel numbers will throw an error. If your image + * is an in alternate format, use the overload that takes a PixelFormat. * - * @param image OpenCV image + * @param image OpenCV Image */ public void putFrame(Mat image) { - // We only support 8-bit images, convert if necessary + // We only support 8 bit channels boolean cleanupRequired = false; Mat finalImage; if (image.depth() == 0) { @@ -64,22 +64,64 @@ public void putFrame(Mat image) { int channels = finalImage.channels(); PixelFormat format; if (channels == 1) { + // 1 channel is assumed Graysacle format = PixelFormat.kGray; + } else if (channels == 2) { + // 2 channels is assumed YUYV + format = PixelFormat.kYUYV; } else if (channels == 3) { + // 3 channels is assumed BGR format = PixelFormat.kBGR; + } else if (channels == 4) { + // 4 channels is assumed BGRA + format = PixelFormat.kBGRA; } else { - throw new VideoException("Unsupported image type"); + throw new VideoException("Unable to get pixel format for " + channels + " channels"); + } + + putFrame(finalImage, format, true); + } finally { + if (cleanupRequired) { + finalImage.release(); } - // TODO old code supported BGRA, but the only way I can support that is slow. - // Update cscore to support BGRA for raw frames + } + } + + /** + * Put an OpenCV image and notify sinks. + * + *

The format of the Mat must match the PixelFormat. You will corrupt memory if they dont. With + * skipVerification false, we will verify the number of channels matches the pixel format. If + * skipVerification is true, this step is skipped and is passed straight through. + * + * @param image OpenCV image + * @param format The pixel format of the image + * @param skipVerification skip verifying pixel format + */ + public void putFrame(Mat image, PixelFormat format, boolean skipVerification) { + // We only support 8-bit images, convert if necessary + boolean cleanupRequired = false; + Mat finalImage; + if (image.depth() == 0) { + finalImage = image; + } else { + finalImage = new Mat(); + image.convertTo(finalImage, 0); + cleanupRequired = true; + } + try { + if (!skipVerification) { + verifyFormat(finalImage, format); + } + long step = image.step1() * image.elemSize1(); CameraServerJNI.putRawSourceFrameData( m_handle, finalImage.dataAddr(), - (int) finalImage.total() * channels, + (int) finalImage.total() * finalImage.channels(), finalImage.width(), finalImage.height(), - image.width(), + (int) step, format.getValue()); } finally { @@ -88,4 +130,52 @@ public void putFrame(Mat image) { } } } + + private void verifyFormat(Mat image, PixelFormat pixelFormat) { + int channels = image.channels(); + switch (pixelFormat) { + case kBGR: + if (channels == 3) { + return; + } + break; + case kBGRA: + if (channels == 4) { + return; + } + break; + case kGray: + if (channels == 1) { + return; + } + break; + case kRGB565: + if (channels == 2) { + return; + } + break; + case kUYVY: + if (channels == 2) { + return; + } + break; + case kY16: + if (channels == 2) { + return; + } + break; + case kYUYV: + if (channels == 2) { + return; + } + break; + case kMJPEG: + if (channels == 1) { + return; + } + break; + default: + break; + } + } } diff --git a/cscore/src/main/native/cpp/Frame.cpp b/cscore/src/main/native/cpp/Frame.cpp index 1a30393e595..af108f46a72 100644 --- a/cscore/src/main/native/cpp/Frame.cpp +++ b/cscore/src/main/native/cpp/Frame.cpp @@ -314,6 +314,54 @@ Image* Frame::ConvertImpl(Image* image, VideoMode::PixelFormat pixelFormat, } } break; + case VideoMode::kBGRA: + // If source is RGB565, YUYV, UYVY, Gray or Y16, need to convert to BGR + // first + if (cur->pixelFormat == VideoMode::kRGB565) { + // Check to see if BGR version already exists... + if (Image* newImage = + GetExistingImage(cur->width, cur->height, VideoMode::kBGR)) { + cur = newImage; + } else { + cur = ConvertRGB565ToBGR(cur); + } + } else if (cur->pixelFormat == VideoMode::kYUYV) { + // Check to see if BGR version already exists... + if (Image* newImage = + GetExistingImage(cur->width, cur->height, VideoMode::kBGR)) { + cur = newImage; + } else { + cur = ConvertYUYVToBGR(cur); + } + } else if (cur->pixelFormat == VideoMode::kUYVY) { + // Check to see if BGR version already exists... + if (Image* newImage = + GetExistingImage(cur->width, cur->height, VideoMode::kBGR)) { + cur = newImage; + } else { + cur = ConvertUYVYToBGR(cur); + } + } else if (cur->pixelFormat == VideoMode::kGray) { + // Check to see if BGR version already exists... + if (Image* newImage = + GetExistingImage(cur->width, cur->height, VideoMode::kBGR)) { + cur = newImage; + } else { + cur = ConvertGrayToBGR(cur); + } + } else if (cur->pixelFormat == VideoMode::kY16) { + // Check to see if BGR version already exists... + if (Image* newImage = + GetExistingImage(cur->width, cur->height, VideoMode::kBGR)) { + cur = newImage; + } else if (Image* newImage = GetExistingImage(cur->width, cur->height, + VideoMode::kGray)) { + cur = ConvertGrayToBGR(newImage); + } else { + cur = ConvertGrayToBGR(ConvertY16ToGray(cur)); + } + } + return ConvertBGRToBGRA(cur); case VideoMode::kYUYV: case VideoMode::kUYVY: default: @@ -664,6 +712,28 @@ Image* Frame::ConvertY16ToGray(Image* image) { return rv; } +Image* Frame::ConvertBGRToBGRA(Image* image) { + if (!image || image->pixelFormat != VideoMode::kBGR) { + return nullptr; + } + + // Allocate a RGB565 image + auto newImage = + m_impl->source.AllocImage(VideoMode::kBGRA, image->width, image->height, + image->width * image->height * 4); + + // Convert + cv::cvtColor(image->AsMat(), newImage->AsMat(), cv::COLOR_BGR2BGRA); + + // Save the result + Image* rv = newImage.release(); + if (m_impl) { + std::scoped_lock lock(m_impl->mutex); + m_impl->images.push_back(rv); + } + return rv; +} + Image* Frame::GetImageImpl(int width, int height, VideoMode::PixelFormat pixelFormat, int requiredJpegQuality, int defaultJpegQuality) { @@ -727,3 +797,16 @@ void Frame::ReleaseFrame() { m_impl->source.ReleaseFrameImpl(std::unique_ptr(m_impl)); m_impl = nullptr; } + +namespace cs { +std::unique_ptr CreateImageFromBGRA(cs::SourceImpl* source, size_t width, + size_t height, size_t stride, + const uint8_t* data) { + cv::Mat finalImage{static_cast(height), static_cast(width), CV_8UC4, + const_cast(data), stride}; + std::unique_ptr dest = source->AllocImage( + VideoMode::PixelFormat::kBGR, width, height, width * height * 3); + cv::cvtColor(finalImage, dest->AsMat(), cv::COLOR_BGRA2BGR); + return dest; +} +} // namespace cs diff --git a/cscore/src/main/native/cpp/Frame.h b/cscore/src/main/native/cpp/Frame.h index 61324636805..d835de9efc2 100644 --- a/cscore/src/main/native/cpp/Frame.h +++ b/cscore/src/main/native/cpp/Frame.h @@ -22,6 +22,10 @@ namespace cs { class SourceImpl; +std::unique_ptr CreateImageFromBGRA(cs::SourceImpl* source, size_t width, + size_t height, size_t stride, + const uint8_t* data); + class Frame { friend class SourceImpl; @@ -206,6 +210,7 @@ class Frame { Image* ConvertGrayToMJPEG(Image* image, int quality); Image* ConvertGrayToY16(Image* image); Image* ConvertY16ToGray(Image* image); + Image* ConvertBGRToBGRA(Image* image); Image* GetImage(int width, int height, VideoMode::PixelFormat pixelFormat) { if (pixelFormat == VideoMode::kMJPEG) { diff --git a/cscore/src/main/native/cpp/Image.h b/cscore/src/main/native/cpp/Image.h index b66a7f17421..27d34a7cf29 100644 --- a/cscore/src/main/native/cpp/Image.h +++ b/cscore/src/main/native/cpp/Image.h @@ -63,6 +63,9 @@ class Image { case VideoMode::kBGR: type = CV_8UC3; break; + case VideoMode::kBGRA: + type = CV_8UC4; + break; case VideoMode::kGray: case VideoMode::kMJPEG: default: @@ -81,6 +84,8 @@ class Image { return 2 * width; case VideoMode::kBGR: return 3 * width; + case VideoMode::kBGRA: + return 4 * width; case VideoMode::kGray: return width; case VideoMode::kMJPEG: diff --git a/cscore/src/main/native/cpp/MjpegServerImpl.cpp b/cscore/src/main/native/cpp/MjpegServerImpl.cpp index 82273e598da..c57dde57f90 100644 --- a/cscore/src/main/native/cpp/MjpegServerImpl.cpp +++ b/cscore/src/main/native/cpp/MjpegServerImpl.cpp @@ -457,6 +457,9 @@ void MjpegServerImpl::ConnThread::SendHTML(wpi::raw_ostream& os, case VideoMode::kBGR: os << "BGR"; break; + case VideoMode::kBGRA: + os << "BGRA"; + break; case VideoMode::kGray: os << "gray"; break; diff --git a/cscore/src/main/native/cpp/RawSourceImpl.cpp b/cscore/src/main/native/cpp/RawSourceImpl.cpp index b5c27803fe9..94da6d06aa3 100644 --- a/cscore/src/main/native/cpp/RawSourceImpl.cpp +++ b/cscore/src/main/native/cpp/RawSourceImpl.cpp @@ -22,31 +22,10 @@ RawSourceImpl::RawSourceImpl(std::string_view name, wpi::Logger& logger, RawSourceImpl::~RawSourceImpl() = default; void RawSourceImpl::PutFrame(const WPI_RawFrame& image) { - int type; - switch (image.pixelFormat) { - case VideoMode::kYUYV: - case VideoMode::kRGB565: - case VideoMode::kY16: - case VideoMode::kUYVY: - type = CV_8UC2; - break; - case VideoMode::kBGR: - type = CV_8UC3; - break; - case VideoMode::kGray: - case VideoMode::kMJPEG: - default: - type = CV_8UC1; - break; - } - cv::Mat finalImage{image.height, image.width, type, image.data, - static_cast(image.stride)}; - std::unique_ptr dest = - AllocImage(static_cast(image.pixelFormat), - image.width, image.height, image.size); - finalImage.copyTo(dest->AsMat()); - - SourceImpl::PutFrame(std::move(dest), wpi::Now()); + auto currentTime = wpi::Now(); + std::string_view data_view{reinterpret_cast(image.data), image.size}; + SourceImpl::PutFrame(static_cast(image.pixelFormat), + image.width, image.height, data_view, currentTime); } namespace cs { diff --git a/cscore/src/main/native/cpp/SourceImpl.cpp b/cscore/src/main/native/cpp/SourceImpl.cpp index 045bdd46923..e30821064ae 100644 --- a/cscore/src/main/native/cpp/SourceImpl.cpp +++ b/cscore/src/main/native/cpp/SourceImpl.cpp @@ -196,6 +196,8 @@ bool SourceImpl::SetConfigJson(const wpi::json& config, CS_Status* status) { mode.pixelFormat = cs::VideoMode::kRGB565; } else if (wpi::equals_lower(str, "bgr")) { mode.pixelFormat = cs::VideoMode::kBGR; + } else if (wpi::equals_lower(str, "bgra")) { + mode.pixelFormat = cs::VideoMode::kBGRA; } else if (wpi::equals_lower(str, "gray")) { mode.pixelFormat = cs::VideoMode::kGray; } else if (wpi::equals_lower(str, "y16")) { @@ -361,6 +363,9 @@ wpi::json SourceImpl::GetConfigJsonObject(CS_Status* status) { case VideoMode::kBGR: pixelFormat = "bgr"; break; + case VideoMode::kBGRA: + pixelFormat = "bgra"; + break; case VideoMode::kGray: pixelFormat = "gray"; break; @@ -450,6 +455,15 @@ std::unique_ptr SourceImpl::AllocImage( void SourceImpl::PutFrame(VideoMode::PixelFormat pixelFormat, int width, int height, std::string_view data, Frame::Time time) { + if (pixelFormat == VideoMode::PixelFormat::kBGRA) { + // Write BGRA as BGR to save a copy + auto image = + CreateImageFromBGRA(this, width, height, width * 4, + reinterpret_cast(data.data())); + PutFrame(std::move(image), time); + return; + } + auto image = AllocImage(pixelFormat, width, height, data.size()); // Copy in image data diff --git a/cscore/src/main/native/cpp/jni/CameraServerJNI.cpp b/cscore/src/main/native/cpp/jni/CameraServerJNI.cpp index e65e9ac6b51..6c133762122 100644 --- a/cscore/src/main/native/cpp/jni/CameraServerJNI.cpp +++ b/cscore/src/main/native/cpp/jni/CameraServerJNI.cpp @@ -12,7 +12,6 @@ #include #include -#include "cscore_cv.h" #include "cscore_raw.h" #include "cscore_runloop.h" #include "edu_wpi_first_cscore_CameraServerJNI.h" diff --git a/cscore/src/main/native/include/cscore_cpp.h b/cscore/src/main/native/include/cscore_cpp.h index 76c37fed915..b57faee0ffe 100644 --- a/cscore/src/main/native/include/cscore_cpp.h +++ b/cscore/src/main/native/include/cscore_cpp.h @@ -68,7 +68,8 @@ struct VideoMode : public CS_VideoMode { kBGR = WPI_PIXFMT_BGR, kGray = WPI_PIXFMT_GRAY, kY16 = WPI_PIXFMT_Y16, - kUYVY = WPI_PIXFMT_UYVY + kUYVY = WPI_PIXFMT_UYVY, + kBGRA = WPI_PIXFMT_BGRA, }; VideoMode() { pixelFormat = 0; diff --git a/cscore/src/main/native/include/cscore_cv.h b/cscore/src/main/native/include/cscore_cv.h index e623f6aea33..aec10d4a55c 100644 --- a/cscore/src/main/native/include/cscore_cv.h +++ b/cscore/src/main/native/include/cscore_cv.h @@ -44,15 +44,36 @@ class CvSource : public ImageSource { int height, int fps); /** - * Put an OpenCV image and notify sinks. + * Put an OpenCV image and notify sinks * - *

Only 8-bit single-channel or 3-channel (with BGR channel order) images - * are supported. If the format, depth or channel order is different, use - * cv::Mat::convertTo() and/or cv::cvtColor() to convert it first. + *

+ * The image format is guessed from the number of channels. The channel + * mapping is as follows. 1: kGray 2: kYUYV 3: BGR 4: BGRA Any other channel + * numbers will throw an error. If your image is an in alternate format, use + * the overload that takes a PixelFormat. * - * @param image OpenCV image + * @param image OpenCV Image */ void PutFrame(cv::Mat& image); + + /** + * Put an OpenCV image and notify sinks. + * + *

+ * The format of the Mat must match the PixelFormat. You will corrupt memory + * if they dont. With skipVerification false, we will verify the number of + * channels matches the pixel format. If skipVerification is true, this step + * is skipped and is passed straight through. + * + * @param image OpenCV image + * @param pixelFormat The pixel format of the image + * @param skipVerification skip verifying pixel format + */ + void PutFrame(cv::Mat& image, VideoMode::PixelFormat pixelFormat, + bool skipVerification); + + private: + static bool VerifyFormat(cv::Mat& image, VideoMode::PixelFormat pixelFormat); }; /** @@ -131,6 +152,8 @@ class CvSink : public ImageSink { uint64_t GrabFrameNoTimeoutDirect(cv::Mat& image); private: + constexpr int GetCvFormat(WPI_PixelFormat pixelFormat); + wpi::RawFrame rawFrame; VideoMode::PixelFormat pixelFormat; }; @@ -145,6 +168,56 @@ inline CvSource::CvSource(std::string_view name, VideoMode::PixelFormat format, &m_status); } +inline bool CvSource::VerifyFormat(cv::Mat& image, + VideoMode::PixelFormat pixelFormat) { + int channels = image.channels(); + switch (pixelFormat) { + case VideoMode::PixelFormat::kBGR: + if (channels == 3) { + return true; + } + break; + case VideoMode::PixelFormat::kBGRA: + if (channels == 4) { + return true; + } + break; + case VideoMode::PixelFormat::kGray: + if (channels == 1) { + return true; + } + break; + case VideoMode::PixelFormat::kRGB565: + if (channels == 2) { + return true; + } + break; + case VideoMode::PixelFormat::kUYVY: + if (channels == 2) { + return true; + } + break; + case VideoMode::PixelFormat::kY16: + if (channels == 2) { + return true; + } + break; + case VideoMode::PixelFormat::kYUYV: + if (channels == 2) { + return true; + } + break; + case VideoMode::PixelFormat::kMJPEG: + if (channels == 1) { + return true; + } + break; + default: + break; + } + return false; +} + inline void CvSource::PutFrame(cv::Mat& image) { // We only support 8-bit images; convert if necessary. cv::Mat finalImage; @@ -155,27 +228,54 @@ inline void CvSource::PutFrame(cv::Mat& image) { } int channels = finalImage.channels(); - WPI_PixelFormat format; + VideoMode::PixelFormat format; if (channels == 1) { - format = WPI_PIXFMT_GRAY; + // 1 channel is assumed Graysacle + format = VideoMode::PixelFormat::kGray; + } else if (channels == 2) { + // 2 channels is assumed YUYV + format = VideoMode::PixelFormat::kYUYV; } else if (channels == 3) { - format = WPI_PIXFMT_BGR; + // 3 channels is assumed BGR + format = VideoMode::PixelFormat::kBGR; + } else if (channels == 4) { + // 4 channels is assumed BGRA + format = VideoMode::PixelFormat::kBGRA; } else { // TODO Error return; } - // TODO old code supported BGRA, but the only way I can support that is slow. - // Update cscore to support BGRA for raw frames + + PutFrame(finalImage, format, true); +} + +inline void CvSource::PutFrame(cv::Mat& image, + VideoMode::PixelFormat pixelFormat, + bool skipVerification) { + // We only support 8-bit images; convert if necessary. + cv::Mat finalImage; + if (image.depth() == CV_8U) { + finalImage = image; + } else { + image.convertTo(finalImage, CV_8U); + } + + if (!skipVerification) { + if (!VerifyFormat(finalImage, pixelFormat)) { + // TODO Error + return; + } + } WPI_RawFrame frame; // use WPI_Frame because we don't want the destructor frame.data = finalImage.data; frame.freeFunc = nullptr; frame.freeCbData = nullptr; - frame.size = finalImage.total() * channels; + frame.size = finalImage.total() * finalImage.channels(); frame.width = finalImage.cols; frame.height = finalImage.rows; - frame.stride = finalImage.cols; - frame.pixelFormat = format; + frame.stride = finalImage.step; + frame.pixelFormat = pixelFormat; m_status = 0; PutSourceFrame(m_handle, frame, &m_status); } @@ -209,16 +309,21 @@ inline uint64_t CvSink::GrabFrameNoTimeout(cv::Mat& image) { return retVal; } -inline constexpr int GetCvFormat(WPI_PixelFormat pixelFormat) { +inline constexpr int CvSink::GetCvFormat(WPI_PixelFormat pixelFormat) { int type = 0; switch (pixelFormat) { case WPI_PIXFMT_YUYV: case WPI_PIXFMT_RGB565: + case WPI_PIXFMT_Y16: + case WPI_PIXFMT_UYVY: type = CV_8UC2; break; case WPI_PIXFMT_BGR: type = CV_8UC3; break; + case WPI_PIXFMT_BGRA: + type = CV_8UC4; + break; case WPI_PIXFMT_GRAY: case WPI_PIXFMT_MJPEG: default: @@ -231,31 +336,33 @@ inline constexpr int GetCvFormat(WPI_PixelFormat pixelFormat) { inline uint64_t CvSink::GrabFrameDirect(cv::Mat& image, double timeout) { rawFrame.height = 0; rawFrame.width = 0; + rawFrame.stride = 0; rawFrame.pixelFormat = pixelFormat; - m_status = GrabSinkFrameTimeout(m_handle, rawFrame, timeout, &m_status); - if (m_status <= 0) { - return m_status; + auto timestamp = GrabSinkFrameTimeout(m_handle, rawFrame, timeout, &m_status); + if (m_status != CS_OK) { + return 0; } image = cv::Mat{rawFrame.height, rawFrame.width, GetCvFormat(static_cast(rawFrame.pixelFormat)), - rawFrame.data}; - return m_status; + rawFrame.data, static_cast(rawFrame.stride)}; + return timestamp; } inline uint64_t CvSink::GrabFrameNoTimeoutDirect(cv::Mat& image) { rawFrame.height = 0; rawFrame.width = 0; + rawFrame.stride = 0; rawFrame.pixelFormat = pixelFormat; - m_status = GrabSinkFrame(m_handle, rawFrame, &m_status); - if (m_status <= 0) { - return m_status; + auto timestamp = GrabSinkFrame(m_handle, rawFrame, &m_status); + if (m_status != CS_OK) { + return 0; } image = cv::Mat{rawFrame.height, rawFrame.width, GetCvFormat(static_cast(rawFrame.pixelFormat)), - rawFrame.data}; - return m_status; + rawFrame.data, static_cast(rawFrame.stride)}; + return timestamp; } } // namespace cs diff --git a/cscore/src/main/native/linux/UsbCameraImpl.cpp b/cscore/src/main/native/linux/UsbCameraImpl.cpp index b50b6d934b9..eca6ab77975 100644 --- a/cscore/src/main/native/linux/UsbCameraImpl.cpp +++ b/cscore/src/main/native/linux/UsbCameraImpl.cpp @@ -80,6 +80,8 @@ static VideoMode::PixelFormat ToPixelFormat(__u32 pixelFormat) { return VideoMode::kRGB565; case V4L2_PIX_FMT_BGR24: return VideoMode::kBGR; + case V4L2_PIX_FMT_ABGR32: + return VideoMode::kBGRA; case V4L2_PIX_FMT_GREY: return VideoMode::kGray; case V4L2_PIX_FMT_Y16: @@ -102,6 +104,8 @@ static __u32 FromPixelFormat(VideoMode::PixelFormat pixelFormat) { return V4L2_PIX_FMT_RGB565; case VideoMode::kBGR: return V4L2_PIX_FMT_BGR24; + case VideoMode::kBGRA: + return V4L2_PIX_FMT_ABGR32; case VideoMode::kGray: return V4L2_PIX_FMT_GREY; case VideoMode::kY16: diff --git a/cscore/src/main/native/objcpp/UsbCameraDelegate.mm b/cscore/src/main/native/objcpp/UsbCameraDelegate.mm index efa0fbf6020..e3a2bfdf601 100644 --- a/cscore/src/main/native/objcpp/UsbCameraDelegate.mm +++ b/cscore/src/main/native/objcpp/UsbCameraDelegate.mm @@ -7,9 +7,6 @@ #include -#include -#include - @implementation UsbCameraDelegate - (id)init { @@ -24,6 +21,8 @@ - (void)captureOutput:(AVCaptureOutput*)captureOutput (void)sampleBuffer; (void)connection; + auto currentTime = wpi::Now(); + auto sharedThis = self.cppImpl.lock(); if (!sharedThis) { return; @@ -52,16 +51,12 @@ - (void)captureOutput:(AVCaptureOutput*)captureOutput return; } - size_t currSize = width * 3 * height; - - auto tmpMat = cv::Mat(height, width, CV_8UC4, baseaddress, rowBytes); - auto image = sharedThis->AllocImage(cs::VideoMode::PixelFormat::kBGR, width, - height, currSize); - cv::cvtColor(tmpMat, image->AsMat(), cv::COLOR_BGRA2BGR); + std::unique_ptr image = cs::CreateImageFromBGRA( + sharedThis.get(), width, height, rowBytes, reinterpret_cast(baseaddress)); CVPixelBufferUnlockBaseAddress(imageBuffer, 0); - sharedThis->objcPutFrame(std::move(image), wpi::Now()); + sharedThis->objcPutFrame(std::move(image), currentTime); } @end diff --git a/cscore/src/main/native/objcpp/UsbCameraImpl.mm b/cscore/src/main/native/objcpp/UsbCameraImpl.mm index ce0744a45cd..b0dfc330a00 100644 --- a/cscore/src/main/native/objcpp/UsbCameraImpl.mm +++ b/cscore/src/main/native/objcpp/UsbCameraImpl.mm @@ -18,7 +18,6 @@ #include "Instance.h" #include "c_util.h" #include "cscore_cpp.h" -#include "opencv2/imgproc.hpp" #include "UsbCameraImpl.h" namespace cs { diff --git a/cscore/src/main/native/windows/UsbCameraImpl.cpp b/cscore/src/main/native/windows/UsbCameraImpl.cpp index cde8c2556b8..606ceb2145d 100644 --- a/cscore/src/main/native/windows/UsbCameraImpl.cpp +++ b/cscore/src/main/native/windows/UsbCameraImpl.cpp @@ -21,9 +21,6 @@ #include #include #include -#include -#include -#include #include #include #include @@ -284,6 +281,8 @@ void UsbCameraImpl::ProcessFrame(IMFSample* videoSample, return; } + auto currentTime = wpi::Now(); + ComPtr buf; if (!SUCCEEDED(videoSample->ConvertToContiguousBuffer(buf.GetAddressOf()))) { @@ -339,56 +338,9 @@ void UsbCameraImpl::ProcessFrame(IMFSample* videoSample, } } - cv::Mat tmpMat; - std::unique_ptr dest; - bool doFinalSet = true; - - switch (mode.pixelFormat) { - case cs::VideoMode::PixelFormat::kMJPEG: { - // Special case - PutFrame(VideoMode::kMJPEG, mode.width, mode.height, - {reinterpret_cast(ptr), length}, wpi::Now()); - doFinalSet = false; - break; - } - case cs::VideoMode::PixelFormat::kGray: - tmpMat = cv::Mat(mode.height, mode.width, CV_8UC1, ptr, pitch); - dest = AllocImage(VideoMode::kGray, tmpMat.cols, tmpMat.rows, - tmpMat.total()); - tmpMat.copyTo(dest->AsMat()); - break; - case cs::VideoMode::PixelFormat::kY16: - tmpMat = cv::Mat(mode.height, mode.width, CV_8UC2, ptr, pitch); - dest = - AllocImage(VideoMode::kY16, tmpMat.cols, tmpMat.rows, tmpMat.total()); - tmpMat.copyTo(dest->AsMat()); - break; - case cs::VideoMode::PixelFormat::kBGR: - tmpMat = cv::Mat(mode.height, mode.width, CV_8UC3, ptr, pitch); - dest = AllocImage(VideoMode::kBGR, tmpMat.cols, tmpMat.rows, - tmpMat.total() * 3); - tmpMat.copyTo(dest->AsMat()); - break; - case cs::VideoMode::PixelFormat::kYUYV: - tmpMat = cv::Mat(mode.height, mode.width, CV_8UC2, ptr, pitch); - dest = AllocImage(VideoMode::kYUYV, tmpMat.cols, tmpMat.rows, - tmpMat.total() * 2); - tmpMat.copyTo(dest->AsMat()); - break; - case cs::VideoMode::PixelFormat::kUYVY: - tmpMat = cv::Mat(mode.height, mode.width, CV_8UC2, ptr, pitch); - dest = AllocImage(VideoMode::kUYVY, tmpMat.cols, tmpMat.rows, - tmpMat.total() * 2); - tmpMat.copyTo(dest->AsMat()); - break; - default: - doFinalSet = false; - break; - } - - if (doFinalSet) { - PutFrame(std::move(dest), wpi::Now()); - } + std::string_view data_view{reinterpret_cast(ptr), length}; + SourceImpl::PutFrame(static_cast(mode.pixelFormat), + mode.width, mode.height, data_view, currentTime); if (buffer2d) { buffer2d->Unlock2D(); @@ -480,8 +432,6 @@ static cs::VideoMode::PixelFormat GetFromGUID(const GUID& guid) { return cs::VideoMode::PixelFormat::kY16; } else if (IsEqualGUID(guid, MFVideoFormat_YUY2)) { return cs::VideoMode::PixelFormat::kYUYV; - } else if (IsEqualGUID(guid, MFVideoFormat_RGB24)) { - return cs::VideoMode::PixelFormat::kBGR; } else if (IsEqualGUID(guid, MFVideoFormat_MJPG)) { return cs::VideoMode::PixelFormat::kMJPEG; } else if (IsEqualGUID(guid, MFVideoFormat_RGB565)) { diff --git a/wpiutil/src/main/java/edu/wpi/first/util/PixelFormat.java b/wpiutil/src/main/java/edu/wpi/first/util/PixelFormat.java index 809bfd0361b..9951f8683ae 100644 --- a/wpiutil/src/main/java/edu/wpi/first/util/PixelFormat.java +++ b/wpiutil/src/main/java/edu/wpi/first/util/PixelFormat.java @@ -21,7 +21,9 @@ public enum PixelFormat { /** Grayscale, 16 bpp. */ kY16(6), /** YUV 4:2:2, 16 bpp. */ - kUYVY(7); + kUYVY(7), + /** BGRA 8-8-8-8. 32 bpp. */ + kBGRA(8); private final int value; diff --git a/wpiutil/src/main/native/include/wpi/RawFrame.h b/wpiutil/src/main/native/include/wpi/RawFrame.h index 1345ff134e0..2e925b1e0ba 100644 --- a/wpiutil/src/main/native/include/wpi/RawFrame.h +++ b/wpiutil/src/main/native/include/wpi/RawFrame.h @@ -55,6 +55,7 @@ enum WPI_PixelFormat { WPI_PIXFMT_GRAY, // Grayscale, 8 bpp WPI_PIXFMT_Y16, // Grayscale, 16 bpp WPI_PIXFMT_UYVY, // YUV 4:2:2, 16 bpp + WPI_PIXFMT_BGRA, // BGRA 8-8-8-8-, 32 bpp }; // Returns nonzero if the frame data was allocated/reallocated