From d16e05df8c805b283e4e99254200731a2b730103 Mon Sep 17 00:00:00 2001 From: Sam Carlberg Date: Wed, 8 Nov 2017 23:04:57 -0500 Subject: [PATCH 01/10] Use cscore for PublishVideoOperation Updates JavaCV to 1.3.3 and javacpp-opencv to 3.2.0 --- build.gradle | 15 +- .../composite/PublishVideoOperation.java | 188 +++++++----------- .../composite/SaveImageOperation.java | 4 +- .../core/operations/opencv/MinMaxLoc.java | 9 +- .../grip/core/sources/CameraSourceTest.java | 2 +- 5 files changed, 94 insertions(+), 124 deletions(-) diff --git a/build.gradle b/build.gradle index e1ba42be5e..a736357745 100644 --- a/build.gradle +++ b/build.gradle @@ -225,10 +225,10 @@ project(":core") { dependencies { compile group: 'com.google.code.findbugs', name: 'jsr305', version: '3.0.1' - compile group: 'org.bytedeco', name: 'javacv', version: '1.1' - compile group: 'org.bytedeco.javacpp-presets', name: 'opencv', version: '3.0.0-1.1' - compile group: 'org.bytedeco.javacpp-presets', name: 'opencv', version: '3.0.0-1.1', classifier: os - compile group: 'org.bytedeco.javacpp-presets', name: 'opencv-3.0.0-1.1', classifier: 'linux-frc' + compile group: 'org.bytedeco', name: 'javacv', version: '1.3.3' + compile group: 'org.bytedeco.javacpp-presets', name: 'opencv', version: '3.2.0-1.3' + compile group: 'org.bytedeco.javacpp-presets', name: 'opencv', version: '3.2.0-1.3', classifier: os + //compile group: 'org.bytedeco.javacpp-presets', name: 'opencv-3.0.0-1.1', classifier: 'linux-frc' compile group: 'org.bytedeco.javacpp-presets', name: 'videoinput', version: '0.200-1.1', classifier: os compile group: 'org.bytedeco.javacpp-presets', name: 'ffmpeg', version: '0.200-1.1', classifier: os compile group: 'org.python', name: 'jython', version: '2.7.0' @@ -253,6 +253,13 @@ project(":core") { compile group: 'org.ros.rosjava_messages', name: 'grip_msgs', version: '0.0.1' compile group: 'edu.wpi.first.wpilib.networktables.java', name: 'NetworkTables', version: '3.1.2', classifier: 'desktop' compile group: 'edu.wpi.first.wpilib.networktables.java', name: 'NetworkTables', version: '3.1.2', classifier: 'arm' + + // cscore dependencies + compile group: 'edu.wpi.first.cscore', name: 'cscore-java', version: '1.1.0-beta-2' + compile group: 'edu.wpi.first.cscore', name: 'cscore-jni', version: '1.1.0-beta-2', classifier: 'all' + runtime group: 'edu.wpi.first.wpiutil', name: 'wpiutil-java', version: '3.+' + compile group: 'org.opencv', name: 'opencv-java', version: '3.2.0' + compile group: 'org.opencv', name: 'opencv-jni', version: '3.2.0', classifier: 'all' } mainClassName = 'edu.wpi.grip.core.Main' diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java index a30153bb9b..bd9aa7d2be 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java @@ -10,34 +10,42 @@ import com.google.common.collect.ImmutableList; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import edu.wpi.cscore.CameraServerJNI; +import edu.wpi.cscore.CvSource; +import edu.wpi.cscore.MjpegServer; +import edu.wpi.cscore.VideoMode; +import edu.wpi.first.wpilibj.networktables.NetworkTable; -import org.bytedeco.javacpp.BytePointer; -import org.bytedeco.javacpp.IntPointer; +import org.bytedeco.javacpp.opencv_core; +import org.opencv.core.Mat; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.net.ServerSocket; -import java.net.Socket; +import java.lang.reflect.Field; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; -import static org.bytedeco.javacpp.opencv_core.Mat; -import static org.bytedeco.javacpp.opencv_imgcodecs.CV_IMWRITE_JPEG_QUALITY; -import static org.bytedeco.javacpp.opencv_imgcodecs.imencode; +import static org.bytedeco.javacpp.opencv_core.CV_8S; +import static org.bytedeco.javacpp.opencv_core.CV_8U; /** * Publish an M-JPEG stream with the protocol used by SmartDashboard and the FRC Dashboard. This * allows FRC teams to view video streams on their dashboard during competition even when GRIP has - * exclusive access to the camera. In addition, an intermediate processed image in the pipeline - * could be published instead. Based on WPILib's CameraServer class: - * https://github.com/robotpy/allwpilib/blob/master/wpilibj/src/athena/java/edu/wpi/first/wpilibj - * /CameraServer.java + * exclusive access to the camera. Uses cscore to host the image streaming server. */ public class PublishVideoOperation implements Operation { private static final Logger logger = Logger.getLogger(PublishVideoOperation.class.getName()); + + static { + try { + // Loading the CameraServerJNI class will load the appropriate platform-specific OpenCV JNI + CameraServerJNI.getHostname(); + } catch (Throwable e) { + logger.log(Level.SEVERE, "CameraServerJNI load failed! Exiting", e); + System.exit(31); + } + } + public static final OperationDescription DESCRIPTION = OperationDescription.builder() .name("Publish Video") @@ -46,110 +54,40 @@ public class PublishVideoOperation implements Operation { .icon(Icon.iconStream("publish-video")) .build(); private static final int PORT = 1180; - private static final byte[] MAGIC_NUMBER = {0x01, 0x00, 0x00, 0x00}; @SuppressWarnings("PMD.AssignmentToNonFinalStatic") private static int numSteps; - private final Object imageLock = new Object(); - private final BytePointer imagePointer = new BytePointer(); - private final Thread serverThread; - private final InputSocket inputSocket; + private static final int MAX_STEP_COUNT = 10; + + private final InputSocket inputSocket; private final InputSocket qualitySocket; - @SuppressWarnings("PMD.SingularField") - private volatile boolean connected = false; - /** - * Listens for incoming connections on port 1180 and writes JPEG data whenever there's a new - * frame. - */ - private final Runnable runServer = () -> { - // Loop forever (or at least until the thread is interrupted). This lets us recover from the - // dashboard - // disconnecting or the network connection going away temporarily. - while (!Thread.currentThread().isInterrupted()) { - try (ServerSocket serverSocket = new ServerSocket(PORT)) { - logger.info("Starting camera server"); - - try (Socket socket = serverSocket.accept()) { - logger.info("Got connection from " + socket.getInetAddress()); - connected = true; - - DataOutputStream socketOutputStream = new DataOutputStream(socket.getOutputStream()); - DataInputStream socketInputStream = new DataInputStream(socket.getInputStream()); - - byte[] buffer = new byte[128 * 1024]; - int bufferSize; - - final int fps = socketInputStream.readInt(); - final int compression = socketInputStream.readInt(); - final int size = socketInputStream.readInt(); - - if (compression != -1) { - logger.warning("Dashboard video should be in HW mode"); - } - - final long frameDuration = 1000000000L / fps; - long startTime = System.nanoTime(); - - while (!socket.isClosed() && !Thread.currentThread().isInterrupted()) { - // Wait for the main thread to put a new image. This happens whenever perform() is - // called with - // a new input. - synchronized (imageLock) { - imageLock.wait(); - - // Copy the image data into a pre-allocated buffer, growing it if necessary - bufferSize = imagePointer.limit(); - if (bufferSize > buffer.length) { - buffer = new byte[imagePointer.limit()]; - } - imagePointer.get(buffer, 0, bufferSize); - } - - // The FRC dashboard image protocol consists of a magic number, the size of the image - // data, - // and the image data itself. - socketOutputStream.write(MAGIC_NUMBER); - socketOutputStream.writeInt(bufferSize); - socketOutputStream.write(buffer, 0, bufferSize); - - // Limit the FPS to whatever the dashboard requested - int remainingTime = (int) (frameDuration - (System.nanoTime() - startTime)); - if (remainingTime > 0) { - Thread.sleep(remainingTime / 1000000, remainingTime % 1000000); - } - - startTime = System.nanoTime(); - } - } - } catch (IOException e) { - logger.log(Level.WARNING, e.getMessage(), e); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); // This is really unnecessary since the thread is - // about to exit - logger.info("Shutting down camera server"); - return; - } finally { - connected = false; - } - } - }; + private final MjpegServer server; + private final CvSource serverSource; + private static final NetworkTable cameraPublisherTable = + NetworkTable.getTable("/CameraPublisher"); + private final Mat publishMat = new Mat(); + private long lastFrame = -1; @SuppressWarnings("JavadocMethod") @SuppressFBWarnings(value = "ST_WRITE_TO_STATIC_FROM_INSTANCE_METHOD", justification = "Do not need to synchronize inside of a constructor") public PublishVideoOperation(InputSocket.Factory inputSocketFactory) { - if (numSteps != 0) { - throw new IllegalStateException("Only one instance of PublishVideoOperation may exist"); + if (numSteps >= MAX_STEP_COUNT) { + throw new IllegalStateException( + "Only " + MAX_STEP_COUNT + " instances of PublishVideoOperation may exist"); } this.inputSocket = inputSocketFactory.create(SocketHints.Inputs.createMatSocketHint("Image", false)); this.qualitySocket = inputSocketFactory.create(SocketHints.Inputs .createNumberSliderSocketHint("Quality", 80, 0, 100)); - numSteps++; - serverThread = new Thread(runServer, "Camera Server"); - serverThread.setDaemon(true); - serverThread.start(); + server = new MjpegServer("GRIP server " + numSteps, PORT + numSteps); + serverSource = new CvSource("GRIP CvSource" + numSteps, VideoMode.PixelFormat.kMJPEG, 0, 0, 0); + server.setSource(serverSource); + cameraPublisherTable.putStringArray("streams", + new String[]{CameraServerJNI.getHostname() + ":" + server.getPort()}); + + numSteps++; } @Override @@ -167,25 +105,49 @@ public List getOutputSockets() { @Override public void perform() { - if (!connected) { - return; // Don't waste any time converting images if there's no dashboard connected - } - - if (inputSocket.getValue().get().empty()) { + final long now = System.nanoTime(); + opencv_core.Mat input = inputSocket.getValue().get(); + if (input.empty() || input.isNull()) { throw new IllegalArgumentException("Input image must not be empty"); } - synchronized (imageLock) { - imencode(".jpeg", inputSocket.getValue().get(), imagePointer, - new IntPointer(CV_IMWRITE_JPEG_QUALITY, qualitySocket.getValue().get().intValue())); - imageLock.notifyAll(); + copyJavaCvToOpenCvMat(input, publishMat); + serverSource.putFrame(publishMat); + if (lastFrame != -1) { + long dt = now - lastFrame; + serverSource.setFPS((int) (1e9 / dt)); } + lastFrame = now; + server.setSource(serverSource); } @Override public synchronized void cleanUp() { // Stop the video server if there are no Publish Video steps left - serverThread.interrupt(); numSteps--; } + + private void copyJavaCvToOpenCvMat(opencv_core.Mat javaCvMat, Mat openCvMat) { + if (javaCvMat.depth() != CV_8U && javaCvMat.depth() != CV_8S) { + throw new IllegalArgumentException("Only 8-bit depth images are supported"); + } + + final opencv_core.Size size = javaCvMat.size(); + + // Make sure the output resolution is up to date + serverSource.setResolution(size.width(), size.height()); + + // Make the OpenCV Mat object point to the same block of memory as the JavaCV object. + // This requires no data transfers or copies and is O(1) instead of O(n) + if (javaCvMat.address() != openCvMat.nativeObj) { + try { + Field nativeObjField = Mat.class.getField("nativeObj"); + nativeObjField.setAccessible(true); + nativeObjField.setLong(openCvMat, javaCvMat.address()); + } catch (ReflectiveOperationException e) { + logger.log(Level.WARNING, "Could not set native object pointer", e); + } + } + } + } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/SaveImageOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/SaveImageOperation.java index f948bbec73..0c2d69f3ca 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/SaveImageOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/SaveImageOperation.java @@ -124,9 +124,9 @@ public void perform() { imencode("." + fileTypesSocket.getValue().get(), inputSocket.getValue().get(), imagePointer, new IntPointer(CV_IMWRITE_JPEG_QUALITY, qualitySocket.getValue().get().intValue())); byte[] buffer = new byte[128 * 1024]; - int bufferSize = imagePointer.limit(); + int bufferSize = (int) imagePointer.limit(); if (bufferSize > buffer.length) { - buffer = new byte[imagePointer.limit()]; + buffer = new byte[bufferSize]; } imagePointer.get(buffer, 0, bufferSize); diff --git a/core/src/main/java/edu/wpi/grip/core/operations/opencv/MinMaxLoc.java b/core/src/main/java/edu/wpi/grip/core/operations/opencv/MinMaxLoc.java index cc0771597c..63a85d95e9 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/opencv/MinMaxLoc.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/opencv/MinMaxLoc.java @@ -8,6 +8,7 @@ import com.google.common.collect.ImmutableList; +import org.bytedeco.javacpp.DoublePointer; import org.bytedeco.javacpp.opencv_core; import org.bytedeco.javacpp.opencv_core.Mat; import org.bytedeco.javacpp.opencv_core.Point; @@ -85,14 +86,14 @@ public void perform() { if (mask.empty()) { mask = null; } - final double[] minVal = new double[1]; - final double[] maxVal = new double[1]; + DoublePointer minVal = new DoublePointer(0.0); + DoublePointer maxVal = new DoublePointer(0.0); final Point minLoc = minLocSocket.getValue().get(); final Point maxLoc = maxLocSocket.getValue().get(); opencv_core.minMaxLoc(src, minVal, maxVal, minLoc, maxLoc, mask); - minValSocket.setValue(minVal[0]); - maxValSocket.setValue(maxVal[0]); + minValSocket.setValue(minVal.get(0)); + maxValSocket.setValue(maxVal.get(0)); minLocSocket.setValue(minLocSocket.getValue().get()); maxLocSocket.setValue(maxLocSocket.getValue().get()); } diff --git a/core/src/test/java/edu/wpi/grip/core/sources/CameraSourceTest.java b/core/src/test/java/edu/wpi/grip/core/sources/CameraSourceTest.java index 29447211f4..2eaac028cc 100644 --- a/core/src/test/java/edu/wpi/grip/core/sources/CameraSourceTest.java +++ b/core/src/test/java/edu/wpi/grip/core/sources/CameraSourceTest.java @@ -281,7 +281,7 @@ static class MockFrameGrabber extends FrameGrabber { for (int y = 0; y < frameIdx.rows(); y++) { for (int x = 0; x < frameIdx.cols(); x++) { for (int z = 0; z < frameIdx.channels(); z++) { - frameIdx.putDouble(new int[]{y, x, z}, y + x + z); + frameIdx.putDouble(new long[]{y, x, z}, y + x + z); } } } From af7ebeb5d6cc779339d7d0817680cd74a090dd14 Mon Sep 17 00:00:00 2001 From: Sam Carlberg Date: Wed, 8 Nov 2017 23:22:45 -0500 Subject: [PATCH 02/10] Smart port reuse --- .../composite/PublishVideoOperation.java | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java index bd9aa7d2be..c3c4c1b6a0 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java @@ -20,9 +20,13 @@ import org.opencv.core.Mat; import java.lang.reflect.Field; +import java.util.Deque; +import java.util.LinkedList; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.bytedeco.javacpp.opencv_core.CV_8S; import static org.bytedeco.javacpp.opencv_core.CV_8U; @@ -57,7 +61,11 @@ public class PublishVideoOperation implements Operation { @SuppressWarnings("PMD.AssignmentToNonFinalStatic") private static int numSteps; - private static final int MAX_STEP_COUNT = 10; + private static final int MAX_STEP_COUNT = 10; // limit ports to 1180-1189 + private static final Deque availablePorts = + Stream.iterate(PORT, i -> i + 1) + .limit(MAX_STEP_COUNT) + .collect(Collectors.toCollection(LinkedList::new)); private final InputSocket inputSocket; private final InputSocket qualitySocket; @@ -81,11 +89,13 @@ public PublishVideoOperation(InputSocket.Factory inputSocketFactory) { this.qualitySocket = inputSocketFactory.create(SocketHints.Inputs .createNumberSliderSocketHint("Quality", 80, 0, 100)); - server = new MjpegServer("GRIP server " + numSteps, PORT + numSteps); - serverSource = new CvSource("GRIP CvSource" + numSteps, VideoMode.PixelFormat.kMJPEG, 0, 0, 0); + int ourPort = availablePorts.removeFirst(); + + server = new MjpegServer("GRIP video publishing server " + ourPort, ourPort); + serverSource = new CvSource("GRIP CvSource:" + ourPort, VideoMode.PixelFormat.kMJPEG, 0, 0, 0); server.setSource(serverSource); cameraPublisherTable.putStringArray("streams", - new String[]{CameraServerJNI.getHostname() + ":" + server.getPort()}); + new String[]{CameraServerJNI.getHostname() + ":" + ourPort}); numSteps++; } @@ -125,6 +135,7 @@ public void perform() { public synchronized void cleanUp() { // Stop the video server if there are no Publish Video steps left numSteps--; + availablePorts.addFirst(server.getPort()); } private void copyJavaCvToOpenCvMat(opencv_core.Mat javaCvMat, Mat openCvMat) { From 1691326efa5c2d00c82a3db17782d6d75b1d119c Mon Sep 17 00:00:00 2001 From: Sam Carlberg Date: Wed, 8 Nov 2017 23:56:49 -0500 Subject: [PATCH 03/10] Improve NetworkTable code --- .../composite/PublishVideoOperation.java | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java index c3c4c1b6a0..7ea09c707a 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java @@ -15,6 +15,7 @@ import edu.wpi.cscore.MjpegServer; import edu.wpi.cscore.VideoMode; import edu.wpi.first.wpilibj.networktables.NetworkTable; +import edu.wpi.first.wpilibj.tables.ITable; import org.bytedeco.javacpp.opencv_core; import org.opencv.core.Mat; @@ -59,6 +60,8 @@ public class PublishVideoOperation implements Operation { .build(); private static final int PORT = 1180; + @SuppressWarnings("PMD.AssignmentToNonFinalStatic") + private static int totalStepCount; @SuppressWarnings("PMD.AssignmentToNonFinalStatic") private static int numSteps; private static final int MAX_STEP_COUNT = 10; // limit ports to 1180-1189 @@ -71,8 +74,11 @@ public class PublishVideoOperation implements Operation { private final InputSocket qualitySocket; private final MjpegServer server; private final CvSource serverSource; - private static final NetworkTable cameraPublisherTable = - NetworkTable.getTable("/CameraPublisher"); + + // Write to the /CameraPublisher table so the MJPEG streams are discoverable by other + // applications connected to the same NetworkTable server (eg Shuffleboard) + private static final ITable cameraPublisherTable = NetworkTable.getTable("/CameraPublisher"); + private final ITable ourTable; private final Mat publishMat = new Mat(); private long lastFrame = -1; @@ -91,13 +97,17 @@ public PublishVideoOperation(InputSocket.Factory inputSocketFactory) { int ourPort = availablePorts.removeFirst(); - server = new MjpegServer("GRIP video publishing server " + ourPort, ourPort); - serverSource = new CvSource("GRIP CvSource:" + ourPort, VideoMode.PixelFormat.kMJPEG, 0, 0, 0); + server = new MjpegServer("GRIP video publishing server " + totalStepCount, ourPort); + serverSource = new CvSource("GRIP CvSource " + totalStepCount, + VideoMode.PixelFormat.kMJPEG, 0, 0, 0); server.setSource(serverSource); - cameraPublisherTable.putStringArray("streams", - new String[]{CameraServerJNI.getHostname() + ":" + ourPort}); + + ourTable = cameraPublisherTable.getSubTable("GRIP-" + totalStepCount); + ourTable.putStringArray("streams", + new String[]{CameraServerJNI.getHostname() + ":" + ourPort + "/?action=stream"}); numSteps++; + totalStepCount++; } @Override @@ -115,7 +125,7 @@ public List getOutputSockets() { @Override public void perform() { - final long now = System.nanoTime(); + final long now = System.nanoTime(); // NOPMD opencv_core.Mat input = inputSocket.getValue().get(); if (input.empty() || input.isNull()) { throw new IllegalArgumentException("Input image must not be empty"); @@ -136,6 +146,7 @@ public synchronized void cleanUp() { // Stop the video server if there are no Publish Video steps left numSteps--; availablePorts.addFirst(server.getPort()); + ourTable.getKeys().forEach(ourTable::delete); } private void copyJavaCvToOpenCvMat(opencv_core.Mat javaCvMat, Mat openCvMat) { From 86829a734b2a81333dfd26b1b1781aecb459bc24 Mon Sep 17 00:00:00 2001 From: Sam Carlberg Date: Thu, 9 Nov 2017 12:47:48 -0500 Subject: [PATCH 04/10] Updates from review. Add lots of documentation, fix hostname on windows Set a flag when cscore can't be loaded to make the operation perform() fail, instead of crashing to desktop Still need to do mac hostname resolution --- .../composite/PublishVideoOperation.java | 141 ++++++++++++++---- 1 file changed, 108 insertions(+), 33 deletions(-) diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java index 7ea09c707a..bb1cdf43e9 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java @@ -17,10 +17,15 @@ import edu.wpi.first.wpilibj.networktables.NetworkTable; import edu.wpi.first.wpilibj.tables.ITable; +import org.apache.commons.lang.SystemUtils; import org.bytedeco.javacpp.opencv_core; import org.opencv.core.Mat; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; import java.lang.reflect.Field; +import java.util.Arrays; import java.util.Deque; import java.util.LinkedList; import java.util.List; @@ -29,9 +34,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.bytedeco.javacpp.opencv_core.CV_8S; -import static org.bytedeco.javacpp.opencv_core.CV_8U; - /** * Publish an M-JPEG stream with the protocol used by SmartDashboard and the FRC Dashboard. This * allows FRC teams to view video streams on their dashboard during competition even when GRIP has @@ -41,14 +43,23 @@ public class PublishVideoOperation implements Operation { private static final Logger logger = Logger.getLogger(PublishVideoOperation.class.getName()); + /** + * Flags whether or not cscore was loaded. If it could not be loaded, the MJPEG streaming server + * can't be started, preventing this operation from running. + */ + private static final boolean cscoreLoaded; + static { + boolean loaded; try { // Loading the CameraServerJNI class will load the appropriate platform-specific OpenCV JNI CameraServerJNI.getHostname(); + loaded = true; } catch (Throwable e) { - logger.log(Level.SEVERE, "CameraServerJNI load failed! Exiting", e); - System.exit(31); + logger.log(Level.SEVERE, "CameraServerJNI load failed!", e); + loaded = false; } + cscoreLoaded = loaded; } public static final OperationDescription DESCRIPTION = @@ -58,15 +69,15 @@ public class PublishVideoOperation implements Operation { .category(OperationDescription.Category.NETWORK) .icon(Icon.iconStream("publish-video")) .build(); - private static final int PORT = 1180; + private static final int INITIAL_PORT = 1180; + private static final int MAX_STEP_COUNT = 10; // limit ports to 1180-1189 @SuppressWarnings("PMD.AssignmentToNonFinalStatic") private static int totalStepCount; @SuppressWarnings("PMD.AssignmentToNonFinalStatic") private static int numSteps; - private static final int MAX_STEP_COUNT = 10; // limit ports to 1180-1189 private static final Deque availablePorts = - Stream.iterate(PORT, i -> i + 1) + Stream.iterate(INITIAL_PORT, i -> i + 1) .limit(MAX_STEP_COUNT) .collect(Collectors.toCollection(LinkedList::new)); @@ -77,7 +88,7 @@ public class PublishVideoOperation implements Operation { // Write to the /CameraPublisher table so the MJPEG streams are discoverable by other // applications connected to the same NetworkTable server (eg Shuffleboard) - private static final ITable cameraPublisherTable = NetworkTable.getTable("/CameraPublisher"); + private final ITable cameraPublisherTable = NetworkTable.getTable("/CameraPublisher"); // NOPMD private final ITable ourTable; private final Mat publishMat = new Mat(); private long lastFrame = -1; @@ -95,16 +106,22 @@ public PublishVideoOperation(InputSocket.Factory inputSocketFactory) { this.qualitySocket = inputSocketFactory.create(SocketHints.Inputs .createNumberSliderSocketHint("Quality", 80, 0, 100)); - int ourPort = availablePorts.removeFirst(); - - server = new MjpegServer("GRIP video publishing server " + totalStepCount, ourPort); - serverSource = new CvSource("GRIP CvSource " + totalStepCount, - VideoMode.PixelFormat.kMJPEG, 0, 0, 0); - server.setSource(serverSource); - - ourTable = cameraPublisherTable.getSubTable("GRIP-" + totalStepCount); - ourTable.putStringArray("streams", - new String[]{CameraServerJNI.getHostname() + ":" + ourPort + "/?action=stream"}); + if (cscoreLoaded) { + int ourPort = availablePorts.removeFirst(); + + server = new MjpegServer("GRIP video publishing server " + totalStepCount, ourPort); + serverSource = new CvSource("GRIP CvSource " + totalStepCount, + VideoMode.PixelFormat.kMJPEG, 0, 0, 0); + server.setSource(serverSource); + + ourTable = cameraPublisherTable.getSubTable("GRIP-" + totalStepCount); + ourTable.putStringArray("streams", + new String[]{"mjpeg:http://" + getHostName() + ":" + ourPort + "/?action=stream"}); + } else { + server = null; + serverSource = null; + ourTable = null; + } numSteps++; totalStepCount++; @@ -126,39 +143,60 @@ public List getOutputSockets() { @Override public void perform() { final long now = System.nanoTime(); // NOPMD + + if (!cscoreLoaded) { + throw new IllegalStateException( + "cscore could not be loaded. The image streaming server cannot be started."); + } + opencv_core.Mat input = inputSocket.getValue().get(); if (input.empty() || input.isNull()) { throw new IllegalArgumentException("Input image must not be empty"); } copyJavaCvToOpenCvMat(input, publishMat); + // Make sure the output resolution is up to date. Might not be needed, depends on cscore updates + serverSource.setResolution(input.size().width(), input.size().height()); serverSource.putFrame(publishMat); if (lastFrame != -1) { long dt = now - lastFrame; serverSource.setFPS((int) (1e9 / dt)); } lastFrame = now; - server.setSource(serverSource); } @Override public synchronized void cleanUp() { - // Stop the video server if there are no Publish Video steps left numSteps--; - availablePorts.addFirst(server.getPort()); - ourTable.getKeys().forEach(ourTable::delete); - } - - private void copyJavaCvToOpenCvMat(opencv_core.Mat javaCvMat, Mat openCvMat) { - if (javaCvMat.depth() != CV_8U && javaCvMat.depth() != CV_8S) { - throw new IllegalArgumentException("Only 8-bit depth images are supported"); + if (cscoreLoaded) { + availablePorts.addFirst(server.getPort()); + ourTable.getKeys().forEach(ourTable::delete); + serverSource.setConnected(false); + serverSource.free(); + server.free(); } + } - final opencv_core.Size size = javaCvMat.size(); - - // Make sure the output resolution is up to date - serverSource.setResolution(size.width(), size.height()); - + /** + * Copies the data from a JavaCV Mat wrapper object into an OpenCV Mat wrapper object so it's + * usable by the {@link CvSource} for this operation. + * + *

Since the JavaCV and OpenCV bindings both target the same native version of OpenCV, this is + * implemented by simply changing the OpenCV Mat's native pointer to be the same as the one for + * the JavaCV Mat. This prevents memory copies and resizing/reallocating memory for the OpenCV + * wrapper to fit the source image. Updating the pointer is a simple field write (albeit via + * reflection), which is much faster and easier than allocating and copying byte buffers.

+ * + *

A caveat to this approach is that the memory layout used by the OpenCV binaries bundled with + * both wrapper libraries must be identical. Using the same OpenCV version for both + * libraries should be enough.

+ * + * @param javaCvMat the JavaCV Mat wrapper object to copy from + * @param openCvMat the OpenCV Mat wrapper object to copy into + * @throws RuntimeException if the OpenCV native pointer could not be set + */ + private static void copyJavaCvToOpenCvMat(opencv_core.Mat javaCvMat, Mat openCvMat) + throws RuntimeException { // Make the OpenCV Mat object point to the same block of memory as the JavaCV object. // This requires no data transfers or copies and is O(1) instead of O(n) if (javaCvMat.address() != openCvMat.nativeObj) { @@ -168,7 +206,44 @@ private void copyJavaCvToOpenCvMat(opencv_core.Mat javaCvMat, Mat openCvMat) { nativeObjField.setLong(openCvMat, javaCvMat.address()); } catch (ReflectiveOperationException e) { logger.log(Level.WARNING, "Could not set native object pointer", e); + throw new RuntimeException("Could not copy the image", e); + } + } + } + + /** + * Multi platform method for getting the hostname of the local computer. cscore's + * {@link CameraServerJNI#getHostname() getHostName() function} only works on Linux, so we need to + * implement the method for Windows and Mac ourselves. + */ + private static String getHostName() { + if (SystemUtils.IS_OS_WINDOWS) { + // Use the Windows `hostname` command-line utility + // This will return a single line of text containing the hostname, no parsing required + ProcessBuilder builder = new ProcessBuilder("hostname"); + Process hostname; + try { + hostname = builder.start(); + } catch (IOException e) { + logger.log(Level.WARNING, "Could not start hostname process", e); + return ""; + } + try (BufferedReader in = + new BufferedReader(new InputStreamReader(hostname.getInputStream()))) { + return in.readLine() + ".local"; + } catch (IOException e) { + logger.log(Level.WARNING, "Could not read the hostname process output", e); + return ""; } + } else if (SystemUtils.IS_OS_LINUX) { + // cscore already defines it for linux + return CameraServerJNI.getHostname(); + } else if (SystemUtils.IS_OS_MAC) { + // todo + return "TODO-MAC"; + } else { + throw new UnsupportedOperationException( + "Unsupported operating system " + System.getProperty("os.name")); } } From a65ffd262c19065a2cc386364c1500d379fe5968 Mon Sep 17 00:00:00 2001 From: Sam Carlberg Date: Thu, 9 Nov 2017 15:28:50 -0500 Subject: [PATCH 05/10] Use InetAddress.localhost() for getting hostname, add test for mat copy --- .../composite/PublishVideoOperation.java | 63 +++++----------- .../composite/PublishVideoOperationTest.java | 74 +++++++++++++++++++ 2 files changed, 93 insertions(+), 44 deletions(-) create mode 100644 core/src/test/java/edu/wpi/grip/core/operations/composite/PublishVideoOperationTest.java diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java index bb1cdf43e9..fe1092761c 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java @@ -7,6 +7,7 @@ import edu.wpi.grip.core.sockets.SocketHints; import edu.wpi.grip.core.util.Icon; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; @@ -17,15 +18,12 @@ import edu.wpi.first.wpilibj.networktables.NetworkTable; import edu.wpi.first.wpilibj.tables.ITable; -import org.apache.commons.lang.SystemUtils; import org.bytedeco.javacpp.opencv_core; import org.opencv.core.Mat; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; import java.lang.reflect.Field; -import java.util.Arrays; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.Deque; import java.util.LinkedList; import java.util.List; @@ -115,8 +113,16 @@ public PublishVideoOperation(InputSocket.Factory inputSocketFactory) { server.setSource(serverSource); ourTable = cameraPublisherTable.getSubTable("GRIP-" + totalStepCount); - ourTable.putStringArray("streams", - new String[]{"mjpeg:http://" + getHostName() + ":" + ourPort + "/?action=stream"}); + try { + InetAddress localHost = InetAddress.getLocalHost(); + ourTable.putStringArray("streams", + new String[]{ + generateStreamUrl(localHost.getHostName(), ourPort), + generateStreamUrl(localHost.getHostAddress(), ourPort) + }); + } catch (UnknownHostException e) { + ourTable.putStringArray("streams", new String[0]); + } } else { server = null; serverSource = null; @@ -177,6 +183,10 @@ public synchronized void cleanUp() { } } + private static String generateStreamUrl(String host, int port) { + return String.format("mjpeg:http://%s:%d/?action=stream", host, port); + } + /** * Copies the data from a JavaCV Mat wrapper object into an OpenCV Mat wrapper object so it's * usable by the {@link CvSource} for this operation. @@ -195,7 +205,8 @@ public synchronized void cleanUp() { * @param openCvMat the OpenCV Mat wrapper object to copy into * @throws RuntimeException if the OpenCV native pointer could not be set */ - private static void copyJavaCvToOpenCvMat(opencv_core.Mat javaCvMat, Mat openCvMat) + @VisibleForTesting + static void copyJavaCvToOpenCvMat(opencv_core.Mat javaCvMat, Mat openCvMat) throws RuntimeException { // Make the OpenCV Mat object point to the same block of memory as the JavaCV object. // This requires no data transfers or copies and is O(1) instead of O(n) @@ -211,40 +222,4 @@ private static void copyJavaCvToOpenCvMat(opencv_core.Mat javaCvMat, Mat openCvM } } - /** - * Multi platform method for getting the hostname of the local computer. cscore's - * {@link CameraServerJNI#getHostname() getHostName() function} only works on Linux, so we need to - * implement the method for Windows and Mac ourselves. - */ - private static String getHostName() { - if (SystemUtils.IS_OS_WINDOWS) { - // Use the Windows `hostname` command-line utility - // This will return a single line of text containing the hostname, no parsing required - ProcessBuilder builder = new ProcessBuilder("hostname"); - Process hostname; - try { - hostname = builder.start(); - } catch (IOException e) { - logger.log(Level.WARNING, "Could not start hostname process", e); - return ""; - } - try (BufferedReader in = - new BufferedReader(new InputStreamReader(hostname.getInputStream()))) { - return in.readLine() + ".local"; - } catch (IOException e) { - logger.log(Level.WARNING, "Could not read the hostname process output", e); - return ""; - } - } else if (SystemUtils.IS_OS_LINUX) { - // cscore already defines it for linux - return CameraServerJNI.getHostname(); - } else if (SystemUtils.IS_OS_MAC) { - // todo - return "TODO-MAC"; - } else { - throw new UnsupportedOperationException( - "Unsupported operating system " + System.getProperty("os.name")); - } - } - } diff --git a/core/src/test/java/edu/wpi/grip/core/operations/composite/PublishVideoOperationTest.java b/core/src/test/java/edu/wpi/grip/core/operations/composite/PublishVideoOperationTest.java new file mode 100644 index 0000000000..42a2d0b077 --- /dev/null +++ b/core/src/test/java/edu/wpi/grip/core/operations/composite/PublishVideoOperationTest.java @@ -0,0 +1,74 @@ +package edu.wpi.grip.core.operations.composite; + +import edu.wpi.grip.util.Files; + +import org.bytedeco.javacpp.opencv_core; +import org.junit.BeforeClass; +import org.junit.Test; +import org.opencv.core.Mat; + +import java.nio.ByteBuffer; + +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; + +public class PublishVideoOperationTest { + + @BeforeClass + public static void initialize() { + // Make sure the OpenCV JNI is loaded + blackHole(PublishVideoOperation.DESCRIPTION); + } + + @Test + public void testCopyJavaCvToOpenCvMat() { + // given + final Mat openCvMat = new Mat(); + + // then (with the GRIP logo) + test(Files.imageFile.createMat(), openCvMat); + + // and again (with gompei) to confirm that changing the input will be cleanly copied to the + // output image and cleanly overwrite any existing data + test(Files.gompeiJpegFile.createMat(), openCvMat); + } + + private static void test(opencv_core.Mat javaCvMat, Mat openCvMat) { + // when + PublishVideoOperation.copyJavaCvToOpenCvMat(javaCvMat, openCvMat); + + // then + + // test the basic properties (same size, type, etc.) + assertEquals("Wrong width", javaCvMat.cols(), openCvMat.cols()); + assertEquals("Wrong height", javaCvMat.rows(), openCvMat.rows()); + assertEquals("Wrong type", javaCvMat.type(), openCvMat.type()); + assertEquals("Wrong channel amount", javaCvMat.channels(), openCvMat.channels()); + assertEquals("Wrong bit depth", javaCvMat.depth(), openCvMat.depth()); + + // test the raw data bytes - they should be identical + final int width = javaCvMat.cols(); + final int height = javaCvMat.rows(); + final int channels = javaCvMat.channels(); + + final ByteBuffer buffer = javaCvMat.createBuffer(); + assertThat("JavaCV byte buffer is smaller than expected!", + buffer.capacity(), greaterThanOrEqualTo(width * height * channels)); + + final byte[] javaCvData = new byte[width * height * channels]; + buffer.get(javaCvData); + + final byte[] openCvData = new byte[width * height * channels]; + openCvMat.get(0, 0, openCvData); + + assertArrayEquals("Wrong data bytes", javaCvData, openCvData); + } + + // workaround for FindBugs reporting unused variables + private static void blackHole(Object ignore) { + // nop + } + +} From 40343c8e9ab2889178e7fd25adc788db684f8ddf Mon Sep 17 00:00:00 2001 From: Sam Carlberg Date: Fri, 10 Nov 2017 12:51:23 -0500 Subject: [PATCH 06/10] Use Mat ctor for copying pointer, use NetworkInterface to get host IPs --- .../composite/PublishVideoOperation.java | 106 +++++++++------- .../composite/PublishVideoOperationTest.java | 118 ++++++++++-------- 2 files changed, 131 insertions(+), 93 deletions(-) diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java index fe1092761c..7e5653c2f1 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java @@ -21,9 +21,11 @@ import org.bytedeco.javacpp.opencv_core; import org.opencv.core.Mat; -import java.lang.reflect.Field; -import java.net.InetAddress; -import java.net.UnknownHostException; +import java.net.Inet4Address; +import java.net.NetworkInterface; +import java.net.SocketException; +import java.util.Collection; +import java.util.Collections; import java.util.Deque; import java.util.LinkedList; import java.util.List; @@ -47,6 +49,8 @@ public class PublishVideoOperation implements Operation { */ private static final boolean cscoreLoaded; + private static final List networkInterfaces; + static { boolean loaded; try { @@ -58,6 +62,15 @@ public class PublishVideoOperation implements Operation { loaded = false; } cscoreLoaded = loaded; + + List interfaces; + try { + interfaces = Collections.list(NetworkInterface.getNetworkInterfaces()); + } catch (SocketException e) { + logger.log(Level.SEVERE, "Could not get the local network interfaces", e); + interfaces = Collections.emptyList(); + } + networkInterfaces = interfaces; } public static final OperationDescription DESCRIPTION = @@ -88,7 +101,7 @@ public class PublishVideoOperation implements Operation { // applications connected to the same NetworkTable server (eg Shuffleboard) private final ITable cameraPublisherTable = NetworkTable.getTable("/CameraPublisher"); // NOPMD private final ITable ourTable; - private final Mat publishMat = new Mat(); + private Mat publishMat = null; private long lastFrame = -1; @SuppressWarnings("JavadocMethod") @@ -114,13 +127,11 @@ public PublishVideoOperation(InputSocket.Factory inputSocketFactory) { ourTable = cameraPublisherTable.getSubTable("GRIP-" + totalStepCount); try { - InetAddress localHost = InetAddress.getLocalHost(); - ourTable.putStringArray("streams", - new String[]{ - generateStreamUrl(localHost.getHostName(), ourPort), - generateStreamUrl(localHost.getHostAddress(), ourPort) - }); - } catch (UnknownHostException e) { + List networkInterfaces = + Collections.list(NetworkInterface.getNetworkInterfaces()); + ourTable.putStringArray("streams", generateStreams(networkInterfaces, ourPort)); + } catch (SocketException e) { + logger.log(Level.WARNING, "Could not enumerate the local network interfaces", e); ourTable.putStringArray("streams", new String[0]); } } else { @@ -160,7 +171,16 @@ public void perform() { throw new IllegalArgumentException("Input image must not be empty"); } - copyJavaCvToOpenCvMat(input, publishMat); + // "copy" the input data to an OpenCV mat for cscore to use + // This basically just wraps the mat pointer in a different wrapper object + // No copies are performed, but it means we have to be careful about making sure we use the + // same version of JavaCV and OpenCV to minimize the risk of binary incompatibility. + // This copy only needs to happen once, since the operation input image is always the same + // object that gets copied into. The data address will change, however, if the image is resized + // or changes type. + if (publishMat == null || publishMat.nativeObj != input.address()) { + publishMat = new Mat(input.address()); + } // Make sure the output resolution is up to date. Might not be needed, depends on cscore updates serverSource.setResolution(input.size().width(), input.size().height()); serverSource.putFrame(publishMat); @@ -180,46 +200,46 @@ public synchronized void cleanUp() { serverSource.setConnected(false); serverSource.free(); server.free(); + if (publishMat != null) { + publishMat.release(); + } } } - private static String generateStreamUrl(String host, int port) { - return String.format("mjpeg:http://%s:%d/?action=stream", host, port); + /** + * Generates an array of stream URLs that allow third-party applications to discover the + * appropriate URLs that can stream MJPEG. The URLs will all point to the same physical machine, + * but may use different network interfaces (eg WiFi and ethernet). + * + * @param networkInterfaces the local network interfaces + * @param serverPort the port the mjpeg streaming server is running on + * @return an array of URLs that can be used to connect to the MJPEG streaming server + */ + @VisibleForTesting + static String[] generateStreams(Collection networkInterfaces, int serverPort) { + return networkInterfaces.stream() + .flatMap(i -> Collections.list(i.getInetAddresses()).stream()) + .filter(a -> a instanceof Inet4Address) // IPv6 isn't well supported, stick to IPv4 + .filter(a -> !a.isLoopbackAddress()) // loopback addresses only work for local processes + .distinct() + .flatMap(a -> Stream.of( + generateStreamUrl(a.getHostName(), serverPort), + generateStreamUrl(a.getHostAddress(), serverPort))) + .distinct() + .toArray(String[]::new); } /** - * Copies the data from a JavaCV Mat wrapper object into an OpenCV Mat wrapper object so it's - * usable by the {@link CvSource} for this operation. - * - *

Since the JavaCV and OpenCV bindings both target the same native version of OpenCV, this is - * implemented by simply changing the OpenCV Mat's native pointer to be the same as the one for - * the JavaCV Mat. This prevents memory copies and resizing/reallocating memory for the OpenCV - * wrapper to fit the source image. Updating the pointer is a simple field write (albeit via - * reflection), which is much faster and easier than allocating and copying byte buffers.

- * - *

A caveat to this approach is that the memory layout used by the OpenCV binaries bundled with - * both wrapper libraries must be identical. Using the same OpenCV version for both - * libraries should be enough.

+ * Generates a URL that can be used to connect to an MJPEG stream provided by cscore. The host + * should be a non-loopback IPv4 address that is resolvable by applications running on non-local + * machines. * - * @param javaCvMat the JavaCV Mat wrapper object to copy from - * @param openCvMat the OpenCV Mat wrapper object to copy into - * @throws RuntimeException if the OpenCV native pointer could not be set + * @param host the server host + * @param port the port the server is running on */ @VisibleForTesting - static void copyJavaCvToOpenCvMat(opencv_core.Mat javaCvMat, Mat openCvMat) - throws RuntimeException { - // Make the OpenCV Mat object point to the same block of memory as the JavaCV object. - // This requires no data transfers or copies and is O(1) instead of O(n) - if (javaCvMat.address() != openCvMat.nativeObj) { - try { - Field nativeObjField = Mat.class.getField("nativeObj"); - nativeObjField.setAccessible(true); - nativeObjField.setLong(openCvMat, javaCvMat.address()); - } catch (ReflectiveOperationException e) { - logger.log(Level.WARNING, "Could not set native object pointer", e); - throw new RuntimeException("Could not copy the image", e); - } - } + static String generateStreamUrl(String host, int port) { + return String.format("mjpeg:http://%s:%d/?action=stream", host, port); } } diff --git a/core/src/test/java/edu/wpi/grip/core/operations/composite/PublishVideoOperationTest.java b/core/src/test/java/edu/wpi/grip/core/operations/composite/PublishVideoOperationTest.java index 42a2d0b077..283f5a749c 100644 --- a/core/src/test/java/edu/wpi/grip/core/operations/composite/PublishVideoOperationTest.java +++ b/core/src/test/java/edu/wpi/grip/core/operations/composite/PublishVideoOperationTest.java @@ -1,74 +1,92 @@ package edu.wpi.grip.core.operations.composite; -import edu.wpi.grip.util.Files; - -import org.bytedeco.javacpp.opencv_core; -import org.junit.BeforeClass; import org.junit.Test; -import org.opencv.core.Mat; -import java.nio.ByteBuffer; +import java.lang.reflect.Constructor; +import java.net.Inet4Address; +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.util.Arrays; +import java.util.List; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.junit.Assert.assertArrayEquals; +import static edu.wpi.grip.core.operations.composite.PublishVideoOperation.generateStreamUrl; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; public class PublishVideoOperationTest { - @BeforeClass - public static void initialize() { - // Make sure the OpenCV JNI is loaded - blackHole(PublishVideoOperation.DESCRIPTION); - } - @Test - public void testCopyJavaCvToOpenCvMat() { + public void testGenerateStreams() { // given - final Mat openCvMat = new Mat(); - - // then (with the GRIP logo) - test(Files.imageFile.createMat(), openCvMat); - - // and again (with gompei) to confirm that changing the input will be cleanly copied to the - // output image and cleanly overwrite any existing data - test(Files.gompeiJpegFile.createMat(), openCvMat); - } + final String firstHost = "localhost"; + final int firstAddress = 0x7F_00_00_01; // loopback 127.0.0.1 + final String secondHost = "driver-station"; + final int secondAddress = 0x0A_01_5A_05; // 10.1.90.5, FRC driver station IP + final String thirdHost = "network-mask"; + final int thirdAddress = 0xFF_FF_FF_FF; // 255.255.255.255, not loopback + final List networkInterfaces = + Arrays.asList( + newNetworkInterface( + "MockNetworkInterface0", 0, + new InetAddress[]{ + newInet4Address(firstHost, firstAddress) + }), + newNetworkInterface("MockNetworkInterface1", 1, + new InetAddress[]{ + newInet4Address(secondHost, secondAddress), + newInet4Address(thirdHost, thirdAddress) + }) + ); + final int port = 54321; - private static void test(opencv_core.Mat javaCvMat, Mat openCvMat) { // when - PublishVideoOperation.copyJavaCvToOpenCvMat(javaCvMat, openCvMat); + final String[] streams = PublishVideoOperation.generateStreams(networkInterfaces, port); // then + assertEquals("Four URLs should have been generated", 4, streams.length); - // test the basic properties (same size, type, etc.) - assertEquals("Wrong width", javaCvMat.cols(), openCvMat.cols()); - assertEquals("Wrong height", javaCvMat.rows(), openCvMat.rows()); - assertEquals("Wrong type", javaCvMat.type(), openCvMat.type()); - assertEquals("Wrong channel amount", javaCvMat.channels(), openCvMat.channels()); - assertEquals("Wrong bit depth", javaCvMat.depth(), openCvMat.depth()); - - // test the raw data bytes - they should be identical - final int width = javaCvMat.cols(); - final int height = javaCvMat.rows(); - final int channels = javaCvMat.channels(); - - final ByteBuffer buffer = javaCvMat.createBuffer(); - assertThat("JavaCV byte buffer is smaller than expected!", - buffer.capacity(), greaterThanOrEqualTo(width * height * channels)); + // stream URLs should be generated only for non-loopback IPv4 addresses + assertEquals(generateStreamUrl(secondHost, port), streams[0]); + assertEquals(generateStreamUrl(formatIpv4Address(secondAddress), port), streams[1]); + assertEquals(generateStreamUrl(thirdHost, port), streams[2]); + assertEquals(generateStreamUrl(formatIpv4Address(thirdAddress), port), streams[3]); - final byte[] javaCvData = new byte[width * height * channels]; - buffer.get(javaCvData); + } - final byte[] openCvData = new byte[width * height * channels]; - openCvMat.get(0, 0, openCvData); + private static String formatIpv4Address(int address) { + return String.format( + "%d.%d.%d.%d", + address >> 24 & 0xFF, + address >> 16 & 0xFF, + address >> 8 & 0xFF, + address & 0xFF + ); + } - assertArrayEquals("Wrong data bytes", javaCvData, openCvData); + private static NetworkInterface newNetworkInterface(String name, + int index, + InetAddress[] addresses) { + try { + Constructor constructor = + NetworkInterface.class.getDeclaredConstructor( + String.class, + int.class, + InetAddress[].class); + constructor.setAccessible(true); + return constructor.newInstance(name, index, addresses); + } catch (ReflectiveOperationException e) { + throw new AssertionError(e); + } } - // workaround for FindBugs reporting unused variables - private static void blackHole(Object ignore) { - // nop + private static Inet4Address newInet4Address(String hostname, int address) { + try { + Constructor constructor = + Inet4Address.class.getDeclaredConstructor(String.class, int.class); + constructor.setAccessible(true); + return constructor.newInstance(hostname, address); + } catch (ReflectiveOperationException e) { + throw new AssertionError(e); + } } } From 93022f70252b58808b37cf380b1b82f56e8c891b Mon Sep 17 00:00:00 2001 From: Sam Carlberg Date: Fri, 10 Nov 2017 14:43:55 -0500 Subject: [PATCH 07/10] Re-add test for compatiblity between JavaCV and OpenCV Fix an issue with codegen tests from the JavaCV update --- .../composite/PublishVideoOperationTest.java | 60 +++++++++++++++++++ .../ui/codegeneration/tools/HelperTools.java | 3 +- 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/core/src/test/java/edu/wpi/grip/core/operations/composite/PublishVideoOperationTest.java b/core/src/test/java/edu/wpi/grip/core/operations/composite/PublishVideoOperationTest.java index 283f5a749c..76d2db509b 100644 --- a/core/src/test/java/edu/wpi/grip/core/operations/composite/PublishVideoOperationTest.java +++ b/core/src/test/java/edu/wpi/grip/core/operations/composite/PublishVideoOperationTest.java @@ -1,19 +1,34 @@ package edu.wpi.grip.core.operations.composite; +import edu.wpi.grip.util.Files; + +import org.bytedeco.javacpp.opencv_core; +import org.junit.BeforeClass; import org.junit.Test; +import org.opencv.core.Mat; import java.lang.reflect.Constructor; import java.net.Inet4Address; import java.net.InetAddress; import java.net.NetworkInterface; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; import static edu.wpi.grip.core.operations.composite.PublishVideoOperation.generateStreamUrl; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; public class PublishVideoOperationTest { + @BeforeClass + public static void loadOpenCvJni() { + // Make sure the OpenCV JNI is loaded + blackHole(PublishVideoOperation.DESCRIPTION); + } + @Test public void testGenerateStreams() { // given @@ -89,4 +104,49 @@ private static Inet4Address newInet4Address(String hostname, int address) { } } + /** + * Make sure that the JavaCV Mat is compatible with the OpenCV Mat. This should check regressions + * from having different or otherwise incompatible versions of the OpenCV binaries bundled with + * JavaCV and the OpenCV library built by WPILib. + */ + @Test + public void testCopyJavaCvToOpenCvMatByRawPointer() { + // Test the GRIP logo + opencv_core.Mat javaCvMat = Files.imageFile.createMat(); + + // when + Mat openCvMat = new Mat(javaCvMat.address()); + + // then + + // test the basic properties (same size, type, etc.) + assertEquals("Wrong width", javaCvMat.cols(), openCvMat.cols()); + assertEquals("Wrong height", javaCvMat.rows(), openCvMat.rows()); + assertEquals("Wrong type", javaCvMat.type(), openCvMat.type()); + assertEquals("Wrong channel amount", javaCvMat.channels(), openCvMat.channels()); + assertEquals("Wrong bit depth", javaCvMat.depth(), openCvMat.depth()); + + // test the raw data bytes - they should be identical + final int width = javaCvMat.cols(); + final int height = javaCvMat.rows(); + final int channels = javaCvMat.channels(); + + final ByteBuffer buffer = javaCvMat.createBuffer(); + assertThat("JavaCV byte buffer is smaller than expected!", + buffer.capacity(), greaterThanOrEqualTo(width * height * channels)); + + final byte[] javaCvData = new byte[width * height * channels]; + buffer.get(javaCvData); + + final byte[] openCvData = new byte[width * height * channels]; + openCvMat.get(0, 0, openCvData); + + assertArrayEquals("Wrong data bytes", javaCvData, openCvData); + } + + // workaround for FindBugs reporting unused variables + private static void blackHole(Object ignore) { + // nop + } + } diff --git a/ui/src/test/java/edu/wpi/grip/ui/codegeneration/tools/HelperTools.java b/ui/src/test/java/edu/wpi/grip/ui/codegeneration/tools/HelperTools.java index d28209e13c..8568fefc15 100644 --- a/ui/src/test/java/edu/wpi/grip/ui/codegeneration/tools/HelperTools.java +++ b/ui/src/test/java/edu/wpi/grip/ui/codegeneration/tools/HelperTools.java @@ -68,7 +68,8 @@ public static double matAvgDiff(Mat mat1, Mat mat2) { */ public static Mat bytedecoMatToCVMat(org.bytedeco.javacpp.opencv_core.Mat input) { UByteIndexer idxer = input.createIndexer(); - Mat out = new Mat(idxer.rows(), idxer.cols(), CvType.CV_8UC(idxer.channels())); + Mat out = new Mat( + (int) idxer.rows(), (int) idxer.cols(), CvType.CV_8UC((int) idxer.channels())); //Mat out = new Mat(idxer.rows(),idxer.cols(),input.type()); for (int row = 0; row < idxer.rows(); row++) { for (int col = 0; col < idxer.cols(); col++) { From b4bff3b6e96599f77ba710addbd32bc7826d91d2 Mon Sep 17 00:00:00 2001 From: Sam Carlberg Date: Sun, 12 Nov 2017 12:41:07 -0500 Subject: [PATCH 08/10] Remove unused static list of network interfaces, use assume() in test --- .../operations/composite/PublishVideoOperation.java | 11 ----------- .../composite/PublishVideoOperationTest.java | 4 ++-- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java index 7e5653c2f1..aa3153ad33 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java @@ -49,8 +49,6 @@ public class PublishVideoOperation implements Operation { */ private static final boolean cscoreLoaded; - private static final List networkInterfaces; - static { boolean loaded; try { @@ -62,15 +60,6 @@ public class PublishVideoOperation implements Operation { loaded = false; } cscoreLoaded = loaded; - - List interfaces; - try { - interfaces = Collections.list(NetworkInterface.getNetworkInterfaces()); - } catch (SocketException e) { - logger.log(Level.SEVERE, "Could not get the local network interfaces", e); - interfaces = Collections.emptyList(); - } - networkInterfaces = interfaces; } public static final OperationDescription DESCRIPTION = diff --git a/core/src/test/java/edu/wpi/grip/core/operations/composite/PublishVideoOperationTest.java b/core/src/test/java/edu/wpi/grip/core/operations/composite/PublishVideoOperationTest.java index 76d2db509b..7faddd3624 100644 --- a/core/src/test/java/edu/wpi/grip/core/operations/composite/PublishVideoOperationTest.java +++ b/core/src/test/java/edu/wpi/grip/core/operations/composite/PublishVideoOperationTest.java @@ -19,7 +19,7 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; +import static org.junit.Assume.assumeThat; public class PublishVideoOperationTest { @@ -132,7 +132,7 @@ public void testCopyJavaCvToOpenCvMatByRawPointer() { final int channels = javaCvMat.channels(); final ByteBuffer buffer = javaCvMat.createBuffer(); - assertThat("JavaCV byte buffer is smaller than expected!", + assumeThat("JavaCV byte buffer is smaller than expected!", buffer.capacity(), greaterThanOrEqualTo(width * height * channels)); final byte[] javaCvData = new byte[width * height * channels]; From c1869f84a7c02bce84fa37d7186236ecf7faa9bd Mon Sep 17 00:00:00 2001 From: Sam Carlberg Date: Wed, 20 Dec 2017 15:15:48 -0500 Subject: [PATCH 09/10] Copy bytes instead of native pointers Should fix problems resulting from incompatible binaries --- .../composite/PublishVideoOperation.java | 14 +---- .../edu/wpi/grip/core/util/OpenCvShims.java | 37 +++++++++++++ .../composite/PublishVideoOperationTest.java | 48 ---------------- .../wpi/grip/core/util/OpenCvShimsTest.java | 55 +++++++++++++++++++ 4 files changed, 95 insertions(+), 59 deletions(-) create mode 100644 core/src/main/java/edu/wpi/grip/core/util/OpenCvShims.java create mode 100644 core/src/test/java/edu/wpi/grip/core/util/OpenCvShimsTest.java diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java index aa3153ad33..e946a7c43a 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java @@ -6,6 +6,7 @@ import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.core.sockets.SocketHints; import edu.wpi.grip.core.util.Icon; +import edu.wpi.grip.core.util.OpenCvShims; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; @@ -90,7 +91,7 @@ public class PublishVideoOperation implements Operation { // applications connected to the same NetworkTable server (eg Shuffleboard) private final ITable cameraPublisherTable = NetworkTable.getTable("/CameraPublisher"); // NOPMD private final ITable ourTable; - private Mat publishMat = null; + private final Mat publishMat = new Mat(); private long lastFrame = -1; @SuppressWarnings("JavadocMethod") @@ -160,16 +161,7 @@ public void perform() { throw new IllegalArgumentException("Input image must not be empty"); } - // "copy" the input data to an OpenCV mat for cscore to use - // This basically just wraps the mat pointer in a different wrapper object - // No copies are performed, but it means we have to be careful about making sure we use the - // same version of JavaCV and OpenCV to minimize the risk of binary incompatibility. - // This copy only needs to happen once, since the operation input image is always the same - // object that gets copied into. The data address will change, however, if the image is resized - // or changes type. - if (publishMat == null || publishMat.nativeObj != input.address()) { - publishMat = new Mat(input.address()); - } + OpenCvShims.copyJavaCvMatToOpenCvMat(input, publishMat); // Make sure the output resolution is up to date. Might not be needed, depends on cscore updates serverSource.setResolution(input.size().width(), input.size().height()); serverSource.putFrame(publishMat); diff --git a/core/src/main/java/edu/wpi/grip/core/util/OpenCvShims.java b/core/src/main/java/edu/wpi/grip/core/util/OpenCvShims.java new file mode 100644 index 0000000000..95b8195e2b --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/util/OpenCvShims.java @@ -0,0 +1,37 @@ +package edu.wpi.grip.core.util; + +import org.bytedeco.javacpp.opencv_core; +import org.opencv.core.Mat; + +import java.nio.ByteBuffer; + +/** + * Shims for working with OpenCV and JavaCV wrappers. + */ +public final class OpenCvShims { + + private OpenCvShims() { + throw new UnsupportedOperationException("This is a utility class!"); + } + + /** + * Copies the data of a JavaCV Mat to an OpenCV mat. + */ + public static void copyJavaCvMatToOpenCvMat(opencv_core.Mat src, Mat dst) { + final int width = src.cols(); + final int height = src.rows(); + final int channels = src.channels(); + + final ByteBuffer buffer = src.createBuffer(); + byte[] bytes = new byte[width * height * channels]; + buffer.get(bytes); + + // Store the data in a temporary mat to get the type, size, etc. correct, since Mat doesn't + // expose methods for directly changing its size or type + final Mat tmp = new Mat(src.rows(), src.cols(), src.type()); + tmp.put(0, 0, bytes); + tmp.copyTo(dst); + tmp.release(); + } + +} diff --git a/core/src/test/java/edu/wpi/grip/core/operations/composite/PublishVideoOperationTest.java b/core/src/test/java/edu/wpi/grip/core/operations/composite/PublishVideoOperationTest.java index 7faddd3624..78810639dc 100644 --- a/core/src/test/java/edu/wpi/grip/core/operations/composite/PublishVideoOperationTest.java +++ b/core/src/test/java/edu/wpi/grip/core/operations/composite/PublishVideoOperationTest.java @@ -1,25 +1,17 @@ package edu.wpi.grip.core.operations.composite; -import edu.wpi.grip.util.Files; - -import org.bytedeco.javacpp.opencv_core; import org.junit.BeforeClass; import org.junit.Test; -import org.opencv.core.Mat; import java.lang.reflect.Constructor; import java.net.Inet4Address; import java.net.InetAddress; import java.net.NetworkInterface; -import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; import static edu.wpi.grip.core.operations.composite.PublishVideoOperation.generateStreamUrl; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import static org.junit.Assume.assumeThat; public class PublishVideoOperationTest { @@ -104,46 +96,6 @@ private static Inet4Address newInet4Address(String hostname, int address) { } } - /** - * Make sure that the JavaCV Mat is compatible with the OpenCV Mat. This should check regressions - * from having different or otherwise incompatible versions of the OpenCV binaries bundled with - * JavaCV and the OpenCV library built by WPILib. - */ - @Test - public void testCopyJavaCvToOpenCvMatByRawPointer() { - // Test the GRIP logo - opencv_core.Mat javaCvMat = Files.imageFile.createMat(); - - // when - Mat openCvMat = new Mat(javaCvMat.address()); - - // then - - // test the basic properties (same size, type, etc.) - assertEquals("Wrong width", javaCvMat.cols(), openCvMat.cols()); - assertEquals("Wrong height", javaCvMat.rows(), openCvMat.rows()); - assertEquals("Wrong type", javaCvMat.type(), openCvMat.type()); - assertEquals("Wrong channel amount", javaCvMat.channels(), openCvMat.channels()); - assertEquals("Wrong bit depth", javaCvMat.depth(), openCvMat.depth()); - - // test the raw data bytes - they should be identical - final int width = javaCvMat.cols(); - final int height = javaCvMat.rows(); - final int channels = javaCvMat.channels(); - - final ByteBuffer buffer = javaCvMat.createBuffer(); - assumeThat("JavaCV byte buffer is smaller than expected!", - buffer.capacity(), greaterThanOrEqualTo(width * height * channels)); - - final byte[] javaCvData = new byte[width * height * channels]; - buffer.get(javaCvData); - - final byte[] openCvData = new byte[width * height * channels]; - openCvMat.get(0, 0, openCvData); - - assertArrayEquals("Wrong data bytes", javaCvData, openCvData); - } - // workaround for FindBugs reporting unused variables private static void blackHole(Object ignore) { // nop diff --git a/core/src/test/java/edu/wpi/grip/core/util/OpenCvShimsTest.java b/core/src/test/java/edu/wpi/grip/core/util/OpenCvShimsTest.java new file mode 100644 index 0000000000..5f09d58c40 --- /dev/null +++ b/core/src/test/java/edu/wpi/grip/core/util/OpenCvShimsTest.java @@ -0,0 +1,55 @@ +package edu.wpi.grip.core.util; + +import edu.wpi.grip.util.Files; + +import org.bytedeco.javacpp.opencv_core; +import org.junit.Test; +import org.opencv.core.CvType; +import org.opencv.core.Mat; + +import java.nio.ByteBuffer; + +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assume.assumeThat; + +public class OpenCvShimsTest { + + @Test + public void testCopyJavaCvToOpenCv() { + // given + final opencv_core.Mat javaCvMat = Files.imageFile.createMat(); + final Mat openCvMat = new Mat(1, 1, CvType.CV_8SC1); + + // when + OpenCvShims.copyJavaCvMatToOpenCvMat(javaCvMat, openCvMat); + + // then + + // test the basic properties (same size, type, etc.) + assertEquals("Wrong width", javaCvMat.cols(), openCvMat.cols()); + assertEquals("Wrong height", javaCvMat.rows(), openCvMat.rows()); + assertEquals("Wrong type", javaCvMat.type(), openCvMat.type()); + assertEquals("Wrong channel amount", javaCvMat.channels(), openCvMat.channels()); + assertEquals("Wrong bit depth", javaCvMat.depth(), openCvMat.depth()); + + // test the raw data bytes - they should be identical + final int width = javaCvMat.cols(); + final int height = javaCvMat.rows(); + final int channels = javaCvMat.channels(); + + final ByteBuffer buffer = javaCvMat.createBuffer(); + assumeThat("JavaCV byte buffer is smaller than expected!", + buffer.capacity(), greaterThanOrEqualTo(width * height * channels)); + + final byte[] javaCvData = new byte[width * height * channels]; + buffer.get(javaCvData); + + final byte[] openCvData = new byte[width * height * channels]; + openCvMat.get(0, 0, openCvData); + + assertArrayEquals("Wrong data bytes", javaCvData, openCvData); + } + +} From 644cab85f1cdf9a0ff82c43a7b63c5a5abc6df82 Mon Sep 17 00:00:00 2001 From: Sam Carlberg Date: Wed, 20 Dec 2017 15:23:05 -0500 Subject: [PATCH 10/10] Remove unneeded preloading --- .../composite/PublishVideoOperationTest.java | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/core/src/test/java/edu/wpi/grip/core/operations/composite/PublishVideoOperationTest.java b/core/src/test/java/edu/wpi/grip/core/operations/composite/PublishVideoOperationTest.java index 78810639dc..283f5a749c 100644 --- a/core/src/test/java/edu/wpi/grip/core/operations/composite/PublishVideoOperationTest.java +++ b/core/src/test/java/edu/wpi/grip/core/operations/composite/PublishVideoOperationTest.java @@ -1,6 +1,5 @@ package edu.wpi.grip.core.operations.composite; -import org.junit.BeforeClass; import org.junit.Test; import java.lang.reflect.Constructor; @@ -15,12 +14,6 @@ public class PublishVideoOperationTest { - @BeforeClass - public static void loadOpenCvJni() { - // Make sure the OpenCV JNI is loaded - blackHole(PublishVideoOperation.DESCRIPTION); - } - @Test public void testGenerateStreams() { // given @@ -96,9 +89,4 @@ private static Inet4Address newInet4Address(String hostname, int address) { } } - // workaround for FindBugs reporting unused variables - private static void blackHole(Object ignore) { - // nop - } - }