diff --git a/core/core.gradle.kts b/core/core.gradle.kts index 3092b71326..a1e1ce870d 100644 --- a/core/core.gradle.kts +++ b/core/core.gradle.kts @@ -1,4 +1,6 @@ import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar +import com.google.common.io.Files +import java.nio.charset.StandardCharsets plugins { id("java") @@ -15,15 +17,21 @@ application { val os = osdetector.classifier.replace("osx", "macosx").replace("x86_32", "x86") val arch = osdetector.arch.replace("x86_64", "x64") +val withCuda = project.hasProperty("cuda") || project.hasProperty("WITH_CUDA") + +if (withCuda) { + version = "$version-cuda" +} + dependencies { api(project(":annotation")) annotationProcessor(project(":annotation")) api(group = "com.google.code.findbugs", name = "jsr305", version = "3.0.1") - api(group = "org.bytedeco", name = "javacv", version = "1.1") - api(group = "org.bytedeco.javacpp-presets", name = "opencv", version = "3.0.0-1.1") - api(group = "org.bytedeco.javacpp-presets", name = "opencv", version = "3.0.0-1.1", classifier = os) + api(group = "org.bytedeco", name = "javacv", version = "1.3") + api(group = "org.bytedeco.javacpp-presets", name = "opencv", version = "3.4.3-1.4.3") + api(group = "org.bytedeco.javacpp-presets", name = "opencv", version = "3.4.3-1.4.3", classifier = if (withCuda) "$os-gpu" else os) api(group = "org.bytedeco.javacpp-presets", name = "videoinput", version = "0.200-1.1", classifier = os) - api(group = "org.bytedeco.javacpp-presets", name = "ffmpeg", version = "0.200-1.1", classifier = os) + api(group = "org.bytedeco.javacpp-presets", name = "ffmpeg", version = "0.200-1.3", classifier = os) api(group = "org.python", name = "jython", version = "2.7.0") api(group = "com.thoughtworks.xstream", name = "xstream", version = "1.4.10") api(group = "org.apache.commons", name = "commons-lang3", version = "3.5") @@ -59,6 +67,18 @@ tasks.withType().configureEach { } } +val writeCudaPropertiesTask = tasks.register("writeCudaProperties") { + description = "Generates a file to let the GRIP runtime know if its using CUDA-accelerated OpenCV." + outputFile = buildDir.resolve("resources/main/edu/wpi/grip/core/CUDA.properties") + comment = "Information about CUDA requirements for the GRIP runtime" + property("edu.wpi.grip.cuda.enabled", withCuda) + property("edu.wpi.grip.cuda.version", "10.0") // opencv-presets 3.4.3-1.4.3 is compiled with CUDA 10.0 +} + +tasks.withType().configureEach { + dependsOn(writeCudaPropertiesTask) +} + tasks.withType().configureEach { /* The icudt54b directory in Jython takes up 9 megabytes and doesn"t seem to do anything useful. */ exclude("org/python/icu/impl/data/icudt54b/") diff --git a/core/src/main/java/edu/wpi/grip/core/GripCoreModule.java b/core/src/main/java/edu/wpi/grip/core/GripCoreModule.java index d1b6e274f6..572e450b51 100644 --- a/core/src/main/java/edu/wpi/grip/core/GripCoreModule.java +++ b/core/src/main/java/edu/wpi/grip/core/GripCoreModule.java @@ -1,5 +1,10 @@ package edu.wpi.grip.core; +import edu.wpi.grip.core.cuda.AccelerationMode; +import edu.wpi.grip.core.cuda.CudaDetector; +import edu.wpi.grip.core.cuda.CudaVerifier; +import edu.wpi.grip.core.cuda.NullAccelerationMode; +import edu.wpi.grip.core.cuda.NullCudaDetector; import edu.wpi.grip.core.events.EventLogger; import edu.wpi.grip.core.events.UnexpectedThrowableEvent; import edu.wpi.grip.core.metrics.BenchmarkRunner; @@ -23,22 +28,18 @@ import com.google.common.eventbus.EventBus; import com.google.common.eventbus.SubscriberExceptionContext; import com.google.inject.AbstractModule; +import com.google.inject.Scopes; import com.google.inject.TypeLiteral; import com.google.inject.assistedinject.FactoryModuleBuilder; import com.google.inject.matcher.Matchers; +import com.google.inject.name.Names; import com.google.inject.spi.InjectionListener; import com.google.inject.spi.TypeEncounter; import com.google.inject.spi.TypeListener; -import java.io.IOException; -import java.util.logging.FileHandler; -import java.util.logging.Handler; +import java.util.Properties; import java.util.logging.Level; -import java.util.logging.LogManager; -import java.util.logging.LogRecord; import java.util.logging.Logger; -import java.util.logging.SimpleFormatter; -import java.util.logging.StreamHandler; import javax.annotation.Nullable; @@ -53,55 +54,6 @@ public class GripCoreModule extends AbstractModule { private static final Logger logger = Logger.getLogger(GripCoreModule.class.getName()); - // This is in a static initialization block so that we don't create a ton of - // log files when running tests - static { - //Set up the global level logger. This handles IO for all loggers. - final Logger globalLogger = LogManager.getLogManager().getLogger(""); - - try { - // Remove the default handlers that stream to System.err - for (Handler handler : globalLogger.getHandlers()) { - globalLogger.removeHandler(handler); - } - - GripFileManager.GRIP_DIRECTORY.mkdirs(); - final Handler fileHandler - = new FileHandler(GripFileManager.GRIP_DIRECTORY.getPath() + "/GRIP.log"); - - //Set level to handler and logger - fileHandler.setLevel(Level.INFO); - globalLogger.setLevel(Level.INFO); - - // We need to stream to System.out instead of System.err - final StreamHandler sh = new StreamHandler(System.out, new SimpleFormatter()) { - - @Override - public synchronized void publish(final LogRecord record) { - super.publish(record); - // For some reason this doesn't happen automatically. - // This will ensure we get all of the logs printed to the console immediately - // when running on a remote device. - flush(); - } - }; - sh.setLevel(Level.CONFIG); - - globalLogger.addHandler(sh); // Add stream handler - - globalLogger.addHandler(fileHandler); //Add the handler to the global logger - - fileHandler.setFormatter(new SimpleFormatter()); //log in text, not xml - - globalLogger.config("Configuration done."); //Log that we are done setting up the logger - globalLogger.config("GRIP Version: " + edu.wpi.grip.core.Main.class.getPackage() - .getImplementationVersion()); - - } catch (IOException exception) { //Something happened setting up file IO - throw new IllegalStateException("Failed to configure the Logger", exception); - } - } - /* * This class should not be used in tests. Use GRIPCoreTestModule for tests. */ @@ -138,6 +90,16 @@ public void hear(TypeLiteral type, TypeEncounter encounter) { bind(ConnectionValidator.class).to(Pipeline.class); bind(Source.SourceFactory.class).to(Source.SourceFactoryImpl.class); + // Bind CUDA-specific stuff to default values + // These will be overridden by the GripCudaModule at app runtime, but this lets + // automated tests assume CPU-only operation modes + bind(CudaDetector.class).to(NullCudaDetector.class); + bind(AccelerationMode.class).to(NullAccelerationMode.class); + bind(CudaVerifier.class).in(Scopes.SINGLETON); + bind(Properties.class) + .annotatedWith(Names.named("cudaProperties")) + .toInstance(new Properties()); + bind(InputSocket.Factory.class).to(InputSocketImpl.FactoryImpl.class); bind(OutputSocket.Factory.class).to(OutputSocketImpl.FactoryImpl.class); install(new FactoryModuleBuilder() diff --git a/core/src/main/java/edu/wpi/grip/core/GripCudaModule.java b/core/src/main/java/edu/wpi/grip/core/GripCudaModule.java new file mode 100644 index 0000000000..1b12362e18 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/GripCudaModule.java @@ -0,0 +1,54 @@ +package edu.wpi.grip.core; + +import edu.wpi.grip.core.cuda.AccelerationMode; +import edu.wpi.grip.core.cuda.CudaAccelerationMode; +import edu.wpi.grip.core.cuda.CudaDetector; +import edu.wpi.grip.core.cuda.CudaVerifier; +import edu.wpi.grip.core.cuda.LoadingCudaDetector; +import edu.wpi.grip.core.cuda.NullAccelerationMode; + +import com.google.inject.AbstractModule; +import com.google.inject.Scopes; +import com.google.inject.name.Names; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; +import java.util.logging.Level; +import java.util.logging.Logger; + +public class GripCudaModule extends AbstractModule { + + private static final Logger logger = Logger.getLogger(GripCudaModule.class.getName()); + private static final String CUDA_ENABLED_KEY = "edu.wpi.grip.cuda.enabled"; + + @Override + protected void configure() { + bind(CudaDetector.class).to(LoadingCudaDetector.class); + + Properties cudaProperties = getCudaProperties(); + bind(Properties.class) + .annotatedWith(Names.named("cudaProperties")) + .toInstance(cudaProperties); + + if (Boolean.valueOf(cudaProperties.getProperty(CUDA_ENABLED_KEY, "false"))) { + bind(AccelerationMode.class).to(CudaAccelerationMode.class); + } else { + bind(AccelerationMode.class).to(NullAccelerationMode.class); + } + + bind(CudaVerifier.class).in(Scopes.SINGLETON); + } + + private Properties getCudaProperties() { + try (InputStream resourceAsStream = getClass().getResourceAsStream("CUDA.properties")) { + Properties cudaProps = new Properties(); + cudaProps.load(resourceAsStream); + return cudaProps; + } catch (IOException e) { + logger.log(Level.WARNING, "Could not read CUDA properties", e); + return new Properties(); + } + } + +} diff --git a/core/src/main/java/edu/wpi/grip/core/Loggers.java b/core/src/main/java/edu/wpi/grip/core/Loggers.java new file mode 100644 index 0000000000..31418e0443 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/Loggers.java @@ -0,0 +1,70 @@ +package edu.wpi.grip.core; + +import java.io.IOException; +import java.util.logging.FileHandler; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogManager; +import java.util.logging.LogRecord; +import java.util.logging.Logger; +import java.util.logging.SimpleFormatter; +import java.util.logging.StreamHandler; + +public final class Loggers { + + private Loggers() { + throw new UnsupportedOperationException("This is a utility class!"); + } + + /** + * Sets up loggers to print to stdout and to ~/GRIP/GRIP.log. This should only be called once + * in the application lifecycle, at startup. + */ + public static void setupLoggers() { + // Set up the global level logger. This handles IO for all loggers. + final Logger globalLogger = LogManager.getLogManager().getLogger(""); + + try { + // Remove the default handlers that stream to System.err + for (Handler handler : globalLogger.getHandlers()) { + globalLogger.removeHandler(handler); + } + + GripFileManager.GRIP_DIRECTORY.mkdirs(); + final Handler fileHandler + = new FileHandler(GripFileManager.GRIP_DIRECTORY.getPath() + "/GRIP.log"); + + //Set level to handler and logger + fileHandler.setLevel(Level.INFO); + globalLogger.setLevel(Level.INFO); + + // We need to stream to System.out instead of System.err + final StreamHandler sh = new StreamHandler(System.out, new SimpleFormatter()) { + + @Override + public synchronized void publish(final LogRecord record) { + super.publish(record); + // For some reason this doesn't happen automatically. + // This will ensure we get all of the logs printed to the console immediately + // when running on a remote device. + flush(); + } + }; + sh.setLevel(Level.CONFIG); + + globalLogger.addHandler(sh); // Add stream handler + + globalLogger.addHandler(fileHandler); //Add the handler to the global logger + + fileHandler.setFormatter(new SimpleFormatter()); //log in text, not xml + + globalLogger.config("Configuration done."); //Log that we are done setting up the logger + globalLogger.config("GRIP Version: " + edu.wpi.grip.core.Main.class.getPackage() + .getImplementationVersion()); + + } catch (IOException exception) { //Something happened setting up file IO + throw new IllegalStateException("Failed to configure the Logger", exception); + } + } + +} diff --git a/core/src/main/java/edu/wpi/grip/core/Main.java b/core/src/main/java/edu/wpi/grip/core/Main.java index d58ca3e9b5..e06def2808 100644 --- a/core/src/main/java/edu/wpi/grip/core/Main.java +++ b/core/src/main/java/edu/wpi/grip/core/Main.java @@ -1,5 +1,6 @@ package edu.wpi.grip.core; +import edu.wpi.grip.core.cuda.CudaVerifier; import edu.wpi.grip.core.events.ExceptionClearedEvent; import edu.wpi.grip.core.events.ExceptionEvent; import edu.wpi.grip.core.exception.GripServerException; @@ -55,8 +56,20 @@ public class Main { @SuppressWarnings("JavadocMethod") public static void main(String[] args) throws IOException, InterruptedException { new CoreCommandLineHelper().parse(args); // Check for help or version before doing anything else - final Injector injector = Guice.createInjector(Modules.override(new GripCoreModule(), - new GripFileModule(), new GripSourcesHardwareModule()).with(new GripNetworkModule())); + Loggers.setupLoggers(); + + // Verify CUDA before using the core module, since that will cause OpenCV to be loaded, + // which will crash the app if we use CUDA and it's not available + GripCudaModule cudaModule = new GripCudaModule(); + CudaVerifier cudaVerifier = Guice.createInjector(cudaModule).getInstance(CudaVerifier.class); + cudaVerifier.verifyCuda(); + + final Injector injector = Guice.createInjector( + Modules.override( + new GripCoreModule(), + new GripFileModule(), + new GripSourcesHardwareModule() + ).with(new GripNetworkModule(), cudaModule)); injector.getInstance(Main.class).start(args); } @@ -81,7 +94,7 @@ public void start(String[] args) throws IOException, InterruptedException { gripServer.start(); } catch (GripServerException e) { logger.log(Level.SEVERE, "The HTTP server could not be started", e); - SafeShutdown.exit(1); + SafeShutdown.exit(SafeShutdown.ExitCode.HTTP_SERVER_COULD_NOT_START); } if (pipelineRunner.state() == Service.State.NEW) { diff --git a/core/src/main/java/edu/wpi/grip/core/MatWrapper.java b/core/src/main/java/edu/wpi/grip/core/MatWrapper.java new file mode 100644 index 0000000000..5636a68662 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/MatWrapper.java @@ -0,0 +1,382 @@ +package edu.wpi.grip.core; + +import org.bytedeco.javacpp.opencv_core; +import org.bytedeco.javacpp.opencv_core.GpuMat; +import org.bytedeco.javacpp.opencv_core.Mat; +import org.bytedeco.javacpp.opencv_core.Size; + +import java.util.Objects; +import java.util.function.Function; + +import static org.bytedeco.javacpp.opencv_core.CV_16S; +import static org.bytedeco.javacpp.opencv_core.CV_16U; +import static org.bytedeco.javacpp.opencv_core.CV_32F; +import static org.bytedeco.javacpp.opencv_core.CV_32S; +import static org.bytedeco.javacpp.opencv_core.CV_64F; +import static org.bytedeco.javacpp.opencv_core.CV_8S; +import static org.bytedeco.javacpp.opencv_core.CV_8U; +import static org.bytedeco.javacpp.opencv_core.IPL_DEPTH_16S; +import static org.bytedeco.javacpp.opencv_core.IPL_DEPTH_16U; +import static org.bytedeco.javacpp.opencv_core.IPL_DEPTH_1U; +import static org.bytedeco.javacpp.opencv_core.IPL_DEPTH_32F; +import static org.bytedeco.javacpp.opencv_core.IPL_DEPTH_32S; +import static org.bytedeco.javacpp.opencv_core.IPL_DEPTH_64F; +import static org.bytedeco.javacpp.opencv_core.IPL_DEPTH_8S; +import static org.bytedeco.javacpp.opencv_core.IPL_DEPTH_8U; + +/** + * Wraps a GPU mat and a CPU mat and allows device memory and host memory + * to be used semi-transparently. A wrapper may change between wrapping an image in host memory or + * an image in GPU memory. A wrapper is used to minimize copies between host and device memory, + * which may take longer than the time savings of using a CUDA-accelerated operation. + * + *

Data is lazily copied between host and device memory when needed. Wrappers that + * are only accessed from CPU operations will never have their data stored in device memory. + * Accessing a wrapper from CUDA land with {@link #getGpu()} will first copy the existing data from + * host memory to device, if the wrapper was most recently accessed from CPU code (from either + * {@link #getCpu()} or {@link #rawCpu()}). This behavior also applies in the reverse, for wrappers + * accessed from CPU land when they have been most recently used from CUDA code. + */ +@SuppressWarnings("PMD.GodClass") +public final class MatWrapper { + + private final Mat cpuMat; + private final GpuMat gpuMat; + + /** + * Flags whether or not the wrapped data is in host memory. Data may still be available in device + * memory, but the value in host memory will be the working version. + */ + private boolean isCpu = true; + + /** + * Flags whether or not the wrapped value has been modified since the most recent read. This + * is used so that the value is only copied between host and device when it's needed. + */ + private boolean changed = false; + + /** + * Creates an empty wrapper. Both mats are empty and the wrapper is treated as a CPU mat. + */ + public static MatWrapper emptyWrapper() { + return new MatWrapper(new Mat(), new GpuMat()); + } + + /** + * Creates a wrapper around an existing mat in host memory. + */ + public static MatWrapper wrap(Mat cpuMat) { + return new MatWrapper(cpuMat, new GpuMat()); + } + + /** + * Creates a wrapper around an existing mat in GPU memory. + */ + public static MatWrapper wrap(GpuMat gpuMat) { + return new MatWrapper(new Mat(), gpuMat); + } + + /** + * Creates a wrapper around existing mats in host and device memory. + * + * @param cpuMat the mat in host memory to use + * @param gpuMat the mat in device memory to use + */ + public static MatWrapper using(Mat cpuMat, GpuMat gpuMat) { + return new MatWrapper(cpuMat, gpuMat); + } + + private MatWrapper(Mat cpuMat, GpuMat gpuMat) { + this.cpuMat = Objects.requireNonNull(cpuMat, "cpuMat"); + this.gpuMat = Objects.requireNonNull(gpuMat, "gpuMat"); + } + + /** + * Checks if this is a wrapper around a CPU mat. + */ + public boolean isCpu() { + return isCpu; + } + + /** + * Checks if this is a wrapper around a GPU mat. + */ + public boolean isGpu() { + return !isCpu; + } + + /** + * Gets the raw CPU mat. This should only be used when this mat is used as a {@code dst} parameter + * to an OpenCV function. If you want to get the current value as a mat in host memory, use + * {@link #getCpu()}. + */ + public Mat rawCpu() { + // Assume the mat is about to be modified as a `dst` parameter to an OpenCV function + // running on the CPU + isCpu = true; + changed = true; + return cpuMat; + } + + /** + * Gets the raw GPU mat. This should only be used when this mat is used as a {@code dst} parameter + * to an OpenCV function. If you want to get the current value as a mat in GPU memory, use + * {@link #getGpu()}. + */ + public GpuMat rawGpu() { + // Assume the mat is about to be modified as a `dst` parameter to an OpenCV function + // running on a CUDA device + isCpu = false; + changed = true; + return gpuMat; + } + + /** + * Gets this mat as a mat in host memory. If this is {@link #isGpu() backed by GPU memory}, the + * device memory will be copied into the CPU mat before being returned. This copy only happens + * after {@link #set(GpuMat) set(GpuMat)} is called, and only once between successive calls; + * invocations of this method after the first copy will not perform another. + */ + public Mat getCpu() { + if (changed && !isCpu) { + gpuMat.download(cpuMat); + changed = false; + } + return cpuMat; + } + + /** + * Gets this mat as a mat in GPU memory. If this is {@link #isCpu() backed by host memory}, the + * host memory will be copied into the GPU mat before being returned. This copy only happens + * after {@link #set(Mat) set(Mat)} is called, and only once between successive calls; + * invocations of this method after the first copy will not perform another. + */ + public GpuMat getGpu() { + if (changed && isCpu) { + gpuMat.upload(cpuMat); + changed = false; + } + return gpuMat; + } + + /** + * Sets this as being backed by an image in host memory. The data in the given mat will be copied + * into the internal CPU mat, but not the GPU mat until {@link #getGpu()} is called. This avoids + * unnecessary memory copies between GPU and host memory. + */ + public void set(Mat mat) { + mat.copyTo(cpuMat); + isCpu = true; + changed = true; + } + + /** + * Sets this as being backed by an image in GPU memory. The data in the given mat will be copied + * into the internal GPU mat, but not the mat residing in host memory until {@link #getCpu()} is + * called. This avoids unnecessary memory copies between GPU and host memory. + */ + public void set(GpuMat mat) { + gpuMat.put(mat); + isCpu = false; + changed = true; + } + + /** + * Sets this as being backed by the given wrapper. This wrapper will be functionally equivalent + * to the one given. + */ + public void set(MatWrapper wrapper) { + if (wrapper.isCpu()) { + set(wrapper.cpuMat); + } else { + set(wrapper.gpuMat); + } + } + + /** + * Copies the data of this wrapper to a mat in host memory. + */ + public void copyTo(Mat mat) { + if (isCpu) { + cpuMat.copyTo(mat); + } else { + gpuMat.download(mat); + } + } + + /** + * Copies the data of this wrapper to a mat in GPU memory. + */ + public void copyTo(GpuMat mat) { + if (isCpu) { + mat.upload(cpuMat); + } else { + mat.put(gpuMat); + } + } + + /** + * Copies the data of this wrapper into another. Equivalent to {@code wrapper.set(this)} + */ + public void copyTo(MatWrapper wrapper) { + wrapper.set(this); + } + + /** + * Extracts a property shared by both Mats and GpuMats. Unfortunately, they don't share a common + * API, so we have to do something like this. + *

+ * Example use: + * 


+   * Size size = extract(Mat::size, GpuMat::size);
+   *
+ *

+ * + * @param ifCpu the function to call if this is backed by a mat in host memory + * @param ifGpu the function to call if this is backed by a mat in GPU memory + * @param the type of the property to extract + */ + private T extract(Function ifCpu, Function ifGpu) { + if (isCpu) { + return ifCpu.apply(cpuMat); + } else { + return ifGpu.apply(gpuMat); + } + } + + /** + * Gets the number of columns in this image. + */ + public int cols() { + return extract(Mat::cols, GpuMat::cols); + } + + /** + * Gets the number of rows in this image. + */ + public int rows() { + return extract(Mat::rows, GpuMat::rows); + } + + /** + * Gets the type of the data format of this image. + */ + public int type() { + return extract(Mat::type, GpuMat::type); + } + + /** + * Gets the number of color channels in this image. + */ + public int channels() { + return extract(Mat::channels, GpuMat::channels); + } + + /** + * Gets the channel depth of this image. + */ + public int depth() { + return extract(Mat::depth, GpuMat::depth); + } + + /** + * Checks if this image is empty. + */ + public boolean empty() { + return extract(Mat::empty, GpuMat::empty); + } + + /** + * Gets the size (width by height) of this image. + */ + public Size size() { + return extract(Mat::size, GpuMat::size); + } + + /** + * Gets the maximum possible value able to be held as a single element in this image. + */ + @SuppressWarnings("PMD") + public double highValue() { + return extract(Mat::highValue, g -> { + double highValue = 0.0; + switch (arrayDepth(g)) { + case IPL_DEPTH_8U: + highValue = 0xFF; + break; + case IPL_DEPTH_16U: + highValue = 0xFFFF; + break; + case IPL_DEPTH_8S: + highValue = Byte.MAX_VALUE; + break; + case IPL_DEPTH_16S: + highValue = Short.MAX_VALUE; + break; + case IPL_DEPTH_32S: + highValue = Integer.MAX_VALUE; + break; + case IPL_DEPTH_1U: + case IPL_DEPTH_32F: + case IPL_DEPTH_64F: + highValue = 1.0; + break; + default: + assert false; + } + return highValue; + }); + } + + private static int arrayDepth(GpuMat m) { + switch (m.depth()) { + case CV_8U: + return IPL_DEPTH_8U; + case CV_8S: + return IPL_DEPTH_8S; + case CV_16U: + return IPL_DEPTH_16U; + case CV_16S: + return IPL_DEPTH_16S; + case CV_32S: + return IPL_DEPTH_32S; + case CV_32F: + return IPL_DEPTH_32F; + case CV_64F: + return IPL_DEPTH_64F; + default: + throw new UnsupportedOperationException("Unsupported depth " + m.depth()); + } + } + + /** + * Allocates new array data if needed. + * + * @param rows New number of rows. + * @param cols New number of columns. + * @param type New matrix type. + */ + public void create(int rows, int cols, int type) { + if (isCpu) { + cpuMat.create(rows, cols, type); + } else { + gpuMat.create(rows, cols, type); + } + changed = true; + } + + /** + * Sets all or some of the array elements to the specified value. + * + * @param value Assigned scalar converted to the actual array type. + */ + public MatWrapper put(opencv_core.Scalar value) { + if (isCpu()) { + cpuMat.put(value); + } else { + gpuMat.setTo(value); + } + changed = true; + return this; + } +} diff --git a/core/src/main/java/edu/wpi/grip/core/cuda/AccelerationMode.java b/core/src/main/java/edu/wpi/grip/core/cuda/AccelerationMode.java new file mode 100644 index 0000000000..06a1b29fb1 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/cuda/AccelerationMode.java @@ -0,0 +1,13 @@ +package edu.wpi.grip.core.cuda; + +/** + * App-wide hardware acceleration mode. + */ +public interface AccelerationMode { + + /** + * Flag marking that GRIP is using CUDA-accelerated OpenCV. + */ + boolean isUsingCuda(); + +} diff --git a/core/src/main/java/edu/wpi/grip/core/cuda/CudaAccelerationMode.java b/core/src/main/java/edu/wpi/grip/core/cuda/CudaAccelerationMode.java new file mode 100644 index 0000000000..215bf59d50 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/cuda/CudaAccelerationMode.java @@ -0,0 +1,8 @@ +package edu.wpi.grip.core.cuda; + +public class CudaAccelerationMode implements AccelerationMode { + @Override + public boolean isUsingCuda() { + return true; + } +} diff --git a/core/src/main/java/edu/wpi/grip/core/cuda/CudaDetector.java b/core/src/main/java/edu/wpi/grip/core/cuda/CudaDetector.java new file mode 100644 index 0000000000..57127d3ba5 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/cuda/CudaDetector.java @@ -0,0 +1,13 @@ +package edu.wpi.grip.core.cuda; + +/** + * Detects CUDA installs. + */ +public interface CudaDetector { + + /** + * Checks if a CUDA runtime is installed that is compatible with what we need for OpenCV. + */ + boolean isCompatibleCudaInstalled(); + +} diff --git a/core/src/main/java/edu/wpi/grip/core/cuda/CudaVerifier.java b/core/src/main/java/edu/wpi/grip/core/cuda/CudaVerifier.java new file mode 100644 index 0000000000..ed61b13915 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/cuda/CudaVerifier.java @@ -0,0 +1,71 @@ +package edu.wpi.grip.core.cuda; + +import edu.wpi.grip.core.util.SafeShutdown; + +import com.google.common.annotations.VisibleForTesting; + +import java.util.Properties; +import java.util.logging.Logger; + +import javax.inject.Inject; +import javax.inject.Named; + +public class CudaVerifier { + + private static final Logger logger = Logger.getLogger(CudaVerifier.class.getName()); + + private static final String CUDA_VERSION_KEY = "edu.wpi.grip.cuda.version"; + + private final AccelerationMode accelerationMode; + private final CudaDetector cudaDetector; + private final String cudaVersion; + + @Inject + public CudaVerifier(AccelerationMode accelerationMode, + CudaDetector cudaDetector, + @Named("cudaProperties") Properties cudaProperties) { + this.accelerationMode = accelerationMode; + this.cudaDetector = cudaDetector; + this.cudaVersion = cudaProperties.getProperty(CUDA_VERSION_KEY, "Unknown"); + } + + /** + * Verifies the presence of a CUDA runtime, if required by the GRIP runtime, and exits the + * app if no compatible CUDA runtime is available. + */ + public void verifyCuda() { + if (!verify()) { + String message = "This version of GRIP requires CUDA version " + + cudaVersion + + " to be installed and an NVIDIA graphics card in your computer. " + + "If your computer does not have an NVIDIA graphics card, use a version of GRIP " + + "without CUDA acceleration. Otherwise, you need to install the appropriate CUDA " + + "runtime for your computer."; + logger.severe(message); + exit(); + } + } + + /** + * Verifies that, if GRIP is using CUDA acceleration, a compatible CUDA runtime is available. If + * GRIP is not using CUDA acceleration, this will always return {@code true}. + * + * @return false if GRIP is using CUDA acceleration but no compatible CUDA runtime is available, + * true otherwise + */ + public boolean verify() { + if (accelerationMode.isUsingCuda()) { + return cudaDetector.isCompatibleCudaInstalled(); + } else { + return true; + } + } + + /** + * Exits the application. + */ + @VisibleForTesting + void exit() { + SafeShutdown.exit(SafeShutdown.ExitCode.CUDA_UNAVAILABLE); + } +} diff --git a/core/src/main/java/edu/wpi/grip/core/cuda/LoadingCudaDetector.java b/core/src/main/java/edu/wpi/grip/core/cuda/LoadingCudaDetector.java new file mode 100644 index 0000000000..f19ffbb84b --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/cuda/LoadingCudaDetector.java @@ -0,0 +1,36 @@ +package edu.wpi.grip.core.cuda; + +import org.bytedeco.javacpp.Loader; +import org.bytedeco.javacpp.opencv_cudaarithm; + +/** + * Checks if CUDA is available by attempting to load one of the OpenCV CUDA class' JNI. If the JNI + * cannot be loaded, then no compatible CUDA runtime is available. This approach is probably the + * most flexible; it's OS-agnostic, since it lets the JVM handle loading the JNI libraries and + * linking, and doesn't require knowledge of CUDA installation locations - it just needs to be on + * the PATH. + * + *

Only one attempt is made to load the JNI, and the result is cached. Any later calls to + * {@link #isCompatibleCudaInstalled()} will simply return the cached value. + */ +public class LoadingCudaDetector implements CudaDetector { + + private volatile boolean hasCuda = false; + private volatile boolean checkedForCuda = false; + + @Override + public boolean isCompatibleCudaInstalled() { + if (!checkedForCuda) { + try { + Loader.load(opencv_cudaarithm.class); + hasCuda = true; + } catch (UnsatisfiedLinkError | NoClassDefFoundError e) { + // Couldn't load the JNI, no compatible CUDA runtime is available + hasCuda = false; + } + checkedForCuda = true; + } + + return hasCuda; + } +} diff --git a/core/src/main/java/edu/wpi/grip/core/cuda/NullAccelerationMode.java b/core/src/main/java/edu/wpi/grip/core/cuda/NullAccelerationMode.java new file mode 100644 index 0000000000..9272f7a4a2 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/cuda/NullAccelerationMode.java @@ -0,0 +1,8 @@ +package edu.wpi.grip.core.cuda; + +public class NullAccelerationMode implements AccelerationMode { + @Override + public boolean isUsingCuda() { + return false; + } +} diff --git a/core/src/main/java/edu/wpi/grip/core/cuda/NullCudaDetector.java b/core/src/main/java/edu/wpi/grip/core/cuda/NullCudaDetector.java new file mode 100644 index 0000000000..bb4ed1449d --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/cuda/NullCudaDetector.java @@ -0,0 +1,8 @@ +package edu.wpi.grip.core.cuda; + +public class NullCudaDetector implements CudaDetector { + @Override + public boolean isCompatibleCudaInstalled() { + return false; + } +} diff --git a/core/src/main/java/edu/wpi/grip/core/events/UnexpectedThrowableEvent.java b/core/src/main/java/edu/wpi/grip/core/events/UnexpectedThrowableEvent.java index b54ff29151..8897e9b1f4 100644 --- a/core/src/main/java/edu/wpi/grip/core/events/UnexpectedThrowableEvent.java +++ b/core/src/main/java/edu/wpi/grip/core/events/UnexpectedThrowableEvent.java @@ -55,7 +55,7 @@ public void handleSafely(UnexpectedThrowableEventHandler handler) { try { logger.log(Level.SEVERE, "Failed to handle safely", throwable); } finally { - SafeShutdown.exit(1); + SafeShutdown.exit(SafeShutdown.ExitCode.MISC_ERROR); } } finally { shutdownIfFatal(); @@ -72,7 +72,7 @@ public void shutdownIfFatal() { logger.log(Level.SEVERE, "Shutting down from error", throwable); } finally { // If all else fails then shutdown - SafeShutdown.exit(1); + SafeShutdown.exit(SafeShutdown.ExitCode.MISC_ERROR); } } } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/CVOperations.java b/core/src/main/java/edu/wpi/grip/core/operations/CVOperations.java index af12ad0da7..7df0d45a75 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/CVOperations.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/CVOperations.java @@ -1,6 +1,5 @@ package edu.wpi.grip.core.operations; - import edu.wpi.grip.core.OperationMetaData; import edu.wpi.grip.core.events.OperationAddedEvent; import edu.wpi.grip.core.operations.opencv.CVOperation; @@ -24,16 +23,48 @@ import com.google.common.eventbus.EventBus; import com.google.inject.Inject; -import org.bytedeco.javacpp.opencv_core; import org.bytedeco.javacpp.opencv_core.Point; import org.bytedeco.javacpp.opencv_core.Scalar; import org.bytedeco.javacpp.opencv_core.Size; +import org.bytedeco.javacpp.opencv_cudaarithm; +import org.bytedeco.javacpp.opencv_cudafilters.Filter; +import org.bytedeco.javacpp.opencv_cudaimgproc; import org.bytedeco.javacpp.opencv_imgproc; +import static org.bytedeco.javacpp.opencv_core.absdiff; +import static org.bytedeco.javacpp.opencv_core.add; +import static org.bytedeco.javacpp.opencv_core.addWeighted; +import static org.bytedeco.javacpp.opencv_core.bitwise_and; +import static org.bytedeco.javacpp.opencv_core.bitwise_not; +import static org.bytedeco.javacpp.opencv_core.bitwise_or; +import static org.bytedeco.javacpp.opencv_core.bitwise_xor; +import static org.bytedeco.javacpp.opencv_core.compare; +import static org.bytedeco.javacpp.opencv_core.divide; +import static org.bytedeco.javacpp.opencv_core.extractChannel; +import static org.bytedeco.javacpp.opencv_core.flip; +import static org.bytedeco.javacpp.opencv_core.max; +import static org.bytedeco.javacpp.opencv_core.min; +import static org.bytedeco.javacpp.opencv_core.multiply; +import static org.bytedeco.javacpp.opencv_core.scaleAdd; +import static org.bytedeco.javacpp.opencv_core.subtract; +import static org.bytedeco.javacpp.opencv_core.transpose; +import static org.bytedeco.javacpp.opencv_cudafilters.createSobelFilter; +import static org.bytedeco.javacpp.opencv_imgproc.GaussianBlur; +import static org.bytedeco.javacpp.opencv_imgproc.Laplacian; +import static org.bytedeco.javacpp.opencv_imgproc.Sobel; +import static org.bytedeco.javacpp.opencv_imgproc.adaptiveThreshold; +import static org.bytedeco.javacpp.opencv_imgproc.applyColorMap; +import static org.bytedeco.javacpp.opencv_imgproc.cvtColor; +import static org.bytedeco.javacpp.opencv_imgproc.dilate; +import static org.bytedeco.javacpp.opencv_imgproc.medianBlur; +import static org.bytedeco.javacpp.opencv_imgproc.rectangle; +import static org.bytedeco.javacpp.opencv_imgproc.resize; +import static org.bytedeco.javacpp.opencv_imgproc.threshold; + /** * A list of all of the raw opencv operations. */ -@SuppressWarnings("PMD.AvoidDuplicateLiterals") +@SuppressWarnings({"PMD.AvoidDuplicateLiterals", "CodeBlock2Expr"}) public class CVOperations { private final EventBus eventBus; @@ -47,138 +78,196 @@ public class CVOperations { this.coreOperations = ImmutableList.of( new OperationMetaData(CVOperation.defaults("CV absdiff", "Calculate the per-element absolute difference of two images."), - templateFactory.createAllMatTwoSource(opencv_core::absdiff)), + templateFactory.createAllMatTwoSourceCuda((src1, src2, useCuda, dst) -> { + if (useCuda) { + opencv_cudaarithm.absdiff(src1.getGpu(), src2.getGpu(), dst.rawGpu()); + } else { + absdiff(src1.getCpu(), src2.getCpu(), dst.rawCpu()); + } + })), new OperationMetaData(CVOperation.defaults("CV add", "Calculate the per-pixel sum of two images."), - templateFactory.createAllMatTwoSource(opencv_core::add)), + templateFactory.createAllMatTwoSourceCuda((src1, src2, useCuda, dst) -> { + if (useCuda) { + opencv_cudaarithm.add(src1.getGpu(), src2.getGpu(), dst.rawGpu()); + } else { + add(src1.getCpu(), src2.getCpu(), dst.rawCpu()); + } + })), new OperationMetaData(CVOperation.defaults("CV addWeighted", "Calculate the weighted sum of two images."), - templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src1", false), + templateFactory.createCuda( + SocketHints.createImageSocketHint("src1"), SocketHints.Inputs.createNumberSpinnerSocketHint("alpha", 0), - SocketHints.Inputs.createMatSocketHint("src2", false), + SocketHints.createImageSocketHint("src2"), SocketHints.Inputs.createNumberSpinnerSocketHint("beta", 0), SocketHints.Inputs.createNumberSpinnerSocketHint("gamma", 0), - SocketHints.Outputs.createMatSocketHint("dst"), - (src1, alpha, src2, beta, gamma, dst) -> { - opencv_core.addWeighted(src1, alpha.doubleValue(), src2, beta.doubleValue(), - gamma.doubleValue(), dst); + SocketHints.createImageSocketHint("dst"), + (src1, alpha, src2, beta, gamma, useCuda, dst) -> { + if (useCuda) { + opencv_cudaarithm.addWeighted(src1.getGpu(), alpha.doubleValue(), src2.getGpu(), + beta.doubleValue(), gamma.doubleValue(), dst.rawGpu()); + } else { + addWeighted(src1.getCpu(), alpha.doubleValue(), src2.getCpu(), + beta.doubleValue(), gamma.doubleValue(), dst.rawCpu()); + } } )), new OperationMetaData(CVOperation.defaults("CV bitwise_and", "Calculate the per-element bitwise conjunction of two images."), - templateFactory.createAllMatTwoSource(opencv_core::bitwise_and)), + templateFactory.createAllMatTwoSourceCuda((src1, src2, useCuda, dst) -> { + if (useCuda) { + opencv_cudaarithm.bitwise_and(src1.getGpu(), src2.getGpu(), dst.rawGpu()); + } else { + bitwise_and(src1.getCpu(), src2.getCpu(), dst.rawCpu()); + } + })), new OperationMetaData(CVOperation.defaults("CV bitwise_not", "Calculate per-element bit-wise inversion of an image."), - templateFactory.createAllMatOneSource(opencv_core::bitwise_not)), + templateFactory.createAllMatOneSourceCuda((src, useCuda, dst) -> { + if (useCuda) { + opencv_cudaarithm.bitwise_not(src.getGpu(), dst.rawGpu()); + } else { + bitwise_not(src.getCpu(), dst.rawCpu()); + } + })), new OperationMetaData(CVOperation.defaults("CV bitwise_or", "Calculate the per-element bit-wise disjunction of two images."), - templateFactory.createAllMatTwoSource(opencv_core::bitwise_or)), + templateFactory.createAllMatTwoSourceCuda((src1, src2, useCuda, dst) -> { + if (useCuda) { + opencv_cudaarithm.bitwise_or(src1.getGpu(), src2.getGpu(), dst.rawGpu()); + } else { + bitwise_or(src1.getCpu(), src2.getCpu(), dst.rawCpu()); + } + })), new OperationMetaData(CVOperation.defaults("CV bitwise_xor", "Calculate the per-element bit-wise \"exclusive or\" on two images."), - templateFactory.createAllMatTwoSource(opencv_core::bitwise_xor)), + templateFactory.createAllMatTwoSourceCuda((src1, src2, useCuda, dst) -> { + if (useCuda) { + opencv_cudaarithm.bitwise_xor(src1.getGpu(), src2.getGpu(), dst.rawGpu()); + } else { + bitwise_xor(src1.getCpu(), src2.getCpu(), dst.rawCpu()); + } + })), new OperationMetaData(CVOperation.defaults("CV compare", "Compare each pixel in two images using a given rule."), - templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src1", false), - SocketHints.Inputs.createMatSocketHint("src2", false), + templateFactory.createCuda( + SocketHints.createImageSocketHint("src1"), + SocketHints.createImageSocketHint("src2"), SocketHints.createEnumSocketHint("cmpop", CmpTypesEnum.CMP_EQ), - SocketHints.Outputs.createMatSocketHint("dst"), - (src1, src2, cmp, dst) -> { - opencv_core.compare(src1, src2, dst, cmp.value); + SocketHints.createImageSocketHint("dst"), + (src1, src2, cmp, useCuda, dst) -> { + int cmpop = cmp.value; + if (useCuda) { + opencv_cudaarithm.compare(src1.getGpu(), src2.getGpu(), dst.rawGpu(), cmpop); + } else { + compare(src1.getCpu(), src2.getCpu(), dst.rawCpu(), cmpop); + } } )), new OperationMetaData(CVOperation.defaults("CV divide", "Perform per-pixel division of two images."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src1", false), - SocketHints.Inputs.createMatSocketHint("src2", false), + SocketHints.createImageSocketHint("src1"), + SocketHints.createImageSocketHint("src2"), SocketHints.Inputs.createNumberSpinnerSocketHint("scale", 1.0, -Double.MAX_VALUE, Double.MAX_VALUE), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src1, src2, scale, dst) -> { - opencv_core.divide(src1, src2, dst, scale.doubleValue(), -1); + divide(src1.getCpu(), src2.getCpu(), dst.rawCpu(), scale.doubleValue(), -1); } )), new OperationMetaData(CVOperation.defaults("CV extractChannel", "Extract a single channel from a image."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + SocketHints.createImageSocketHint("src"), SocketHints.Inputs.createNumberSpinnerSocketHint("channel", 0, 0, Integer .MAX_VALUE), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src1, coi, dst) -> { - opencv_core.extractChannel(src1, dst, coi.intValue()); + extractChannel(src1.getCpu(), dst.rawCpu(), coi.intValue()); } )), new OperationMetaData(CVOperation.defaults("CV flip", "Flip image around vertical, horizontal, or both axes."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + SocketHints.createImageSocketHint("src"), SocketHints.createEnumSocketHint("flipCode", FlipCode.Y_AXIS), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, flipCode, dst) -> { - opencv_core.flip(src, dst, flipCode.value); + flip(src.getCpu(), dst.rawCpu(), flipCode.value); } )), new OperationMetaData(CVOperation.defaults("CV max", "Calculate per-element maximum of two images."), - templateFactory.createAllMatTwoSource(opencv_core::max)), + templateFactory.createAllMatTwoSource((src1, src2, dst) -> { + max(src1.getCpu(), src2.getCpu(), dst.rawCpu()); + })), new OperationMetaData(CVOperation.defaults("CV min", "Calculate the per-element minimum of two images."), - templateFactory.createAllMatTwoSource(opencv_core::min)), + templateFactory.createAllMatTwoSource((src1, src2, dst) -> { + min(src1.getCpu(), src2.getCpu(), dst.rawCpu()); + })), new OperationMetaData(CVOperation.defaults("CV multiply", "Calculate the per-pixel scaled product of two images."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src1", false), - SocketHints.Inputs.createMatSocketHint("src2", false), + SocketHints.createImageSocketHint("src1"), + SocketHints.createImageSocketHint("src2"), SocketHints.Inputs.createNumberSpinnerSocketHint("scale", 1.0, Integer.MIN_VALUE, Integer.MAX_VALUE), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src1, src2, scale, dst) -> { - opencv_core.multiply(src1, src2, dst, scale.doubleValue(), -1); + multiply(src1.getCpu(), src2.getCpu(), dst.getCpu(), scale.doubleValue(), -1); } )), new OperationMetaData(CVOperation.defaults("CV scaleAdd", "Calculate the sum of two images where one image is multiplied by a scalar."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src1", false), + SocketHints.createImageSocketHint("src1"), SocketHints.Inputs.createNumberSpinnerSocketHint("scale", 1.0), - SocketHints.Inputs.createMatSocketHint("src2", false), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("src2"), + SocketHints.createImageSocketHint("dst"), (src1, alpha, src2, dst) -> { - opencv_core.scaleAdd(src1, alpha.doubleValue(), src2, dst); + scaleAdd(src1.getCpu(), alpha.doubleValue(), src2.getCpu(), dst.rawCpu()); } )), new OperationMetaData(CVOperation.defaults("CV subtract", "Calculate the per-pixel difference between two images."), - templateFactory.createAllMatTwoSource(opencv_core::subtract)), + templateFactory.createAllMatTwoSourceCuda((src1, src2, useCuda, dst) -> { + if (useCuda) { + opencv_cudaarithm.subtract(src1.getGpu(), src2.getGpu(), dst.rawGpu()); + } else { + subtract(src1.getCpu(), src2.getCpu(), dst.rawCpu()); + } + })), new OperationMetaData(CVOperation.defaults("CV transpose", "Calculate the transpose of an image."), - templateFactory.createAllMatOneSource(opencv_core::transpose)) + templateFactory.createAllMatOneSource((src, dst) -> { + transpose(src.getCpu(), dst.rawCpu()); + })) ); this.imgprocOperation = ImmutableList.of( new OperationMetaData(CVOperation.defaults("CV adaptiveThreshold", "Transforms a grayscale image to a binary image)."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + SocketHints.createImageSocketHint("src"), SocketHints.Inputs.createNumberSpinnerSocketHint("maxValue", 0.0), SocketHints.createEnumSocketHint("adaptiveMethod", AdaptiveThresholdTypesEnum.ADAPTIVE_THRESH_MEAN_C), @@ -186,9 +275,9 @@ public class CVOperations { CVAdaptThresholdTypesEnum.THRESH_BINARY), SocketHints.Inputs.createNumberSpinnerSocketHint("blockSize", 0.0), SocketHints.Inputs.createNumberSpinnerSocketHint("C", 0.0), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, maxValue, adaptiveMethod, thresholdType, blockSize, c, dst) -> { - opencv_imgproc.adaptiveThreshold(src, dst, maxValue.doubleValue(), + adaptiveThreshold(src.getCpu(), dst.rawCpu(), maxValue.doubleValue(), adaptiveMethod.value, thresholdType.value, blockSize.intValue(), c .doubleValue()); } @@ -197,54 +286,43 @@ public class CVOperations { new OperationMetaData(CVOperation.defaults("CV applyColorMap", "Apply a MATLAB equivalent colormap to an image."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + SocketHints.createImageSocketHint("src"), SocketHints.createEnumSocketHint("colormap", ColormapTypesEnum.COLORMAP_AUTUMN), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, colormap, dst) -> { - opencv_imgproc.applyColorMap(src, dst, colormap.value); - } - )), - - new OperationMetaData(CVOperation.defaults("CV Canny", - "Apply a \"canny edge detection\" algorithm to an image."), - templateFactory.create( - SocketHints.Inputs.createMatSocketHint("image", false), - SocketHints.Inputs.createNumberSpinnerSocketHint("threshold1", 0.0), - SocketHints.Inputs.createNumberSpinnerSocketHint("threshold2", 0.0), - SocketHints.Inputs.createNumberSpinnerSocketHint("apertureSize", 3), - SocketHints.Inputs.createCheckboxSocketHint("L2gradient", false), - SocketHints.Outputs.createMatSocketHint("edges"), - (image, threshold1, threshold2, apertureSize, l2gradient, edges) -> { - opencv_imgproc.Canny(image, edges, threshold1.doubleValue(), threshold2 - .doubleValue(), apertureSize.intValue(), l2gradient); + applyColorMap(src.getCpu(), dst.rawCpu(), colormap.value); } )), new OperationMetaData(CVOperation.defaults("CV cvtColor", "Convert an image from one color space to another."), - templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + templateFactory.createCuda( + SocketHints.createImageSocketHint("src"), SocketHints.createEnumSocketHint("code", ColorConversionCodesEnum.COLOR_BGR2BGRA), - SocketHints.Outputs.createMatSocketHint("dst"), - (src, code, dst) -> { - opencv_imgproc.cvtColor(src, dst, code.value); + SocketHints.createImageSocketHint("dst"), + (src, code, useCuda, dst) -> { + if (useCuda) { + opencv_cudaimgproc.cvtColor(dst.getGpu(), dst.rawGpu(), code.value); + } else { + cvtColor(src.getCpu(), dst.rawCpu(), code.value); + } } )), new OperationMetaData(CVOperation.defaults("CV dilate", "Expands areas of higher values in an image."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), - SocketHints.Inputs.createMatSocketHint("kernel", true), + SocketHints.createImageSocketHint("src"), + SocketHints.createImageSocketHint("kernel"), new SocketHint.Builder<>(Point.class).identifier("anchor").initialValueSupplier( () -> new Point(-1, -1)).build(), SocketHints.Inputs.createNumberSpinnerSocketHint("iterations", 1), SocketHints.createEnumSocketHint("borderType", BorderTypesEnum.BORDER_CONSTANT), new SocketHint.Builder<>(Scalar.class).identifier("borderValue") .initialValueSupplier(opencv_imgproc::morphologyDefaultBorderValue).build(), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, kernel, anchor, iterations, borderType, borderValue, dst) -> { - opencv_imgproc.dilate(src, dst, kernel, anchor, iterations.intValue(), + dilate(src.getCpu(), dst.rawCpu(), kernel.getCpu(), anchor, iterations.intValue(), borderType.value, borderValue); } )), @@ -252,33 +330,33 @@ public class CVOperations { new OperationMetaData(CVOperation.defaults("CV erode", "Expands areas of lower values in an image."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), - SocketHints.Inputs.createMatSocketHint("kernel", true), + SocketHints.createImageSocketHint("src"), + SocketHints.createImageSocketHint("kernel"), new SocketHint.Builder<>(Point.class).identifier("anchor").initialValueSupplier( () -> new Point(-1, -1)).build(), SocketHints.Inputs.createNumberSpinnerSocketHint("iterations", 1), SocketHints.createEnumSocketHint("borderType", BorderTypesEnum.BORDER_CONSTANT), new SocketHint.Builder<>(Scalar.class).identifier("borderValue") .initialValueSupplier(opencv_imgproc::morphologyDefaultBorderValue).build(), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, kernel, anchor, iterations, borderType, borderValue, dst) -> { - opencv_imgproc.erode(src, dst, kernel, anchor, iterations.intValue(), - borderType.value, borderValue); + opencv_imgproc.erode(src.getCpu(), dst.rawCpu(), kernel.getCpu(), anchor, + iterations.intValue(), borderType.value, borderValue); } )), new OperationMetaData(CVOperation.defaults("CV GaussianBlur", "Apply a Gaussian blur to an image."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", true), + SocketHints.createImageSocketHint("src"), new SocketHint.Builder<>(Size.class).identifier("ksize").initialValueSupplier(() -> new Size(1, 1)).build(), SocketHints.Inputs.createNumberSpinnerSocketHint("sigmaX", 0.0), - SocketHints.Inputs.createNumberSpinnerSocketHint("sigmaY", 0.0), SocketHints - .createEnumSocketHint("borderType", CVBorderTypesEnum.BORDER_DEFAULT), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.Inputs.createNumberSpinnerSocketHint("sigmaY", 0.0), + SocketHints.createEnumSocketHint("borderType", CVBorderTypesEnum.BORDER_DEFAULT), + SocketHints.createImageSocketHint("dst"), (src, ksize, sigmaX, sigmaY, borderType, dst) -> { - opencv_imgproc.GaussianBlur(src, dst, ksize, sigmaX.doubleValue(), sigmaY + GaussianBlur(src.getCpu(), dst.rawCpu(), ksize, sigmaX.doubleValue(), sigmaY .doubleValue(), borderType.value); } )), @@ -286,14 +364,14 @@ public class CVOperations { new OperationMetaData(CVOperation.defaults("CV Laplacian", "Find edges by calculating the Laplacian for the given image."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + SocketHints.createImageSocketHint("src"), SocketHints.Inputs.createNumberSpinnerSocketHint("ksize", 1), SocketHints.Inputs.createNumberSpinnerSocketHint("scale", 1.0), SocketHints.Inputs.createNumberSpinnerSocketHint("delta", 0.0), SocketHints.createEnumSocketHint("borderType", BorderTypesEnum.BORDER_DEFAULT), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, ksize, scale, delta, borderType, dst) -> { - opencv_imgproc.Laplacian(src, dst, 0, ksize.intValue(), scale.doubleValue(), + Laplacian(src.getCpu(), dst.rawCpu(), 0, ksize.intValue(), scale.doubleValue(), delta.doubleValue(), borderType.value); } )), @@ -301,18 +379,18 @@ public class CVOperations { new OperationMetaData(CVOperation.defaults("CV medianBlur", "Apply a Median blur to an image."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + SocketHints.createImageSocketHint("src"), SocketHints.Inputs.createNumberSpinnerSocketHint("ksize", 1, 1, Integer.MAX_VALUE), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, ksize, dst) -> { - opencv_imgproc.medianBlur(src, dst, ksize.intValue()); + medianBlur(src.getCpu(), dst.rawCpu(), ksize.intValue()); } )), new OperationMetaData(CVOperation.defaults("CV rectangle", "Draw a rectangle (outline or filled) on an image."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + SocketHints.createImageSocketHint("src"), SocketHints.Inputs.createPointSocketHint("pt1", 0, 0), SocketHints.Inputs.createPointSocketHint("pt2", 0, 0), new SocketHint.Builder<>(Scalar.class).identifier("color").initialValueSupplier( @@ -321,12 +399,12 @@ public class CVOperations { .MIN_VALUE, Integer.MAX_VALUE), SocketHints.createEnumSocketHint("lineType", LineTypesEnum.LINE_8), SocketHints.Inputs.createNumberSpinnerSocketHint("shift", 0), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, pt1, pt2, color, thickness, lineType, shift, dst) -> { // Rectangle only has one input and it modifies it so we have to copy the input // image to the dst src.copyTo(dst); - opencv_imgproc.rectangle(dst, pt1, pt2, color, thickness.intValue(), lineType + rectangle(dst.rawCpu(), pt1, pt2, color, thickness.intValue(), lineType .value, shift.intValue()); } )), @@ -334,49 +412,74 @@ public class CVOperations { new OperationMetaData(CVOperation.defaults("CV resize", "Resizes the image to the specified size."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + SocketHints.createImageSocketHint("src"), new SocketHint.Builder<>(Size.class).identifier("dsize").initialValueSupplier(() -> new Size(0, 0)).build(), SocketHints.Inputs.createNumberSpinnerSocketHint("fx", .25), SocketHints.Inputs .createNumberSpinnerSocketHint("fy", .25), SocketHints.createEnumSocketHint("interpolation", InterpolationFlagsEnum .INTER_LINEAR), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, dsize, fx, fy, interpolation, dst) -> { - opencv_imgproc.resize(src, dst, dsize, fx.doubleValue(), fy.doubleValue(), + resize(src.getCpu(), dst.rawCpu(), dsize, fx.doubleValue(), fy.doubleValue(), interpolation.value); } )), new OperationMetaData(CVOperation.defaults("CV Sobel", "Find edges by calculating the requested derivative order for the given image."), - templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + templateFactory.createCuda( + SocketHints.createImageSocketHint("src"), SocketHints.Inputs.createNumberSpinnerSocketHint("dx", 0), SocketHints.Inputs.createNumberSpinnerSocketHint("dy", 0), SocketHints.Inputs.createNumberSpinnerSocketHint("ksize", 3), SocketHints.Inputs.createNumberSpinnerSocketHint("scale", 1), SocketHints.Inputs.createNumberSpinnerSocketHint("delta", 0), SocketHints.createEnumSocketHint("borderType", BorderTypesEnum.BORDER_DEFAULT), - SocketHints.Outputs.createMatSocketHint("dst"), - (src, dx, dy, ksize, scale, delta, borderType, dst) -> { - opencv_imgproc.Sobel(src, dst, 0, dx.intValue(), dy.intValue(), - ksize.intValue(), scale.doubleValue(), delta.doubleValue(), borderType.value); + SocketHints.createImageSocketHint("dst"), + (src, dx, dy, ksize, scale, delta, borderType, useCuda, dst) -> { + if (useCuda) { + try (Filter sobelFilter = createSobelFilter( + src.type(), + src.type(), + dx.intValue(), + dy.intValue(), + ksize.intValue(), + scale.doubleValue(), + borderType.value, + borderType.value)) { + sobelFilter.apply(src.getGpu(), dst.rawGpu()); + } + } else { + Sobel(src.getCpu(), dst.rawCpu(), 0, dx.intValue(), dy.intValue(), + ksize.intValue(), scale.doubleValue(), delta.doubleValue(), + borderType.value); + } } )), new OperationMetaData(CVOperation.defaults("CV Threshold", "Apply a fixed-level threshold to each array element in an image.", "CV threshold"), - templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + templateFactory.createCuda( + SocketHints.createImageSocketHint("src"), SocketHints.Inputs.createNumberSpinnerSocketHint("thresh", 0), SocketHints.Inputs.createNumberSpinnerSocketHint("maxval", 0), SocketHints.createEnumSocketHint("type", CVThresholdTypesEnum.THRESH_BINARY), - SocketHints.Outputs.createMatSocketHint("dst"), - (src, thresh, maxval, type, dst) -> { - opencv_imgproc.threshold(src, dst, thresh.doubleValue(), maxval.doubleValue(), - type.value); + SocketHints.createImageSocketHint("dst"), + (src, thresh, maxval, type, useCuda, dst) -> { + if (useCuda) { + opencv_cudaarithm.threshold( + src.getGpu(), + dst.rawGpu(), + thresh.doubleValue(), + maxval.doubleValue(), + type.value + ); + } else { + threshold(src.getCpu(), dst.rawCpu(), thresh.doubleValue(), + maxval.doubleValue(), type.value); + } } )) ); diff --git a/core/src/main/java/edu/wpi/grip/core/operations/CudaOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/CudaOperation.java new file mode 100644 index 0000000000..a6e572a3d6 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/operations/CudaOperation.java @@ -0,0 +1,85 @@ +package edu.wpi.grip.core.operations; + +import edu.wpi.grip.core.MatWrapper; +import edu.wpi.grip.core.Operation; +import edu.wpi.grip.core.sockets.CudaSocket; +import edu.wpi.grip.core.sockets.InputSocket; +import edu.wpi.grip.core.sockets.OutputSocket; +import edu.wpi.grip.core.sockets.SocketHint; +import edu.wpi.grip.core.sockets.SocketHints; + +import org.bytedeco.javacpp.opencv_core.GpuMat; +import org.bytedeco.javacpp.opencv_core.Mat; + +/** + * A partial implementation of Operation that has the option to use CUDA acceleration. + */ +public abstract class CudaOperation implements Operation { + + protected final SocketHint inputHint = + SocketHints.createImageSocketHint("Input"); + protected final SocketHint gpuHint = + SocketHints.createBooleanSocketHint("Prefer GPU", false); + protected final SocketHint outputHint = SocketHints.createImageSocketHint("Output"); + + /** + * Default image input socket. + */ + protected final InputSocket inputSocket; + /** + * Input socket telling the operation to prefer to use CUDA acceleration when possible. + */ + protected final CudaSocket gpuSocket; + /** + * Default image output socket. + */ + protected final OutputSocket outputSocket; + + /** + * The mat used for an input to the CPU operation. + */ + protected final Mat cpuIn = new Mat(); + + /** + * The mat used for an input to the CUDA operation. + */ + protected final GpuMat gpuIn = new GpuMat(); + + /** + * The output mat of a CPU operation. + */ + protected final Mat cpuOut = new Mat(); + + /** + * The output mat of a CUDA operation. + */ + protected final GpuMat gpuOut = new GpuMat(); + + protected CudaOperation(InputSocket.Factory isf, OutputSocket.Factory osf) { + inputSocket = isf.create(inputHint); + gpuSocket = isf.createCuda(gpuHint); + outputSocket = osf.create(outputHint); + + inputSocket.setValue(MatWrapper.using(cpuIn, gpuIn)); + outputSocket.setValue(MatWrapper.using(cpuOut, gpuOut)); + } + + @Override + public void cleanUp() { + cpuIn.deallocate(); + gpuIn.deallocate(); + cpuOut.deallocate(); + gpuOut.deallocate(); + } + + /** + * Checks the {@link #gpuSocket} to see if this operation should prefer to use the CUDA codepath. + * + * @return true if this operation should prefer to use CUDA, false if it should only use the CPU + */ + protected boolean preferCuda() { + return gpuSocket.isCudaAvailable() + && gpuSocket.getValue().orElse(false); + } + +} diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/BlobsReport.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/BlobsReport.java index 4671533e8c..881c3b16d9 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/BlobsReport.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/BlobsReport.java @@ -1,6 +1,7 @@ package edu.wpi.grip.core.operations.composite; import edu.wpi.grip.annotation.operation.PublishableObject; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.operations.network.PublishValue; import edu.wpi.grip.core.operations.network.Publishable; import edu.wpi.grip.core.sockets.NoSocketTypeLabel; @@ -10,25 +11,23 @@ import java.util.Collections; import java.util.List; -import static org.bytedeco.javacpp.opencv_core.Mat; - /** * This class is used as the output of operations that detect blobs in an image. */ @PublishableObject @NoSocketTypeLabel public class BlobsReport implements Publishable { - private final Mat input; + private final MatWrapper input; private final List blobs; /** * Create an empty blob report. This is used as the default value for sockets */ public BlobsReport() { - this(new Mat(), Collections.emptyList()); + this(MatWrapper.emptyWrapper(), Collections.emptyList()); } - public BlobsReport(Mat input, List blobs) { + public BlobsReport(MatWrapper input, List blobs) { this.input = input; this.blobs = blobs; } @@ -40,7 +39,7 @@ public List getBlobs() { /** * @return The original image that the blob detection was performed on. */ - public Mat getInput() { + public MatWrapper getInput() { return this.input; } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/BlurOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/BlurOperation.java index 15bc184d7a..326bf7b9c6 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/BlurOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/BlurOperation.java @@ -2,7 +2,9 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; +import edu.wpi.grip.core.operations.CudaOperation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.core.sockets.SocketHint; @@ -11,10 +13,18 @@ import com.google.common.collect.ImmutableList; import com.google.inject.Inject; +import org.bytedeco.javacpp.opencv_core.GpuMat; +import org.bytedeco.javacpp.opencv_cudaimgproc; + import java.util.List; -import static org.bytedeco.javacpp.opencv_core.Mat; +import static org.bytedeco.javacpp.opencv_core.CV_8UC3; +import static org.bytedeco.javacpp.opencv_core.CV_8UC4; import static org.bytedeco.javacpp.opencv_core.Size; +import static org.bytedeco.javacpp.opencv_cudafilters.Filter; +import static org.bytedeco.javacpp.opencv_cudafilters.createGaussianFilter; +import static org.bytedeco.javacpp.opencv_imgproc.CV_BGR2BGRA; +import static org.bytedeco.javacpp.opencv_imgproc.CV_BGRA2BGR; import static org.bytedeco.javacpp.opencv_imgproc.GaussianBlur; import static org.bytedeco.javacpp.opencv_imgproc.bilateralFilter; import static org.bytedeco.javacpp.opencv_imgproc.blur; @@ -27,27 +37,27 @@ summary = "Blurs an image to remove noise", category = OperationCategory.IMAGE_PROCESSING, iconName = "blur") -public class BlurOperation implements Operation { +public class BlurOperation extends CudaOperation { - private final SocketHint inputHint = SocketHints.Inputs.createMatSocketHint("Input", false); private final SocketHint typeHint = SocketHints.createEnumSocketHint("Type", Type.BOX); private final SocketHint radiusHint = SocketHints.Inputs .createNumberSliderSocketHint("Radius", 0.0, 0.0, 100.0); - private final SocketHint outputHint = SocketHints.Inputs.createMatSocketHint("Output", true); - private final InputSocket inputSocket; private final InputSocket typeSocket; private final InputSocket radiusSocket; - private final OutputSocket outputSocket; + + private int lastKernelSize = 0; + // used to covert 3-channel images to 4-channel for CUDA + private final GpuMat upcast = new GpuMat(); + private Filter gpuGaussianFilter; + //private Filter gpuMedianFilter; @Inject @SuppressWarnings("JavadocMethod") - public BlurOperation(InputSocket.Factory inputSocketFactory, - OutputSocket.Factory outputSocketFactory) { - this.inputSocket = inputSocketFactory.create(inputHint); + public BlurOperation(InputSocket.Factory inputSocketFactory, OutputSocket.Factory + outputSocketFactory) { + super(inputSocketFactory, outputSocketFactory); this.typeSocket = inputSocketFactory.create(typeHint); this.radiusSocket = inputSocketFactory.create(radiusHint); - - this.outputSocket = outputSocketFactory.create(outputHint); } @Override @@ -55,7 +65,8 @@ public List getInputSockets() { return ImmutableList.of( inputSocket, typeSocket, - radiusSocket + radiusSocket, + gpuSocket ); } @@ -68,47 +79,115 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = inputSocket.getValue().get(); + final MatWrapper input = inputSocket.getValue().get(); + if (input.empty()) { + return; + } final Type type = typeSocket.getValue().get(); final Number radius = radiusSocket.getValue().get(); - final Mat output = outputSocket.getValue().get(); + final MatWrapper output = outputSocket.getValue().get(); + int imageType; int kernelSize; + boolean kernelChange; + + if (preferCuda()) { + if (input.type() == CV_8UC3) { + // GPU filters generally don't take BGR images, but will take BGRA + // So we convert the BGR image to BGRA here and convert it back at the end + // Note that this doesn't care about the actual pixel format because we're + // converting to the same format, just with an extra channel + imageType = CV_8UC4; + opencv_cudaimgproc.cvtColor(input.getGpu(), upcast, CV_BGR2BGRA); + gpuIn.put(upcast); + } else { + input.copyTo(gpuIn); + imageType = input.type(); + } + } else { + imageType = input.type(); + } switch (type) { case BOX: // Box filter kernels must have an odd size kernelSize = 2 * radius.intValue() + 1; - blur(input, output, new Size(kernelSize, kernelSize)); + + // Don't bother with CUDA acceleration here; CPU is fast enough that memory copies + // will remove the speedups from CUDA + blur(input.getCpu(), output.getCpu(), new Size(kernelSize, kernelSize)); break; case GAUSSIAN: // A Gaussian blur radius is a standard deviation, so a kernel that extends three radii - // in either direction - // from the center should account for 99.7% of the theoretical influence on each pixel. + // in either direction from the center should account for 99.7% of the theoretical + // influence on each pixel. kernelSize = 6 * radius.intValue() + 1; - GaussianBlur(input, output, new Size(kernelSize, kernelSize), radius.doubleValue()); + kernelChange = kernelSize != lastKernelSize; + lastKernelSize = kernelSize; + if (preferCuda()/* && kernelSize < 32*/) { + // GPU gaussian blurs require kernel size in 0..31 + if (kernelChange || gpuGaussianFilter == null) { + gpuGaussianFilter = createGaussianFilter(imageType, imageType, + new Size(kernelSize, kernelSize), radius.doubleValue()); + } + gpuGaussianFilter.apply(gpuIn, gpuOut); + output.set(gpuOut); + } else { + GaussianBlur(input.getCpu(), output.getCpu(), new Size(kernelSize, kernelSize), + radius.doubleValue()); + } break; case MEDIAN: kernelSize = 2 * radius.intValue() + 1; - medianBlur(input, output, kernelSize); + // FIXME: CUDA median filters is broken - run on CPU only for now + /* + kernelChange = kernelSize != lastKernelSize; + lastKernelSize = kernelSize; + if (preferCuda() && imageType == CV_8UC1) { + // GPU median filters only work on grayscale images + if (kernelChange || gpuMedianFilter == null) { + gpuMedianFilter = createMedianFilter(imageType, kernelSize); + } + gpuMedianFilter.apply(gpuIn, gpuOut); + output.set(gpuOut); + } else { + medianBlur(input.getCpu(), output.rawCpu(), kernelSize); + } + */ + medianBlur(input.getCpu(), output.rawCpu(), kernelSize); break; case BILATERAL_FILTER: - bilateralFilter(input, output, -1, radius.doubleValue(), radius.doubleValue()); + if (preferCuda()) { + opencv_cudaimgproc.bilateralFilter(gpuIn, gpuOut, + -1, radius.floatValue(), radius.floatValue() / 6); + output.set(gpuOut); + } else { + bilateralFilter(input.getCpu(), output.rawCpu(), + -1, radius.doubleValue(), radius.doubleValue() / 6); + } break; default: throw new IllegalArgumentException("Illegal blur type: " + type); } + if (preferCuda() && output.type() == CV_8UC4 && input.type() == CV_8UC3) { + // Remove the alpha channel that was added for GPU filtering + opencv_cudaimgproc.cvtColor(output.getGpu(), output.rawGpu(), CV_BGRA2BGR); + } + + //output.set(output); outputSocket.setValue(output); } private enum Type { - BOX("Box Blur"), GAUSSIAN("Gaussian Blur"), MEDIAN("Median Filter"), + BOX("Box Blur"), + GAUSSIAN("Gaussian Blur"), + MEDIAN("Median Filter"), BILATERAL_FILTER("Bilateral Filter"); private final String label; diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/CannyEdgeOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/CannyEdgeOperation.java new file mode 100644 index 0000000000..52c52fe124 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/CannyEdgeOperation.java @@ -0,0 +1,93 @@ +package edu.wpi.grip.core.operations.composite; + +import edu.wpi.grip.annotation.operation.Description; +import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.operations.CudaOperation; +import edu.wpi.grip.core.sockets.InputSocket; +import edu.wpi.grip.core.sockets.OutputSocket; +import edu.wpi.grip.core.sockets.SocketHint; +import edu.wpi.grip.core.sockets.SocketHints; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Inject; + +import org.bytedeco.javacpp.opencv_cudaimgproc.CannyEdgeDetector; + +import java.util.List; + +import static org.bytedeco.javacpp.opencv_cudaimgproc.createCannyEdgeDetector; +import static org.bytedeco.javacpp.opencv_imgproc.Canny; + +/** + * An operation that performs canny edge detection on an image. + */ +@Description(name = "CV Canny", + summary = "Performs canny edge detection on a grayscale image", + category = OperationCategory.OPENCV, + iconName = "opencv") +public class CannyEdgeOperation extends CudaOperation { + + private final SocketHint lowThreshHint + = SocketHints.Inputs.createNumberSpinnerSocketHint("Low threshold", 0); + private final SocketHint highThreshHint + = SocketHints.Inputs.createNumberSpinnerSocketHint("High threshold", 0); + private final SocketHint apertureSizeHint + = SocketHints.Inputs.createNumberSpinnerSocketHint("Aperture size", 0); + private final SocketHint l2gradientHint + = SocketHints.Inputs.createCheckboxSocketHint("L2gradient", false); + + private final InputSocket lowThreshSocket; + private final InputSocket highThreshSocket; + private final InputSocket apertureSizeSocket; + private final InputSocket l2gradientSocket; + + @Inject + protected CannyEdgeOperation(InputSocket.Factory isf, OutputSocket.Factory osf) { + super(isf, osf); + lowThreshSocket = isf.create(lowThreshHint); + highThreshSocket = isf.create(highThreshHint); + apertureSizeSocket = isf.create(apertureSizeHint); + l2gradientSocket = isf.create(l2gradientHint); + } + + @Override + public List getInputSockets() { + return ImmutableList.of( + inputSocket, + lowThreshSocket, + highThreshSocket, + apertureSizeSocket, + l2gradientSocket, + gpuSocket + ); + } + + @Override + public List getOutputSockets() { + return ImmutableList.of( + outputSocket + ); + } + + @Override + public void perform() { + double lowThresh = lowThreshSocket.getValue().get().doubleValue(); + double highThresh = highThreshSocket.getValue().get().doubleValue(); + int apertureSize = apertureSizeSocket.getValue().get().intValue(); + boolean l2gradient = l2gradientSocket.getValue().get(); + if (preferCuda()) { + try (CannyEdgeDetector cannyEdgeDetector = createCannyEdgeDetector( + lowThresh, + highThresh, + apertureSize, + l2gradient)) { + cannyEdgeDetector.detect(inputSocket.getValue().get().getGpu(), + outputSocket.getValue().get().rawGpu()); + } + } else { + Canny(inputSocket.getValue().get().getCpu(), outputSocket.getValue().get().getCpu(), + lowThresh, highThresh, apertureSize, l2gradient); + } + outputSocket.flagChanged(); + } +} diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/CascadeClassifierOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/CascadeClassifierOperation.java index e24bf84308..0bca69bd7d 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/CascadeClassifierOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/CascadeClassifierOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -29,8 +30,8 @@ iconName = "opencv") public class CascadeClassifierOperation implements Operation { - private final SocketHint imageHint = - SocketHints.Inputs.createMatSocketHint("Image", false); + private final SocketHint imageHint = + SocketHints.createImageSocketHint("Image"); private final SocketHint classifierHint = new SocketHint.Builder<>(CascadeClassifier.class) .identifier("Classifier") @@ -49,7 +50,7 @@ public class CascadeClassifierOperation implements Operation { .initialValue(RectsReport.NIL) .build(); - private final InputSocket imageSocket; + private final InputSocket imageSocket; private final InputSocket classifierSocket; private final InputSocket scaleSocket; private final InputSocket minNeighborsSocket; @@ -93,7 +94,8 @@ public void perform() { if (!imageSocket.getValue().isPresent() || !classifierSocket.getValue().isPresent()) { return; } - final Mat image = imageSocket.getValue().get(); + final MatWrapper input = imageSocket.getValue().get(); + final Mat image = input.getCpu(); if (image.empty() || image.channels() != 3) { throw new IllegalArgumentException("A cascade classifier needs a three-channel input"); } @@ -108,7 +110,7 @@ public void perform() { for (int i = 0; i < detections.size(); i++) { rects.add(detections.get(i)); } - output.setValue(new RectsReport(image, rects)); + output.setValue(new RectsReport(input, rects)); } } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/DesaturateOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/DesaturateOperation.java index 59ee2e7c17..00ea135388 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/DesaturateOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/DesaturateOperation.java @@ -2,18 +2,19 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; +import edu.wpi.grip.core.operations.CudaOperation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; -import edu.wpi.grip.core.sockets.SocketHint; -import edu.wpi.grip.core.sockets.SocketHints; import com.google.common.collect.ImmutableList; import com.google.inject.Inject; +import org.bytedeco.javacpp.opencv_cudaimgproc; + import java.util.List; -import static org.bytedeco.javacpp.opencv_core.Mat; import static org.bytedeco.javacpp.opencv_imgproc.COLOR_BGR2GRAY; import static org.bytedeco.javacpp.opencv_imgproc.COLOR_BGRA2GRAY; import static org.bytedeco.javacpp.opencv_imgproc.cvtColor; @@ -25,27 +26,21 @@ summary = "Convert a color image into shades of gray", category = OperationCategory.IMAGE_PROCESSING, iconName = "desaturate") -public class DesaturateOperation implements Operation { - - private final SocketHint inputHint = SocketHints.Inputs.createMatSocketHint("Input", false); - private final SocketHint outputHint = SocketHints.Outputs.createMatSocketHint("Output"); - - private final InputSocket inputSocket; - private final OutputSocket outputSocket; +public class DesaturateOperation extends CudaOperation { @Inject @SuppressWarnings("JavadocMethod") public DesaturateOperation(InputSocket.Factory inputSocketFactory, OutputSocket.Factory outputSocketFactory) { - this.inputSocket = inputSocketFactory.create(inputHint); - this.outputSocket = outputSocketFactory.create(outputHint); + super(inputSocketFactory, outputSocketFactory); } @Override public List getInputSockets() { return ImmutableList.of( - inputSocket + inputSocket, + gpuSocket ); } @@ -58,23 +53,30 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = inputSocket.getValue().get(); - - Mat output = outputSocket.getValue().get(); + final MatWrapper input = inputSocket.getValue().get(); + final MatWrapper output = outputSocket.getValue().get(); switch (input.channels()) { case 1: // If the input is already one channel, it's already desaturated - input.copyTo(output); + output.set(input); break; case 3: - cvtColor(input, output, COLOR_BGR2GRAY); + if (preferCuda()) { + opencv_cudaimgproc.cvtColor(input.getGpu(), output.rawGpu(), COLOR_BGR2GRAY); + } else { + cvtColor(input.getCpu(), output.rawCpu(), COLOR_BGR2GRAY); + } break; case 4: - cvtColor(input, output, COLOR_BGRA2GRAY); + if (preferCuda()) { + opencv_cudaimgproc.cvtColor(input.getGpu(), output.rawGpu(), COLOR_BGRA2GRAY); + } else { + cvtColor(input.getCpu(), output.rawCpu(), COLOR_BGRA2GRAY); + } break; default: diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/DistanceTransformOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/DistanceTransformOperation.java index 17c52ab339..0837d54680 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/DistanceTransformOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/DistanceTransformOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -31,15 +32,15 @@ iconName = "opencv") public class DistanceTransformOperation implements Operation { - private final SocketHint srcHint = SocketHints.Inputs.createMatSocketHint("Input", false); + private final SocketHint srcHint = SocketHints.createImageSocketHint("Input"); private final SocketHint typeHint = SocketHints.createEnumSocketHint("Type", Type.DIST_L2); private final SocketHint maskSizeHint = SocketHints.createEnumSocketHint("Mask size", MaskSize.ZERO); - private final SocketHint outputHint = SocketHints.Inputs.createMatSocketHint("Output", true); - private final InputSocket srcSocket; + private final SocketHint outputHint = SocketHints.createImageSocketHint("Output"); + private final InputSocket srcSocket; private final InputSocket typeSocket; private final InputSocket maskSizeSocket; - private final OutputSocket outputSocket; + private final OutputSocket outputSocket; @Inject @SuppressWarnings("JavadocMethod") @@ -70,7 +71,7 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = srcSocket.getValue().get(); + final Mat input = srcSocket.getValue().get().getCpu(); if (input.type() != CV_8U) { throw new IllegalArgumentException("Distance transform only works on 8-bit binary images"); @@ -79,12 +80,12 @@ public void perform() { final Type type = typeSocket.getValue().get(); final MaskSize maskSize = maskSizeSocket.getValue().get(); - final Mat output = outputSocket.getValue().get(); + final Mat output = outputSocket.getValue().get().rawCpu(); distanceTransform(input, output, type.value, maskSize.value); output.convertTo(output, CV_8U); - outputSocket.setValue(output); + outputSocket.flagChanged(); } private enum Type { diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/FindBlobsOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/FindBlobsOperation.java index b6efd38fa8..c661a3e37c 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/FindBlobsOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/FindBlobsOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -28,7 +29,7 @@ iconName = "find-blobs") public class FindBlobsOperation implements Operation { - private final SocketHint inputHint = SocketHints.Inputs.createMatSocketHint("Input", false); + private final SocketHint inputHint = SocketHints.createImageSocketHint("Input"); private final SocketHint minAreaHint = SocketHints.Inputs .createNumberSpinnerSocketHint("Min Area", 1); private final SocketHint> circularityHint = SocketHints.Inputs @@ -41,7 +42,7 @@ public class FindBlobsOperation implements Operation { .initialValueSupplier(BlobsReport::new) .build(); - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final InputSocket minAreaSocket; private final InputSocket> circularitySocket; private final InputSocket colorSocket; @@ -80,7 +81,7 @@ public List getOutputSockets() { @Override @SuppressWarnings("unchecked") public void perform() { - final Mat input = inputSocket.getValue().get(); + final Mat input = inputSocket.getValue().get().getCpu(); final Number minArea = minAreaSocket.getValue().get(); final List circularity = circularitySocket.getValue().get(); final Boolean darkBlobs = colorSocket.getValue().get(); @@ -109,6 +110,6 @@ public void perform() { blobs.add(new BlobsReport.Blob(keyPoint.pt().x(), keyPoint.pt().y(), keyPoint.size())); } - outputSocket.setValue(new BlobsReport(input, blobs)); + outputSocket.setValue(new BlobsReport(inputSocket.getValue().get(), blobs)); } } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/FindContoursOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/FindContoursOperation.java index 45b8506bdf..18285c16f1 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/FindContoursOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/FindContoursOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -30,8 +31,7 @@ iconName = "find-contours") public class FindContoursOperation implements Operation { - private final SocketHint inputHint = - new SocketHint.Builder<>(Mat.class).identifier("Input").build(); + private final SocketHint inputHint = SocketHints.createImageSocketHint("Input"); private final SocketHint externalHint = SocketHints.createBooleanSocketHint("External Only", false); @@ -41,7 +41,7 @@ public class FindContoursOperation implements Operation { .identifier("Contours").initialValueSupplier(ContoursReport::new).build(); - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final InputSocket externalSocket; private final OutputSocket contoursSocket; @@ -73,7 +73,7 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = inputSocket.getValue().get(); + final MatWrapper input = inputSocket.getValue().get(); if (input.empty()) { return; } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/FindLinesOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/FindLinesOperation.java index 32dc98932f..4f9597ce63 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/FindLinesOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/FindLinesOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -30,12 +31,12 @@ iconName = "find-lines") public class FindLinesOperation implements Operation { - private final SocketHint inputHint = SocketHints.Inputs.createMatSocketHint("Input", false); + private final SocketHint inputHint = SocketHints.createImageSocketHint("Input"); private final SocketHint linesHint = new SocketHint.Builder<>(LinesReport.class) .identifier("Lines").initialValueSupplier(LinesReport::new).build(); - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final OutputSocket linesReportSocket; @@ -63,17 +64,17 @@ public List getOutputSockets() { @Override @SuppressWarnings("unchecked") public void perform() { - final Mat input = inputSocket.getValue().get(); + final MatWrapper input = inputSocket.getValue().get(); final LineSegmentDetector lsd = linesReportSocket.getValue().get().getLineSegmentDetector(); final Mat lines = new Mat(); if (input.channels() == 1) { - lsd.detect(input, lines); + lsd.detect(input.getCpu(), lines); } else { // The line detector works on a single channel. If the input is a color image, we can just // give the line detector a grayscale version of it final Mat tmp = new Mat(); - cvtColor(input, tmp, COLOR_BGR2GRAY); + cvtColor(input.getCpu(), tmp, COLOR_BGR2GRAY); lsd.detect(tmp, lines); tmp.release(); } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/HSLThresholdOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/HSLThresholdOperation.java index 8bf6135d62..b7958754cb 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/HSLThresholdOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/HSLThresholdOperation.java @@ -3,6 +3,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -33,7 +34,7 @@ public class HSLThresholdOperation extends ThresholdOperation { private static final Logger logger = Logger.getLogger(HSLThresholdOperation.class.getName()); - private final SocketHint inputHint = SocketHints.Inputs.createMatSocketHint("Input", false); + private final SocketHint inputHint = SocketHints.createImageSocketHint("Input"); private final SocketHint> hueHint = SocketHints.Inputs .createNumberListRangeSocketHint("Hue", 0.0, 180.0); private final SocketHint> saturationHint = SocketHints.Inputs @@ -41,14 +42,14 @@ public class HSLThresholdOperation extends ThresholdOperation { private final SocketHint> luminanceHint = SocketHints.Inputs .createNumberListRangeSocketHint("Luminance", 0.0, 255.0); - private final SocketHint outputHint = SocketHints.Outputs.createMatSocketHint("Output"); + private final SocketHint outputHint = SocketHints.createImageSocketHint("Output"); - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final InputSocket> hueSocket; private final InputSocket> saturationSocket; private final InputSocket> luminanceSocket; - private final OutputSocket outputSocket; + private final OutputSocket outputSocket; @Inject @SuppressWarnings("JavadocMethod") @@ -82,13 +83,13 @@ public List getOutputSockets() { @Override @SuppressWarnings("unchecked") public void perform() { - final Mat input = inputSocket.getValue().get(); + final Mat input = inputSocket.getValue().get().getCpu(); if (input.channels() != 3) { throw new IllegalArgumentException("HSL Threshold needs a 3-channel input"); } - final Mat output = outputSocket.getValue().get(); + final Mat output = outputSocket.getValue().get().rawCpu(); final List channel1 = hueSocket.getValue().get(); final List channel2 = saturationSocket.getValue().get(); final List channel3 = luminanceSocket.getValue().get(); @@ -111,7 +112,7 @@ public void perform() { try { cvtColor(input, hls, COLOR_BGR2HLS); inRange(hls, low, high, output); - outputSocket.setValue(output); + outputSocket.flagChanged(); } catch (RuntimeException e) { logger.log(Level.WARNING, e.getMessage(), e); } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/HSVThresholdOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/HSVThresholdOperation.java index 839acbd632..5b0ef05d11 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/HSVThresholdOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/HSVThresholdOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -33,7 +34,7 @@ public class HSVThresholdOperation extends ThresholdOperation { private static final Logger logger = Logger.getLogger(HSVThresholdOperation.class.getName()); - private final SocketHint inputHint = SocketHints.Inputs.createMatSocketHint("Input", false); + private final SocketHint inputHint = SocketHints.createImageSocketHint("Input"); private final SocketHint> hueHint = SocketHints.Inputs .createNumberListRangeSocketHint("Hue", 0.0, 180.0); private final SocketHint> saturationHint = SocketHints.Inputs @@ -41,14 +42,14 @@ public class HSVThresholdOperation extends ThresholdOperation { private final SocketHint> valueHint = SocketHints.Inputs .createNumberListRangeSocketHint("Value", 0.0, 255.0); - private final SocketHint outputHint = SocketHints.Outputs.createMatSocketHint("Output"); + private final SocketHint outputHint = SocketHints.createImageSocketHint("Output"); - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final InputSocket> hueSocket; private final InputSocket> saturationSocket; private final InputSocket> valueSocket; - private final OutputSocket outputSocket; + private final OutputSocket outputSocket; @Inject @SuppressWarnings("JavadocMethod") @@ -82,13 +83,13 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = inputSocket.getValue().get(); + final Mat input = inputSocket.getValue().get().getCpu(); if (input.channels() != 3) { throw new IllegalArgumentException("HSV Threshold needs a 3-channel input"); } - final Mat output = outputSocket.getValue().get(); + final Mat output = outputSocket.getValue().get().rawCpu(); final List channel1 = hueSocket.getValue().get(); final List channel2 = saturationSocket.getValue().get(); final List channel3 = valueSocket.getValue().get(); @@ -109,7 +110,7 @@ public void perform() { try { cvtColor(input, hsv, COLOR_BGR2HSV); inRange(hsv, low, high, output); - outputSocket.setValue(output); + outputSocket.flagChanged(); } catch (RuntimeException e) { logger.log(Level.WARNING, e.getMessage(), e); } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/LinesReport.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/LinesReport.java index f878a26d8d..1bc34e855f 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/LinesReport.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/LinesReport.java @@ -1,6 +1,7 @@ package edu.wpi.grip.core.operations.composite; import edu.wpi.grip.annotation.operation.PublishableObject; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.operations.network.PublishValue; import edu.wpi.grip.core.operations.network.Publishable; import edu.wpi.grip.core.sockets.NoSocketTypeLabel; @@ -9,7 +10,6 @@ import java.util.Collections; import java.util.List; -import static org.bytedeco.javacpp.opencv_core.Mat; import static org.bytedeco.javacpp.opencv_imgproc.LineSegmentDetector; import static org.bytedeco.javacpp.opencv_imgproc.createLineSegmentDetector; @@ -24,7 +24,7 @@ @NoSocketTypeLabel public class LinesReport implements Publishable { private final LineSegmentDetector lsd; - private final Mat input; + private final MatWrapper input; private final List lines; /** @@ -32,7 +32,7 @@ public class LinesReport implements Publishable { * LinesReports. */ public LinesReport() { - this(createLineSegmentDetector(), new Mat(), Collections.emptyList()); + this(createLineSegmentDetector(), MatWrapper.emptyWrapper(), Collections.emptyList()); } /** @@ -40,7 +40,7 @@ public LinesReport() { * @param input The input matrix. * @param lines The lines that have been found. */ - public LinesReport(LineSegmentDetector lsd, Mat input, List lines) { + public LinesReport(LineSegmentDetector lsd, MatWrapper input, List lines) { this.lsd = lsd; this.input = input; this.lines = lines; @@ -53,7 +53,7 @@ protected LineSegmentDetector getLineSegmentDetector() { /** * @return The original image that the line detection was performed on. */ - public Mat getInput() { + public MatWrapper getInput() { return this.input; } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/MaskOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/MaskOperation.java index 569ec1f953..4ce0599b5a 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/MaskOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/MaskOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -25,16 +26,16 @@ iconName = "mask") public class MaskOperation implements Operation { - private final SocketHint inputHint = SocketHints.Inputs.createMatSocketHint("Input", false); - private final SocketHint maskHint = SocketHints.Inputs.createMatSocketHint("Mask", false); + private final SocketHint inputHint = SocketHints.createImageSocketHint("Input"); + private final SocketHint maskHint = SocketHints.createImageSocketHint("Mask"); - private final SocketHint outputHint = SocketHints.Outputs.createMatSocketHint("Output"); + private final SocketHint outputHint = SocketHints.createImageSocketHint("Output"); - private final InputSocket inputSocket; - private final InputSocket maskSocket; + private final InputSocket inputSocket; + private final InputSocket maskSocket; - private final OutputSocket outputSocket; + private final OutputSocket outputSocket; @Inject @SuppressWarnings("JavadocMethod") @@ -63,14 +64,14 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = inputSocket.getValue().get(); - final Mat mask = maskSocket.getValue().get(); + final Mat input = inputSocket.getValue().get().getCpu(); + final Mat mask = maskSocket.getValue().get().getCpu(); - final Mat output = outputSocket.getValue().get(); + final Mat output = outputSocket.getValue().get().rawCpu(); // Clear the output to black, then copy the input to it with the mask bitwise_xor(output, output, output); input.copyTo(output, mask); - outputSocket.setValue(output); + outputSocket.flagChanged(); } } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/NormalizeOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/NormalizeOperation.java index 52b331932a..6d570e4d9c 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/NormalizeOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/NormalizeOperation.java @@ -2,7 +2,9 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; +import edu.wpi.grip.core.operations.CudaOperation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.core.sockets.SocketHint; @@ -11,14 +13,15 @@ import com.google.common.collect.ImmutableList; import com.google.inject.Inject; +import org.bytedeco.javacpp.opencv_core; +import org.bytedeco.javacpp.opencv_cudaarithm; + import java.util.List; -import static org.bytedeco.javacpp.opencv_core.Mat; import static org.bytedeco.javacpp.opencv_core.NORM_INF; import static org.bytedeco.javacpp.opencv_core.NORM_L1; import static org.bytedeco.javacpp.opencv_core.NORM_L2; import static org.bytedeco.javacpp.opencv_core.NORM_MINMAX; -import static org.bytedeco.javacpp.opencv_core.normalize; /** * GRIP {@link Operation} for {@link org.bytedeco.javacpp.opencv_core#normalize}. @@ -27,40 +30,35 @@ summary = "Normalizes or remaps the values of pixels in an image", category = OperationCategory.IMAGE_PROCESSING, iconName = "opencv") -public class NormalizeOperation implements Operation { +public class NormalizeOperation extends CudaOperation { - private final SocketHint srcHint = SocketHints.Inputs.createMatSocketHint("Input", false); private final SocketHint typeHint = SocketHints.createEnumSocketHint("Type", Type.MINMAX); private final SocketHint aHint = SocketHints.Inputs .createNumberSpinnerSocketHint("Alpha", 0.0, 0, Double.MAX_VALUE); private final SocketHint bHint = SocketHints.Inputs .createNumberSpinnerSocketHint("Beta", 255, 0, Double.MAX_VALUE); - private final SocketHint dstHint = SocketHints.Inputs.createMatSocketHint("Output", true); - private final InputSocket srcSocket; private final InputSocket typeSocket; private final InputSocket alphaSocket; private final InputSocket betaSocket; - private final OutputSocket outputSocket; @Inject @SuppressWarnings("JavadocMethod") public NormalizeOperation(InputSocket.Factory inputSocketFactory, OutputSocket.Factory outputSocketFactory) { - this.srcSocket = inputSocketFactory.create(srcHint); + super(inputSocketFactory, outputSocketFactory); this.typeSocket = inputSocketFactory.create(typeHint); this.alphaSocket = inputSocketFactory.create(aHint); this.betaSocket = inputSocketFactory.create(bHint); - - this.outputSocket = outputSocketFactory.create(dstHint); } @Override public List getInputSockets() { return ImmutableList.of( - srcSocket, + inputSocket, typeSocket, alphaSocket, - betaSocket + betaSocket, + gpuSocket ); } @@ -73,16 +71,21 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = srcSocket.getValue().get(); - final Type type = typeSocket.getValue().get(); - final Number a = alphaSocket.getValue().get(); - final Number b = betaSocket.getValue().get(); - - final Mat output = outputSocket.getValue().get(); - - normalize(input, output, a.doubleValue(), b.doubleValue(), type.value, -1, null); + final MatWrapper input = inputSocket.getValue().get(); + final int type = typeSocket.getValue().get().value; + final double a = alphaSocket.getValue().get().doubleValue(); + final double b = betaSocket.getValue().get().doubleValue(); + + final MatWrapper output = outputSocket.getValue().get(); + + if (preferCuda() && input.channels() == 1) { + // CUDA normalize only works on single-channel images + opencv_cudaarithm.normalize(input.getGpu(), output.rawGpu(), a, b, type, -1); + } else { + opencv_core.normalize(input.getCpu(), output.rawCpu(), a, b, type, -1, null); + } - outputSocket.setValue(output); + outputSocket.flagChanged(); } private enum Type { diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java index 0cf13c9282..2131766fe9 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -24,7 +25,6 @@ import java.util.logging.Level; import java.util.logging.Logger; -import static org.bytedeco.javacpp.opencv_core.Mat; import static org.bytedeco.javacpp.opencv_imgcodecs.CV_IMWRITE_JPEG_QUALITY; import static org.bytedeco.javacpp.opencv_imgcodecs.imencode; @@ -51,7 +51,7 @@ public class PublishVideoOperation implements Operation { private final Object imageLock = new Object(); private final BytePointer imagePointer = new BytePointer(); private final Thread serverThread; - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final InputSocket qualitySocket; @SuppressWarnings("PMD.SingularField") private volatile boolean connected = false; @@ -99,9 +99,9 @@ public class PublishVideoOperation implements Operation { } // Copy the image data into a pre-allocated buffer, growing it if necessary - bufferSize = imagePointer.limit(); + bufferSize = (int) imagePointer.limit(); if (bufferSize > buffer.length) { - buffer = new byte[imagePointer.limit()]; + buffer = new byte[(int) imagePointer.limit()]; } imagePointer.get(buffer, 0, bufferSize); hasImage = false; @@ -144,8 +144,7 @@ public PublishVideoOperation(InputSocket.Factory inputSocketFactory) { if (numSteps != 0) { throw new IllegalStateException("Only one instance of PublishVideoOperation may exist"); } - this.inputSocket = inputSocketFactory.create(SocketHints.Inputs.createMatSocketHint("Image", - false)); + this.inputSocket = inputSocketFactory.create(SocketHints.createImageSocketHint("Image")); this.qualitySocket = inputSocketFactory.create(SocketHints.Inputs .createNumberSliderSocketHint("Quality", 80, 0, 100)); numSteps++; @@ -179,7 +178,7 @@ public void perform() { } synchronized (imageLock) { - imencode(".jpeg", inputSocket.getValue().get(), imagePointer, + imencode(".jpeg", inputSocket.getValue().get().getCpu(), imagePointer, new IntPointer(CV_IMWRITE_JPEG_QUALITY, qualitySocket.getValue().get().intValue())); hasImage = true; imageLock.notifyAll(); diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/RGBThresholdOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/RGBThresholdOperation.java index 481327ebf8..43ecb57dd9 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/RGBThresholdOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/RGBThresholdOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -30,7 +31,7 @@ public class RGBThresholdOperation extends ThresholdOperation { private static final Logger logger = Logger.getLogger(RGBThresholdOperation.class.getName()); - private final SocketHint inputHint = SocketHints.Inputs.createMatSocketHint("Input", false); + private final SocketHint inputHint = SocketHints.createImageSocketHint("Input"); private final SocketHint> redHint = SocketHints.Inputs .createNumberListRangeSocketHint("Red", 0.0, 255.0); private final SocketHint> greenHint = SocketHints.Inputs @@ -38,15 +39,15 @@ public class RGBThresholdOperation extends ThresholdOperation { private final SocketHint> blueHint = SocketHints.Inputs .createNumberListRangeSocketHint("Blue", 0.0, 255.0); - private final SocketHint outputHint = SocketHints.Outputs.createMatSocketHint("Output"); + private final SocketHint outputHint = SocketHints.createImageSocketHint("Output"); - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final InputSocket> redSocket; private final InputSocket> greenSocket; private final InputSocket> blueSocket; - private final OutputSocket outputSocket; + private final OutputSocket outputSocket; @Inject @SuppressWarnings("JavadocMethod") @@ -79,13 +80,13 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = inputSocket.getValue().get(); + final Mat input = inputSocket.getValue().get().getCpu(); if (input.channels() != 3) { throw new IllegalArgumentException("RGB Threshold needs a 3-channel input"); } - final Mat output = outputSocket.getValue().get(); + final Mat output = outputSocket.getValue().get().rawCpu(); final List channel1 = redSocket.getValue().get(); final List channel2 = greenSocket.getValue().get(); final List channel3 = blueSocket.getValue().get(); @@ -106,7 +107,7 @@ public void perform() { try { inRange(input, low, high, output); - outputSocket.setValue(output); + outputSocket.flagChanged(); } catch (RuntimeException e) { logger.log(Level.WARNING, e.getMessage(), e); } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/RectsReport.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/RectsReport.java index 0b33581057..1ed7405e10 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/RectsReport.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/RectsReport.java @@ -1,13 +1,13 @@ package edu.wpi.grip.core.operations.composite; import edu.wpi.grip.annotation.operation.PublishableObject; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.operations.network.PublishValue; import edu.wpi.grip.core.operations.network.Publishable; import edu.wpi.grip.core.sockets.NoSocketTypeLabel; import com.google.common.collect.ImmutableList; -import org.bytedeco.javacpp.opencv_core.Mat; import org.bytedeco.javacpp.opencv_core.Rect; import java.util.ArrayList; @@ -20,12 +20,13 @@ @NoSocketTypeLabel public class RectsReport implements Publishable { - private final Mat image; + private final MatWrapper image; private final List rectangles; - public static final RectsReport NIL = new RectsReport(new Mat(), new ArrayList<>()); + public static final RectsReport NIL + = new RectsReport(MatWrapper.emptyWrapper(), new ArrayList<>()); - public RectsReport(Mat image, List rectangles) { + public RectsReport(MatWrapper image, List rectangles) { this.image = image; this.rectangles = ImmutableList.copyOf(rectangles); } @@ -33,7 +34,7 @@ public RectsReport(Mat image, List rectangles) { /** * Gets the image the rectangles are for. */ - public Mat getImage() { + public MatWrapper getImage() { return image; } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/ResizeOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/ResizeOperation.java index 20ea8817a3..45b59f9daa 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/ResizeOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/ResizeOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -32,19 +33,18 @@ iconName = "resize") public class ResizeOperation implements Operation { - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final InputSocket widthSocket; private final InputSocket heightSocket; private final InputSocket interpolationSocket; - private final OutputSocket outputSocket; + private final OutputSocket outputSocket; @Inject @SuppressWarnings("JavadocMethod") public ResizeOperation(InputSocket.Factory inputSocketFactory, OutputSocket.Factory outputSocketFactory) { - this.inputSocket = inputSocketFactory.create(SocketHints.Inputs - .createMatSocketHint("Input", false)); + this.inputSocket = inputSocketFactory.create(SocketHints.createImageSocketHint("Input")); this.widthSocket = inputSocketFactory.create(SocketHints.Inputs .createNumberSpinnerSocketHint("Width", 640)); this.heightSocket = inputSocketFactory.create(SocketHints.Inputs @@ -52,8 +52,7 @@ public ResizeOperation(InputSocket.Factory inputSocketFactory, OutputSocket.Fact this.interpolationSocket = inputSocketFactory .create(SocketHints.createEnumSocketHint("Interpolation", Interpolation.CUBIC)); - this.outputSocket = outputSocketFactory.create(SocketHints.Outputs - .createMatSocketHint("Output")); + this.outputSocket = outputSocketFactory.create(SocketHints.createImageSocketHint("Output")); } @Override @@ -75,17 +74,17 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = inputSocket.getValue().get(); + final Mat input = inputSocket.getValue().get().getCpu(); final Number width = widthSocket.getValue().get(); final Number height = heightSocket.getValue().get(); final Interpolation interpolation = interpolationSocket.getValue().get(); - final Mat output = outputSocket.getValue().get(); + final Mat output = outputSocket.getValue().get().rawCpu(); resize(input, output, new Size(width.intValue(), height.intValue()), 0.0, 0.0, interpolation .value); - outputSocket.setValue(output); + outputSocket.flagChanged(); } private enum Interpolation { diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/SaveImageOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/SaveImageOperation.java index 6f6fc08e60..90b2f9f130 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/SaveImageOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/SaveImageOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.core.FileManager; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -21,7 +22,6 @@ import java.util.Locale; import java.util.concurrent.TimeUnit; -import static org.bytedeco.javacpp.opencv_core.Mat; import static org.bytedeco.javacpp.opencv_imgcodecs.CV_IMWRITE_JPEG_QUALITY; import static org.bytedeco.javacpp.opencv_imgcodecs.imencode; @@ -33,8 +33,8 @@ iconName = "publish-video") public class SaveImageOperation implements Operation { - private final SocketHint inputHint - = SocketHints.Inputs.createMatSocketHint("Input", false); + private final SocketHint inputHint + = SocketHints.createImageSocketHint("Input"); private final SocketHint fileTypeHint = SocketHints.createEnumSocketHint("File type", FileTypes.JPEG); private final SocketHint qualityHint @@ -44,15 +44,15 @@ public class SaveImageOperation implements Operation { private final SocketHint activeHint = SocketHints.Inputs.createCheckboxSocketHint("Active", false); - private final SocketHint outputHint = SocketHints.Outputs.createMatSocketHint("Output"); + private final SocketHint outputHint = SocketHints.createImageSocketHint("Output"); - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final InputSocket fileTypesSocket; private final InputSocket qualitySocket; private final InputSocket periodSocket; private final InputSocket activeSocket; - private final OutputSocket outputSocket; + private final OutputSocket outputSocket; private final FileManager fileManager; private final BytePointer imagePointer = new BytePointer(); @@ -61,7 +61,8 @@ public class SaveImageOperation implements Operation { = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH-mm-ss-SSS"); private enum FileTypes { - JPEG, PNG; + JPEG, + PNG; @Override public String toString() { @@ -117,12 +118,14 @@ public void perform() { stopwatch.reset(); stopwatch.start(); - imencode("." + fileTypesSocket.getValue().get(), inputSocket.getValue().get(), imagePointer, + imencode("." + fileTypesSocket.getValue().get(), + inputSocket.getValue().get().getCpu(), + imagePointer, new IntPointer(CV_IMWRITE_JPEG_QUALITY, qualitySocket.getValue().get().intValue())); byte[] buffer = new byte[128 * 1024]; - int bufferSize = imagePointer.limit(); + int bufferSize = (int) imagePointer.limit(); if (bufferSize > buffer.length) { - buffer = new byte[imagePointer.limit()]; + buffer = new byte[(int) imagePointer.limit()]; } imagePointer.get(buffer, 0, bufferSize); diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/ThresholdMoving.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/ThresholdMoving.java index 5831b6864d..ce9d79d193 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/ThresholdMoving.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/ThresholdMoving.java @@ -1,6 +1,7 @@ package edu.wpi.grip.core.operations.composite; import edu.wpi.grip.annotation.operation.Description; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -23,16 +24,16 @@ + " previous and next image.") public class ThresholdMoving implements Operation { - private final InputSocket imageSocket; - private final OutputSocket outputSocket; + private final InputSocket imageSocket; + private final OutputSocket outputSocket; private final Mat lastImage; @Inject @SuppressWarnings("JavadocMethod") public ThresholdMoving(InputSocket.Factory inputSocketFactory, OutputSocket.Factory outputSocketFactory) { - imageSocket = inputSocketFactory.create(SocketHints.Inputs.createMatSocketHint("image", false)); - outputSocket = outputSocketFactory.create(SocketHints.Outputs.createMatSocketHint("moved")); + imageSocket = inputSocketFactory.create(SocketHints.createImageSocketHint("image")); + outputSocket = outputSocketFactory.create(SocketHints.createImageSocketHint("moved")); lastImage = new Mat(); } @@ -52,14 +53,14 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = imageSocket.getValue().get(); + final Mat input = imageSocket.getValue().get().getCpu(); final Size lastSize = lastImage.size(); final Size inputSize = input.size(); if (!lastImage.empty() && lastSize.height() == inputSize.height() && lastSize.width() == inputSize.width()) { - opencv_core.absdiff(input, lastImage, outputSocket.getValue().get()); + opencv_core.absdiff(input, lastImage, outputSocket.getValue().get().rawCpu()); } input.copyTo(lastImage); - outputSocket.setValue(outputSocket.getValue().get()); + outputSocket.flagChanged(); } } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/WatershedOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/WatershedOperation.java index 30a8f1a748..d84ebd835a 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/WatershedOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/WatershedOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -47,7 +48,7 @@ iconName = "opencv") public class WatershedOperation implements Operation { - private final SocketHint srcHint = SocketHints.Inputs.createMatSocketHint("Input", false); + private final SocketHint srcHint = SocketHints.createImageSocketHint("Input"); private final SocketHint contoursHint = new SocketHint.Builder<>(ContoursReport.class) .identifier("Contours") @@ -60,7 +61,7 @@ public class WatershedOperation implements Operation { .initialValueSupplier(ContoursReport::new) .build(); - private final InputSocket srcSocket; + private final InputSocket srcSocket; private final InputSocket contoursSocket; private final OutputSocket outputSocket; @@ -100,7 +101,7 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = srcSocket.getValue().get(); + final Mat input = srcSocket.getValue().get().getCpu(); if (input.type() != CV_8UC3) { throw new IllegalArgumentException("Watershed only works on 8-bit, 3-channel images"); } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/network/networktables/NTManager.java b/core/src/main/java/edu/wpi/grip/core/operations/network/networktables/NTManager.java index 4ffa01f351..ede2f88b09 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/network/networktables/NTManager.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/network/networktables/NTManager.java @@ -30,6 +30,7 @@ import java.util.function.Consumer; import java.util.logging.Level; import java.util.logging.Logger; + import javax.inject.Inject; import static com.google.common.base.Preconditions.checkNotNull; diff --git a/core/src/main/java/edu/wpi/grip/core/operations/network/ros/JavaToMessageConverter.java b/core/src/main/java/edu/wpi/grip/core/operations/network/ros/JavaToMessageConverter.java index 85a8f14757..0cda4e8179 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/network/ros/JavaToMessageConverter.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/network/ros/JavaToMessageConverter.java @@ -136,7 +136,7 @@ public void convert(J javaType, Message message, MessageFactory messageFactory) } private abstract static class SimpleConverter extends - JavaToMessageConverter { + JavaToMessageConverter { private final BiConsumer messageDataAssigner; private SimpleConverter(String type, BiConsumer messageDataAssigner) { diff --git a/core/src/main/java/edu/wpi/grip/core/operations/opencv/MatFieldAccessor.java b/core/src/main/java/edu/wpi/grip/core/operations/opencv/MatFieldAccessor.java index 2b8ba3c2fa..47311e2fa3 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/opencv/MatFieldAccessor.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/opencv/MatFieldAccessor.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.core.sockets.SocketHint; @@ -22,7 +23,7 @@ public class MatFieldAccessor implements CVOperation { private static final Mat defaultsMat = new Mat(); - private final SocketHint matHint = SocketHints.Inputs.createMatSocketHint("Input", false); + private final SocketHint matHint = SocketHints.createImageSocketHint("Input"); private final SocketHint sizeHint = SocketHints.Inputs.createSizeSocketHint("size", true); private final SocketHint emptyHint = SocketHints.Outputs .createBooleanSocketHint("empty", defaultsMat.empty()); @@ -36,7 +37,7 @@ public class MatFieldAccessor implements CVOperation { .createNumberSocketHint("high value", defaultsMat.highValue()); - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final OutputSocket sizeSocket; private final OutputSocket emptySocket; @@ -80,13 +81,13 @@ public List getOutputSockets() { @Override public void perform() { - final Mat inputMat = inputSocket.getValue().get(); + MatWrapper wrapper = inputSocket.getValue().get(); - sizeSocket.setValue(inputMat.size()); - emptySocket.setValue(inputMat.empty()); - channelsSocket.setValue(inputMat.channels()); - colsSocket.setValue(inputMat.cols()); - rowsSocket.setValue(inputMat.rows()); - highValueSocket.setValue(inputMat.highValue()); + sizeSocket.setValue(wrapper.size()); + emptySocket.setValue(wrapper.empty()); + channelsSocket.setValue(wrapper.channels()); + colsSocket.setValue(wrapper.cols()); + rowsSocket.setValue(wrapper.rows()); + highValueSocket.setValue(wrapper.highValue()); } } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/opencv/MinMaxLoc.java b/core/src/main/java/edu/wpi/grip/core/operations/opencv/MinMaxLoc.java index 84e364026a..5d975d46a1 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/opencv/MinMaxLoc.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/opencv/MinMaxLoc.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.core.sockets.SocketHint; @@ -10,6 +11,7 @@ import com.google.common.collect.ImmutableList; import com.google.inject.Inject; +import org.bytedeco.javacpp.DoublePointer; import org.bytedeco.javacpp.opencv_core; import org.bytedeco.javacpp.opencv_core.Mat; import org.bytedeco.javacpp.opencv_core.Point; @@ -25,10 +27,8 @@ iconName = "opencv") public class MinMaxLoc implements CVOperation { - private final SocketHint srcInputHint = SocketHints.Inputs - .createMatSocketHint("Image", false); - private final SocketHint maskInputHint = SocketHints.Inputs - .createMatSocketHint("Mask", true); + private final SocketHint srcInputHint = SocketHints.createImageSocketHint("Image"); + private final SocketHint maskInputHint = SocketHints.createImageSocketHint("Mask"); private final SocketHint minValOutputHint = SocketHints.Outputs .createNumberSocketHint("Min Val", 0); @@ -40,8 +40,8 @@ public class MinMaxLoc implements CVOperation { private final SocketHint maxLocOutputHint = SocketHints.Outputs .createPointSocketHint("Max Loc"); - private final InputSocket srcSocket; - private final InputSocket maskSocket; + private final InputSocket srcSocket; + private final InputSocket maskSocket; private final OutputSocket minValSocket; private final OutputSocket maxValSocket; @@ -81,20 +81,20 @@ public List getOutputSockets() { @Override public void perform() { - final Mat src = srcSocket.getValue().get(); - Mat mask = maskSocket.getValue().get(); + final Mat src = srcSocket.getValue().get().getCpu(); + Mat mask = maskSocket.getValue().get().getCpu(); if (mask.empty()) { mask = null; } - final double[] minVal = new double[1]; - final double[] maxVal = new double[1]; + DoublePointer minVal = new DoublePointer(0.0); + DoublePointer maxVal = new DoublePointer(0.0); final Point minLoc = minLocSocket.getValue().get(); final Point maxLoc = maxLocSocket.getValue().get(); opencv_core.minMaxLoc(src, minVal, maxVal, minLoc, maxLoc, mask); - minValSocket.setValue(minVal[0]); - maxValSocket.setValue(maxVal[0]); - minLocSocket.setValue(minLocSocket.getValue().get()); - maxLocSocket.setValue(maxLocSocket.getValue().get()); + minValSocket.setValue(minVal.get()); + maxValSocket.setValue(maxVal.get()); + minLocSocket.flagChanged(); + maxLocSocket.flagChanged(); } } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/FiveSourceOneDestinationCudaOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/FiveSourceOneDestinationCudaOperation.java new file mode 100644 index 0000000000..4abb81ce7c --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/FiveSourceOneDestinationCudaOperation.java @@ -0,0 +1,79 @@ +package edu.wpi.grip.core.operations.templated; + +import edu.wpi.grip.core.operations.CudaOperation; +import edu.wpi.grip.core.sockets.InputSocket; +import edu.wpi.grip.core.sockets.OutputSocket; +import edu.wpi.grip.core.sockets.SocketHint; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +final class FiveSourceOneDestinationCudaOperation extends CudaOperation { + + private final InputSocket input1; + private final InputSocket input2; + private final InputSocket input3; + private final InputSocket input4; + private final InputSocket input5; + private final OutputSocket output; + private final Performer performer; + + FiveSourceOneDestinationCudaOperation(InputSocket.Factory isf, + OutputSocket.Factory osf, + SocketHint t1SocketHint, + SocketHint t2SocketHint, + SocketHint t3SocketHint, + SocketHint t4SocketHint, + SocketHint t5SocketHint, + SocketHint rSocketHint, + Performer performer) { + super(isf, osf); + this.input1 = isf.create(t1SocketHint); + this.input2 = isf.create(t2SocketHint); + this.input3 = isf.create(t3SocketHint); + this.input4 = isf.create(t4SocketHint); + this.input5 = isf.create(t5SocketHint); + this.output = osf.create(rSocketHint); + this.performer = performer; + } + + @Override + public List getInputSockets() { + return ImmutableList.of( + input1, + input2, + input3, + input4, + input5, + gpuSocket + ); + } + + @Override + public List getOutputSockets() { + return ImmutableList.of( + output + ); + } + + @Override + public void perform() { + performer.perform( + input1.getValue().get(), + input2.getValue().get(), + input3.getValue().get(), + input4.getValue().get(), + input5.getValue().get(), + preferCuda(), + output.getValue().get() + ); + output.flagChanged(); + } + + @FunctionalInterface + public interface Performer { + void perform(T1 src1, T2 src2, T3 src3, T4 src4, T5 src5, boolean preferCuda, R dst); + } + +} diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/FiveSourceOneDestinationOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/FiveSourceOneDestinationOperation.java index 80fea16669..bb2be4a3cc 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/templated/FiveSourceOneDestinationOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/FiveSourceOneDestinationOperation.java @@ -59,7 +59,7 @@ public void perform() { input4.getValue().get(), input5.getValue().get(), output.getValue().get()); - output.setValue(output.getValue().get()); + output.flagChanged(); } @FunctionalInterface diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/FourSourceOneDestinationCudaOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/FourSourceOneDestinationCudaOperation.java new file mode 100644 index 0000000000..f69cbc4419 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/FourSourceOneDestinationCudaOperation.java @@ -0,0 +1,74 @@ +package edu.wpi.grip.core.operations.templated; + +import edu.wpi.grip.core.operations.CudaOperation; +import edu.wpi.grip.core.sockets.InputSocket; +import edu.wpi.grip.core.sockets.OutputSocket; +import edu.wpi.grip.core.sockets.SocketHint; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +final class FourSourceOneDestinationCudaOperation extends CudaOperation { + + private final InputSocket input1; + private final InputSocket input2; + private final InputSocket input3; + private final InputSocket input4; + private final OutputSocket output; + private final Performer performer; + + FourSourceOneDestinationCudaOperation(InputSocket.Factory isf, + OutputSocket.Factory osf, + SocketHint t1SocketHint, + SocketHint t2SocketHint, + SocketHint t3SocketHint, + SocketHint t4SocketHint, + SocketHint rSocketHint, + Performer performer) { + super(isf, osf); + this.input1 = isf.create(t1SocketHint); + this.input2 = isf.create(t2SocketHint); + this.input3 = isf.create(t3SocketHint); + this.input4 = isf.create(t4SocketHint); + this.output = osf.create(rSocketHint); + this.performer = performer; + } + + @Override + public List getInputSockets() { + return ImmutableList.of( + input1, + input2, + input3, + input4, + gpuSocket + ); + } + + @Override + public List getOutputSockets() { + return ImmutableList.of( + output + ); + } + + @Override + public void perform() { + performer.perform( + input1.getValue().get(), + input2.getValue().get(), + input3.getValue().get(), + input4.getValue().get(), + preferCuda(), + output.getValue().get() + ); + output.flagChanged(); + } + + @FunctionalInterface + public interface Performer { + void perform(T1 src1, T2 src2, T3 src3, T4 src4, boolean preferCuda, R dst); + } + +} diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/FourSourceOneDestinationOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/FourSourceOneDestinationOperation.java index de040f9e18..302a0a0079 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/templated/FourSourceOneDestinationOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/FourSourceOneDestinationOperation.java @@ -72,7 +72,7 @@ public void perform() { input4.getValue().get(), output.getValue().get() ); - output.setValue(output.getValue().get()); + output.flagChanged(); } @FunctionalInterface diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/OneSourceOneDestinationCudaOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/OneSourceOneDestinationCudaOperation.java new file mode 100644 index 0000000000..68cc21728f --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/OneSourceOneDestinationCudaOperation.java @@ -0,0 +1,55 @@ +package edu.wpi.grip.core.operations.templated; + +import edu.wpi.grip.core.operations.CudaOperation; +import edu.wpi.grip.core.sockets.InputSocket; +import edu.wpi.grip.core.sockets.OutputSocket; +import edu.wpi.grip.core.sockets.SocketHint; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +final class OneSourceOneDestinationCudaOperation extends CudaOperation { + + private final InputSocket input1; + private final OutputSocket output; + private final Performer performer; + + OneSourceOneDestinationCudaOperation(InputSocket.Factory isf, + OutputSocket.Factory osf, + SocketHint t1SocketHint, + SocketHint rSocketHint, + Performer performer) { + super(isf, osf); + this.input1 = isf.create(t1SocketHint); + this.output = osf.create(rSocketHint); + this.performer = performer; + } + + @Override + public List getInputSockets() { + return ImmutableList.of( + input1, + gpuSocket + ); + } + + @Override + public List getOutputSockets() { + return ImmutableList.of( + output + ); + } + + @Override + public void perform() { + performer.perform(input1.getValue().get(), preferCuda(), output.getValue().get()); + output.flagChanged(); + } + + @FunctionalInterface + public interface Performer { + void perform(T1 src1, boolean preferCuda, R dst); + } + +} diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/OneSourceOneDestinationOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/OneSourceOneDestinationOperation.java index 826497f9c8..22cefe9844 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/templated/OneSourceOneDestinationOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/OneSourceOneDestinationOperation.java @@ -40,7 +40,7 @@ public List getOutputSockets() { @SuppressWarnings("OptionalGetWithoutIsPresent") public void perform() { performer.perform(input1.getValue().get(), output.getValue().get()); - output.setValue(output.getValue().get()); + output.flagChanged(); } @FunctionalInterface diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/SevenSourceOneDestinationCudaOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/SevenSourceOneDestinationCudaOperation.java new file mode 100644 index 0000000000..b7429dd41e --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/SevenSourceOneDestinationCudaOperation.java @@ -0,0 +1,91 @@ +package edu.wpi.grip.core.operations.templated; + +import edu.wpi.grip.core.operations.CudaOperation; +import edu.wpi.grip.core.sockets.InputSocket; +import edu.wpi.grip.core.sockets.OutputSocket; +import edu.wpi.grip.core.sockets.SocketHint; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +final class SevenSourceOneDestinationCudaOperation + extends CudaOperation { + + private final InputSocket input1; + private final InputSocket input2; + private final InputSocket input3; + private final InputSocket input4; + private final InputSocket input5; + private final InputSocket input6; + private final InputSocket input7; + private final OutputSocket output; + private final Performer performer; + + SevenSourceOneDestinationCudaOperation(InputSocket.Factory isf, + OutputSocket.Factory osf, + SocketHint t1SocketHint, + SocketHint t2SocketHint, + SocketHint t3SocketHint, + SocketHint t4SocketHint, + SocketHint t5SocketHint, + SocketHint t6SocketHint, + SocketHint t7SocketHint, + SocketHint rSocketHint, + Performer performer) { + super(isf, osf); + this.input1 = isf.create(t1SocketHint); + this.input2 = isf.create(t2SocketHint); + this.input3 = isf.create(t3SocketHint); + this.input4 = isf.create(t4SocketHint); + this.input5 = isf.create(t5SocketHint); + this.input6 = isf.create(t6SocketHint); + this.input7 = isf.create(t7SocketHint); + this.output = osf.create(rSocketHint); + this.performer = performer; + } + + @Override + public List getInputSockets() { + return ImmutableList.of( + input1, + input2, + input3, + input4, + input5, + input6, + input7, + gpuSocket + ); + } + + @Override + public List getOutputSockets() { + return ImmutableList.of( + output + ); + } + + @Override + public void perform() { + performer.perform( + input1.getValue().get(), + input2.getValue().get(), + input3.getValue().get(), + input4.getValue().get(), + input5.getValue().get(), + input6.getValue().get(), + input7.getValue().get(), + preferCuda(), + output.getValue().get() + ); + output.flagChanged(); + } + + @FunctionalInterface + public interface Performer { + void perform(T1 src1, T2 src2, T3 src3, T4 src4, T5 src5, T6 src6, T7 src7, + boolean preferCuda, R dst); + } + +} diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/SevenSourceOneDestinationOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/SevenSourceOneDestinationOperation.java index 017e41bbaf..650ee93884 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/templated/SevenSourceOneDestinationOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/SevenSourceOneDestinationOperation.java @@ -69,7 +69,7 @@ public void perform() { input7.getValue().get(), output.getValue().get() ); - output.setValue(output.getValue().get()); + output.flagChanged(); } @FunctionalInterface diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/SixSourceOneDestinationCudaOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/SixSourceOneDestinationCudaOperation.java new file mode 100644 index 0000000000..a426027a43 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/SixSourceOneDestinationCudaOperation.java @@ -0,0 +1,84 @@ +package edu.wpi.grip.core.operations.templated; + +import edu.wpi.grip.core.operations.CudaOperation; +import edu.wpi.grip.core.sockets.InputSocket; +import edu.wpi.grip.core.sockets.OutputSocket; +import edu.wpi.grip.core.sockets.SocketHint; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +final class SixSourceOneDestinationCudaOperation extends CudaOperation { + + private final InputSocket input1; + private final InputSocket input2; + private final InputSocket input3; + private final InputSocket input4; + private final InputSocket input5; + private final InputSocket input6; + private final OutputSocket output; + private final Performer performer; + + SixSourceOneDestinationCudaOperation(InputSocket.Factory isf, + OutputSocket.Factory osf, + SocketHint t1SocketHint, + SocketHint t2SocketHint, + SocketHint t3SocketHint, + SocketHint t4SocketHint, + SocketHint t5SocketHint, + SocketHint t6SocketHint, + SocketHint rSocketHint, + Performer performer) { + super(isf, osf); + this.input1 = isf.create(t1SocketHint); + this.input2 = isf.create(t2SocketHint); + this.input3 = isf.create(t3SocketHint); + this.input4 = isf.create(t4SocketHint); + this.input5 = isf.create(t5SocketHint); + this.input6 = isf.create(t6SocketHint); + this.output = osf.create(rSocketHint); + this.performer = performer; + } + + @Override + public List getInputSockets() { + return ImmutableList.of( + input1, + input2, + input3, + input4, + input5, + input6, + gpuSocket + ); + } + + @Override + public List getOutputSockets() { + return ImmutableList.of( + output + ); + } + + @Override + public void perform() { + performer.perform( + input1.getValue().get(), + input2.getValue().get(), + input3.getValue().get(), + input4.getValue().get(), + input5.getValue().get(), + input6.getValue().get(), + preferCuda(), + output.getValue().get() + ); + output.flagChanged(); + } + + @FunctionalInterface + public interface Performer { + void perform(T1 src1, T2 src2, T3 src3, T4 src4, T5 src5, T6 src6, boolean preferCuda, R dst); + } + +} diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/SixSourceOneDestinationOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/SixSourceOneDestinationOperation.java index 6c1ece513b..4c03bd1c3e 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/templated/SixSourceOneDestinationOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/SixSourceOneDestinationOperation.java @@ -65,7 +65,7 @@ public void perform() { input6.getValue().get(), output.getValue().get() ); - output.setValue(output.getValue().get()); + output.flagChanged(); } @FunctionalInterface diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/TemplateFactory.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/TemplateFactory.java index 415e248d1d..31a4862bd9 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/templated/TemplateFactory.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/TemplateFactory.java @@ -1,6 +1,7 @@ package edu.wpi.grip.core.operations.templated; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -9,8 +10,6 @@ import com.google.inject.Singleton; -import org.bytedeco.javacpp.opencv_core.Mat; - import java.util.function.Supplier; /** @@ -113,38 +112,137 @@ public Supplier create( performer); } + public Supplier createCuda( + SocketHint t1SocketHint, + SocketHint rSocketHint, + OneSourceOneDestinationCudaOperation.Performer performer) { + return () -> new OneSourceOneDestinationCudaOperation<>(isf, osf, t1SocketHint, + rSocketHint, performer); + } + + public Supplier createCuda( + SocketHint t1SocketHint, + SocketHint t2SocketHint, + SocketHint rSocketHint, + TwoSourceOneDestinationCudaOperation.Performer performer) { + return () -> new TwoSourceOneDestinationCudaOperation<>(isf, osf, t1SocketHint, t2SocketHint, + rSocketHint, performer); + } + + public Supplier createCuda( + SocketHint t1SocketHint, + SocketHint t2SocketHint, + SocketHint t3SocketHint, + SocketHint rSocketHint, + ThreeSourceOneDestinationCudaOperation.Performer performer) { + return () -> new ThreeSourceOneDestinationCudaOperation<>(isf, osf, t1SocketHint, t2SocketHint, + t3SocketHint, rSocketHint, performer); + } + + public Supplier createCuda( + SocketHint t1SocketHint, + SocketHint t2SocketHint, + SocketHint t3SocketHint, + SocketHint t4SocketHint, + SocketHint rSocketHint, + FourSourceOneDestinationCudaOperation.Performer performer) { + return () -> new FourSourceOneDestinationCudaOperation<>(isf, osf, t1SocketHint, t2SocketHint, + t3SocketHint, t4SocketHint, rSocketHint, performer); + } + + public Supplier createCuda( + SocketHint t1SocketHint, + SocketHint t2SocketHint, + SocketHint t3SocketHint, + SocketHint t4SocketHint, + SocketHint t5SocketHint, + SocketHint rSocketHint, + FiveSourceOneDestinationCudaOperation.Performer performer) { + return () -> new FiveSourceOneDestinationCudaOperation<>(isf, osf, t1SocketHint, t2SocketHint, + t3SocketHint, t4SocketHint, t5SocketHint, rSocketHint, performer); + } + + public Supplier createCuda( + SocketHint t1SocketHint, + SocketHint t2SocketHint, + SocketHint t3SocketHint, + SocketHint t4SocketHint, + SocketHint t5SocketHint, + SocketHint t6SocketHint, + SocketHint rSocketHint, + SixSourceOneDestinationCudaOperation.Performer performer) { + return () -> new SixSourceOneDestinationCudaOperation<>(isf, osf, t1SocketHint, t2SocketHint, + t3SocketHint, t4SocketHint, t5SocketHint, t6SocketHint, rSocketHint, performer); + } + + public Supplier createCuda( + SocketHint t1SocketHint, + SocketHint t2SocketHint, + SocketHint t3SocketHint, + SocketHint t4SocketHint, + SocketHint t5SocketHint, + SocketHint t6SocketHint, + SocketHint t7SocketHint, + SocketHint rSocketHint, + SevenSourceOneDestinationCudaOperation.Performer performer) { + return () -> new SevenSourceOneDestinationCudaOperation<>(isf, osf, t1SocketHint, t2SocketHint, + t3SocketHint, t4SocketHint, t5SocketHint, t6SocketHint, t7SocketHint, rSocketHint, + performer); + } + public Supplier createAllMatTwoSource( - SocketHint matSocketHint, - SocketHint matSocketHint2, - SocketHint matSocketHint3, - TwoSourceOneDestinationOperation.Performer performer) { + SocketHint matSocketHint, + SocketHint matSocketHint2, + SocketHint matSocketHint3, + TwoSourceOneDestinationOperation.Performer performer) { return create(matSocketHint, matSocketHint2, matSocketHint3, performer); } - public Supplier createAllMatTwoSource(TwoSourceOneDestinationOperation - .Performer performer) { - return createAllMatTwoSource(srcSocketHint(Mat.class, 1), srcSocketHint(Mat.class, 2), + public Supplier createAllMatTwoSource( + TwoSourceOneDestinationOperation.Performer performer) { + return createAllMatTwoSource(srcSocketHint(MatWrapper.class, 1), + srcSocketHint(MatWrapper.class, 2), dstMatSocketHint(), performer); } + public Supplier createAllMatTwoSourceCuda( + TwoSourceOneDestinationCudaOperation.Performer + performer) { + return createCuda( + srcSocketHint(MatWrapper.class, 1), + srcSocketHint(MatWrapper.class, 2), + dstMatSocketHint(), + performer + ); + } + public Supplier createAllMatOneSource( - SocketHint matSocketHint, - SocketHint matSocketHint2, - OneSourceOneDestinationOperation.Performer performer) { + SocketHint matSocketHint, + SocketHint matSocketHint2, + OneSourceOneDestinationOperation.Performer performer) { return create(matSocketHint, matSocketHint2, performer); } - public Supplier createAllMatOneSource(OneSourceOneDestinationOperation - .Performer performer) { - return createAllMatOneSource(srcSocketHint(Mat.class, 1), dstMatSocketHint(), performer); + public Supplier createAllMatOneSource( + OneSourceOneDestinationOperation.Performer performer) { + return createAllMatOneSource(srcSocketHint(MatWrapper.class, 1), dstMatSocketHint(), + performer); } + public Supplier createAllMatOneSourceCuda( + OneSourceOneDestinationCudaOperation.Performer performer) { + return createCuda( + srcSocketHint(MatWrapper.class, 1), + dstMatSocketHint(), + performer + ); + } private SocketHint srcSocketHint(Class srcType, int index) { return new SocketHint.Builder<>(srcType).identifier("src" + index).build(); } - private SocketHint dstMatSocketHint() { - return SocketHints.Outputs.createMatSocketHint("dst"); + private SocketHint dstMatSocketHint() { + return SocketHints.createImageSocketHint("dst"); } } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/ThreeSourceOneDestinationCudaOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/ThreeSourceOneDestinationCudaOperation.java new file mode 100644 index 0000000000..bee043b757 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/ThreeSourceOneDestinationCudaOperation.java @@ -0,0 +1,69 @@ +package edu.wpi.grip.core.operations.templated; + +import edu.wpi.grip.core.operations.CudaOperation; +import edu.wpi.grip.core.sockets.InputSocket; +import edu.wpi.grip.core.sockets.OutputSocket; +import edu.wpi.grip.core.sockets.SocketHint; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +final class ThreeSourceOneDestinationCudaOperation extends CudaOperation { + + private final InputSocket input1; + private final InputSocket input2; + private final InputSocket input3; + private final OutputSocket output; + private final Performer performer; + + ThreeSourceOneDestinationCudaOperation(InputSocket.Factory isf, + OutputSocket.Factory osf, + SocketHint t1SocketHint, + SocketHint t2SocketHint, + SocketHint t3SocketHint, + SocketHint rSocketHint, + Performer performer) { + super(isf, osf); + this.input1 = isf.create(t1SocketHint); + this.input2 = isf.create(t2SocketHint); + this.input3 = isf.create(t3SocketHint); + this.output = osf.create(rSocketHint); + this.performer = performer; + } + + @Override + public List getInputSockets() { + return ImmutableList.of( + input1, + input2, + input3, + gpuSocket + ); + } + + @Override + public List getOutputSockets() { + return ImmutableList.of( + output + ); + } + + @Override + public void perform() { + performer.perform( + input1.getValue().get(), + input2.getValue().get(), + input3.getValue().get(), + preferCuda(), + output.getValue().get() + ); + output.flagChanged(); + } + + @FunctionalInterface + public interface Performer { + void perform(T1 src1, T2 src2, T3 src3, boolean preferCuda, R dst); + } + +} diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/ThreeSourceOneDestinationOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/ThreeSourceOneDestinationOperation.java index 071ab60518..8556934f2a 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/templated/ThreeSourceOneDestinationOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/ThreeSourceOneDestinationOperation.java @@ -46,7 +46,7 @@ public List getOutputSockets() { public void perform() { performer.perform(input1.getValue().get(), input2.getValue().get(), input3.getValue().get(), output.getValue().get()); - output.setValue(output.getValue().get()); + output.flagChanged(); } @FunctionalInterface diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/TwoSourceOneDestinationCudaOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/TwoSourceOneDestinationCudaOperation.java new file mode 100644 index 0000000000..02381f100b --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/TwoSourceOneDestinationCudaOperation.java @@ -0,0 +1,64 @@ +package edu.wpi.grip.core.operations.templated; + +import edu.wpi.grip.core.operations.CudaOperation; +import edu.wpi.grip.core.sockets.InputSocket; +import edu.wpi.grip.core.sockets.OutputSocket; +import edu.wpi.grip.core.sockets.SocketHint; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +final class TwoSourceOneDestinationCudaOperation extends CudaOperation { + + private final InputSocket input1; + private final InputSocket input2; + private final OutputSocket output; + private final Performer performer; + + TwoSourceOneDestinationCudaOperation(InputSocket.Factory isf, + OutputSocket.Factory osf, + SocketHint t1SocketHint, + SocketHint t2SocketHint, + SocketHint rSocketHint, + Performer performer) { + super(isf, osf); + this.input1 = isf.create(t1SocketHint); + this.input2 = isf.create(t2SocketHint); + this.output = osf.create(rSocketHint); + this.performer = performer; + } + + @Override + public List getInputSockets() { + return ImmutableList.of( + input1, + input2, + gpuSocket + ); + } + + @Override + public List getOutputSockets() { + return ImmutableList.of( + output + ); + } + + @Override + public void perform() { + performer.perform( + input1.getValue().get(), + input2.getValue().get(), + preferCuda(), + output.getValue().get() + ); + output.flagChanged(); + } + + @FunctionalInterface + public interface Performer { + void perform(T1 src1, T2 src2, boolean preferCuda, R dst); + } + +} diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/TwoSourceOneDestinationOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/TwoSourceOneDestinationOperation.java index 874aab3d0f..39802834a8 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/templated/TwoSourceOneDestinationOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/TwoSourceOneDestinationOperation.java @@ -44,7 +44,7 @@ public List getOutputSockets() { @SuppressWarnings("OptionalGetWithoutIsPresent") public void perform() { performer.perform(input1.getValue().get(), input2.getValue().get(), output.getValue().get()); - output.setValue(output.getValue().get()); + output.flagChanged(); } @FunctionalInterface diff --git a/core/src/main/java/edu/wpi/grip/core/sockets/CudaSocket.java b/core/src/main/java/edu/wpi/grip/core/sockets/CudaSocket.java new file mode 100644 index 0000000000..d28d6c6099 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/sockets/CudaSocket.java @@ -0,0 +1,62 @@ +package edu.wpi.grip.core.sockets; + +import edu.wpi.grip.core.cuda.CudaDetector; + +import com.google.common.eventbus.EventBus; + +import java.util.Objects; +import java.util.Optional; + +import javax.annotation.Nullable; + +/** + * A type of input socket that lets an operation know that it should prefer to use a + * CUDA-accelerated code path. If no compatible CUDA runtime is available, sockets of this type + * will always have a value of {@code false} and cannot be changed. + */ +public class CudaSocket extends InputSocketImpl { + + private static final Optional NO = Optional.of(false); + + private final boolean isCudaAvailable; + + CudaSocket(EventBus eventBus, CudaDetector detector, SocketHint socketHint) { + super(eventBus, socketHint); + + // Cache the value, since it's not likely that the user will install or uninstall a CUDA + // runtime while the app is running + isCudaAvailable = detector.isCompatibleCudaInstalled(); + } + + @Override + public Optional getValue() { + if (isCudaAvailable) { + return super.getValue(); + } else { + return NO; + } + } + + @Override + public void setValue(@Nullable Boolean value) { + if (isCudaAvailable) { + super.setValue(value); + } else { + super.setValue(false); + } + } + + @Override + public void setValueOptional(Optional optionalValue) { + Objects.requireNonNull(optionalValue, "optionalValue"); + if (isCudaAvailable) { + super.setValueOptional(optionalValue); + } else { + super.setValueOptional(NO); + } + } + + public boolean isCudaAvailable() { + return isCudaAvailable; + } +} diff --git a/core/src/main/java/edu/wpi/grip/core/sockets/InputSocket.java b/core/src/main/java/edu/wpi/grip/core/sockets/InputSocket.java index b582c84986..15ed50e5c2 100644 --- a/core/src/main/java/edu/wpi/grip/core/sockets/InputSocket.java +++ b/core/src/main/java/edu/wpi/grip/core/sockets/InputSocket.java @@ -30,6 +30,8 @@ public interface InputSocket extends Socket { interface Factory { InputSocket create(SocketHint hint); + + CudaSocket createCuda(SocketHint hint); } /** diff --git a/core/src/main/java/edu/wpi/grip/core/sockets/InputSocketImpl.java b/core/src/main/java/edu/wpi/grip/core/sockets/InputSocketImpl.java index 40456510e5..32b450ad41 100644 --- a/core/src/main/java/edu/wpi/grip/core/sockets/InputSocketImpl.java +++ b/core/src/main/java/edu/wpi/grip/core/sockets/InputSocketImpl.java @@ -2,6 +2,7 @@ import edu.wpi.grip.core.Connection; +import edu.wpi.grip.core.cuda.CudaDetector; import com.google.common.eventbus.EventBus; import com.google.inject.Inject; @@ -56,16 +57,23 @@ public void onValueChanged() { @Singleton public static class FactoryImpl implements Factory { private final EventBus eventBus; + private final CudaDetector cudaDetector; @Inject - FactoryImpl(EventBus eventBus) { + FactoryImpl(EventBus eventBus, CudaDetector cudaDetector) { this.eventBus = eventBus; + this.cudaDetector = cudaDetector; } @Override public InputSocket create(SocketHint hint) { return new InputSocketImpl<>(eventBus, hint); } + + @Override + public CudaSocket createCuda(SocketHint hint) { + return new CudaSocket(eventBus, cudaDetector, hint); + } } diff --git a/core/src/main/java/edu/wpi/grip/core/sockets/Socket.java b/core/src/main/java/edu/wpi/grip/core/sockets/Socket.java index c8cdcfd038..70866617b1 100644 --- a/core/src/main/java/edu/wpi/grip/core/sockets/Socket.java +++ b/core/src/main/java/edu/wpi/grip/core/sockets/Socket.java @@ -41,6 +41,15 @@ default void onValueChanged() { /* no-op */ } + /** + * Notifies this socket that the value changed. This is usually only needed for sockets that + * contain mutable data such as images or other native classes (Point, Size, etc) that are + * written to by OpenCV operations. + */ + default void flagChanged() { + setValueOptional(getValue()); + } + /** * @return The value currently stored in this socket. */ diff --git a/core/src/main/java/edu/wpi/grip/core/sockets/SocketHint.java b/core/src/main/java/edu/wpi/grip/core/sockets/SocketHint.java index 0341102b28..b0005e73df 100644 --- a/core/src/main/java/edu/wpi/grip/core/sockets/SocketHint.java +++ b/core/src/main/java/edu/wpi/grip/core/sockets/SocketHint.java @@ -1,5 +1,7 @@ package edu.wpi.grip.core.sockets; +import edu.wpi.grip.core.MatWrapper; + import com.google.common.base.MoreObjects; import org.bytedeco.javacpp.opencv_objdetect.CascadeClassifier; @@ -121,7 +123,7 @@ public String getTypeLabel() { || type.equals(List.class)) { // Enums labels are kind of redundant, and Lists actually represent ranges return ""; - } else if (Mat.class.equals(type)) { + } else if (Mat.class.equals(type) || MatWrapper.class.equals(type)) { // "Mats" represent images return "Image"; } else if (CascadeClassifier.class.equals(type)) { diff --git a/core/src/main/java/edu/wpi/grip/core/sockets/SocketHints.java b/core/src/main/java/edu/wpi/grip/core/sockets/SocketHints.java index 6fbb8d1e32..50924e6876 100644 --- a/core/src/main/java/edu/wpi/grip/core/sockets/SocketHints.java +++ b/core/src/main/java/edu/wpi/grip/core/sockets/SocketHints.java @@ -1,9 +1,10 @@ package edu.wpi.grip.core.sockets; +import edu.wpi.grip.core.MatWrapper; + import com.google.common.reflect.TypeToken; -import org.bytedeco.javacpp.opencv_core.Mat; import org.bytedeco.javacpp.opencv_core.Point; import org.bytedeco.javacpp.opencv_core.Size; @@ -23,6 +24,13 @@ public final class SocketHints { private SocketHints() { /* no op */ } + public static SocketHint createImageSocketHint(String identifier) { + return new SocketHint.Builder<>(MatWrapper.class) + .identifier(identifier) + .initialValueSupplier(MatWrapper::emptyWrapper) + .build(); + } + @SuppressWarnings("unchecked") public static > SocketHint createEnumSocketHint(final String identifier, final T defaultValue) { @@ -93,11 +101,6 @@ private static SocketHint.Builder createNumberSocketHintBuilder(final St public static final class Inputs { private Inputs() { /* no op */ } - public static SocketHint createMatSocketHint(final String identifier, - final boolean withDefault) { - return createObjectSocketHintBuilder(identifier, Mat.class, Mat::new, withDefault).build(); - } - public static SocketHint createSizeSocketHint(final String identifier, final boolean withDefault) { return createObjectSocketHintBuilder(identifier, Size.class, Size::new, withDefault).build(); @@ -170,10 +173,6 @@ public static SocketHint createCheckboxSocketHint( public static final class Outputs { private Outputs() { /* no op */ } - public static SocketHint createMatSocketHint(final String identifier) { - return Inputs.createMatSocketHint(identifier, true); - } - public static SocketHint createPointSocketHint(final String identifier) { return Inputs.createPointSocketHint(identifier, true); } diff --git a/core/src/main/java/edu/wpi/grip/core/sockets/SocketImpl.java b/core/src/main/java/edu/wpi/grip/core/sockets/SocketImpl.java index 84b902b727..2e4e437b7f 100644 --- a/core/src/main/java/edu/wpi/grip/core/sockets/SocketImpl.java +++ b/core/src/main/java/edu/wpi/grip/core/sockets/SocketImpl.java @@ -62,6 +62,12 @@ public void setValueOptional(Optional optionalValue) { eventBus.post(new SocketChangedEvent(this)); } + @Override + public void flagChanged() { + onValueChanged(); + eventBus.post(new SocketChangedEvent(this)); + } + @Override public Optional getValue() { if (!this.value.isPresent()) { diff --git a/core/src/main/java/edu/wpi/grip/core/sources/CameraSource.java b/core/src/main/java/edu/wpi/grip/core/sources/CameraSource.java index 3e3a95bba6..f22da6a2df 100644 --- a/core/src/main/java/edu/wpi/grip/core/sources/CameraSource.java +++ b/core/src/main/java/edu/wpi/grip/core/sources/CameraSource.java @@ -1,6 +1,7 @@ package edu.wpi.grip.core.sources; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Source; import edu.wpi.grip.core.events.SourceHasPendingUpdateEvent; import edu.wpi.grip.core.events.SourceRemovedEvent; @@ -79,11 +80,10 @@ public class CameraSource extends Source implements RestartableService { private final Properties properties; - private final SocketHint imageOutputHint = SocketHints.Inputs.createMatSocketHint("Image", - true); + private final SocketHint imageOutputHint = SocketHints.createImageSocketHint("Image"); private final SocketHint frameRateOutputHint = SocketHints.createNumberSocketHint("Frame Rate", 0); - private final OutputSocket frameOutputSocket; + private final OutputSocket frameOutputSocket; private final OutputSocket frameRateOutputSocket; private final Supplier grabberSupplier; private final AtomicBoolean isNewFrame = new AtomicBoolean(false); @@ -240,9 +240,9 @@ protected boolean updateOutputSockets() { // The camera frame thread should not try to modify the transfer mat while it is being // written to the pipeline synchronized (currentFrameTransferMat) { - currentFrameTransferMat.copyTo(frameOutputSocket.getValue().get()); + frameOutputSocket.getValue().ifPresent(m -> m.set(currentFrameTransferMat)); } - frameOutputSocket.setValueOptional(frameOutputSocket.getValue()); + frameOutputSocket.flagChanged(); // Update the frame rate value frameRateOutputSocket.setValue(frameRate); diff --git a/core/src/main/java/edu/wpi/grip/core/sources/HttpSource.java b/core/src/main/java/edu/wpi/grip/core/sources/HttpSource.java index 3c36260cd5..c9663dc70a 100644 --- a/core/src/main/java/edu/wpi/grip/core/sources/HttpSource.java +++ b/core/src/main/java/edu/wpi/grip/core/sources/HttpSource.java @@ -1,6 +1,7 @@ package edu.wpi.grip.core.sources; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Source; import edu.wpi.grip.core.events.SourceHasPendingUpdateEvent; import edu.wpi.grip.core.events.SourceRemovedEvent; @@ -19,7 +20,6 @@ import com.thoughtworks.xstream.annotations.XStreamAlias; import org.bytedeco.javacpp.opencv_core.Mat; -import org.bytedeco.javacpp.opencv_imgcodecs; import java.util.HashMap; import java.util.List; @@ -27,6 +27,9 @@ import java.util.Properties; import java.util.function.Consumer; +import static org.bytedeco.javacpp.opencv_imgcodecs.CV_LOAD_IMAGE_COLOR; +import static org.bytedeco.javacpp.opencv_imgcodecs.imdecode; + /** * Provides a way to generate a {@link Mat Mat} from an image that has been POSTed to the * internal HTTP server. @@ -50,9 +53,9 @@ public class HttpSource extends Source { */ private final HttpImageHandler imageHandler; - private final OutputSocket imageOutput; - private final SocketHint outputHint = SocketHints.Outputs.createMatSocketHint("Image"); - private final Mat image = new Mat(); + private final OutputSocket imageOutput; + private final SocketHint outputHint = SocketHints.createImageSocketHint("Image"); + private final MatWrapper image = MatWrapper.emptyWrapper(); private final Consumer callback; private final EventBus eventBus; private String path; @@ -99,7 +102,7 @@ public interface Factory { } private void setImage(Mat image) { - image.copyTo(this.image); + this.image.set(image); eventBus.post(new SourceHasPendingUpdateEvent(this)); } @@ -121,7 +124,8 @@ protected boolean updateOutputSockets() { // No data, don't bother converting return false; } - imageOutput.setValue(opencv_imgcodecs.imdecode(image, opencv_imgcodecs.CV_LOAD_IMAGE_COLOR)); + imageOutput.getValue().get().set(imdecode(image.getCpu(), CV_LOAD_IMAGE_COLOR)); + imageOutput.flagChanged(); return true; } diff --git a/core/src/main/java/edu/wpi/grip/core/sources/ImageFileSource.java b/core/src/main/java/edu/wpi/grip/core/sources/ImageFileSource.java index 94fd3989ec..db3808d5f4 100644 --- a/core/src/main/java/edu/wpi/grip/core/sources/ImageFileSource.java +++ b/core/src/main/java/edu/wpi/grip/core/sources/ImageFileSource.java @@ -1,5 +1,6 @@ package edu.wpi.grip.core.sources; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Source; import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.core.sockets.SocketHint; @@ -36,8 +37,8 @@ public final class ImageFileSource extends Source { private final String name; private final String path; - private final SocketHint imageOutputHint = SocketHints.Outputs.createMatSocketHint("Image"); - private final OutputSocket outputSocket; + private final SocketHint imageOutputHint = SocketHints.createImageSocketHint("Image"); + private final OutputSocket outputSocket; /** * @param exceptionWitnessFactory Factory to create the exceptionWitness @@ -116,8 +117,8 @@ private void loadImage(String path) throws IOException { } private void loadImage(String path, final int flags) throws IOException { - ImageLoadingUtility.loadImage(path, flags, this.outputSocket.getValue().get()); - this.outputSocket.setValue(this.outputSocket.getValue().get()); + ImageLoadingUtility.loadImage(path, flags, this.outputSocket.getValue().get().rawCpu()); + this.outputSocket.flagChanged(); } diff --git a/core/src/main/java/edu/wpi/grip/core/sources/MultiImageFileSource.java b/core/src/main/java/edu/wpi/grip/core/sources/MultiImageFileSource.java index 0c51b9cc2e..9553f60cd3 100644 --- a/core/src/main/java/edu/wpi/grip/core/sources/MultiImageFileSource.java +++ b/core/src/main/java/edu/wpi/grip/core/sources/MultiImageFileSource.java @@ -1,5 +1,6 @@ package edu.wpi.grip.core.sources; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.PreviousNext; import edu.wpi.grip.core.Source; import edu.wpi.grip.core.events.SourceHasPendingUpdateEvent; @@ -38,9 +39,8 @@ public final class MultiImageFileSource extends Source implements PreviousNext { private static final String INDEX_PROPERTY = "index"; private static final String SIZE_PROPERTY = "numImages"; - private final SocketHint imageOutputHint = SocketHints.Inputs.createMatSocketHint("Image", - true); - private final OutputSocket outputSocket; + private final SocketHint imageOutputHint = SocketHints.createImageSocketHint("Image"); + private final OutputSocket outputSocket; private final EventBus eventBus; private final List paths; @@ -116,7 +116,9 @@ private static String getPathProperty(int index) { * paths. * * @param paths The paths of all of the images. + * * @return The list of Mats loaded from the file system. + * * @throws IOException if one of the images fails to load */ private static Mat[] createImagesArray(List paths) throws IOException { @@ -167,8 +169,11 @@ protected List createOutputSockets() { @Override protected boolean updateOutputSockets() { - if (!currentImage.equals(outputSocket.getValue())) { - outputSocket.setValueOptional(currentImage); + if (outputSocket.getValue() + .map(m -> m.getCpu()) + .map(m -> !currentImage.get().equals(m)) + .orElse(false)) { + outputSocket.getValue().ifPresent(m -> currentImage.ifPresent(m::set)); return true; } else { return false; @@ -192,6 +197,7 @@ public Properties getProperties() { * remain within the bounds of the image array. * * @param delta the value to add to the index when getting the image + * * @return The matrix at the given index in the array. */ private Mat addIndexAndGetImageByOffset(final int delta) { diff --git a/core/src/main/java/edu/wpi/grip/core/sources/VideoFileSource.java b/core/src/main/java/edu/wpi/grip/core/sources/VideoFileSource.java index 0d6375ce66..4b7321aa0c 100644 --- a/core/src/main/java/edu/wpi/grip/core/sources/VideoFileSource.java +++ b/core/src/main/java/edu/wpi/grip/core/sources/VideoFileSource.java @@ -1,5 +1,6 @@ package edu.wpi.grip.core.sources; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Source; import edu.wpi.grip.core.events.SourceHasPendingUpdateEvent; import edu.wpi.grip.core.events.SourceRemovedEvent; @@ -41,9 +42,9 @@ public class VideoFileSource extends Source implements Pausable { private final String path; - private final SocketHint imageHint = SocketHints.Outputs.createMatSocketHint("Image"); + private final SocketHint imageHint = SocketHints.createImageSocketHint("Image"); private final SocketHint fpsHint = SocketHints.Outputs.createNumberSocketHint("FPS", 0); - private final OutputSocket imageSocket; + private final OutputSocket imageSocket; private final OutputSocket fpsSocket; private final Mat workingMat = new Mat(); private final AtomicBoolean isNewFrame = new AtomicBoolean(false); @@ -102,7 +103,7 @@ protected boolean updateOutputSockets() { if (isNewFrame.compareAndSet(true, false)) { // New frame, update outputs synchronized (workingMat) { - workingMat.copyTo(imageSocket.getValue().get()); + workingMat.copyTo(imageSocket.getValue().get().rawCpu()); } imageSocket.setValue(imageSocket.getValue().get()); // force the socket to update return true; diff --git a/core/src/main/java/edu/wpi/grip/core/util/MetaInfReader.java b/core/src/main/java/edu/wpi/grip/core/util/MetaInfReader.java index 85755b5b7c..333df2cbf9 100644 --- a/core/src/main/java/edu/wpi/grip/core/util/MetaInfReader.java +++ b/core/src/main/java/edu/wpi/grip/core/util/MetaInfReader.java @@ -31,7 +31,7 @@ public final class MetaInfReader { * @throws IOException if the file could not be read */ public static Stream> readClasses(String fileName) throws IOException { - return read(fileName) + return readLines(fileName) .map(MetaInfReader::classForNameOrNull) .map(c -> (Class) c) .filter(Objects::nonNull); @@ -45,15 +45,15 @@ public static Stream> readClasses(String fileName) throws IOExcepti * @throws IOException if no file exists with the given name, or if the file exists but cannot * be read * - * @see #read(InputStream) + * @see #readLines(InputStream) */ - private static Stream read(String fileName) throws IOException { + public static Stream readLines(String fileName) throws IOException { checkNotNull(fileName, "fileName"); InputStream stream = MetaInfReader.class.getResourceAsStream("/META-INF/" + fileName); if (stream == null) { throw new IOException("No resource /META-INF/" + fileName + " found"); } - return read(stream); + return readLines(stream); } /** @@ -68,7 +68,7 @@ private static Stream read(String fileName) throws IOException { * @throws IOException if the stream could not be read from or safely closed */ @VisibleForTesting - static Stream read(InputStream inputStream) throws IOException { + static Stream readLines(InputStream inputStream) throws IOException { List lines; try (InputStreamReader reader = new InputStreamReader(inputStream, StandardCharsets.UTF_8); BufferedReader bufferedReader = new BufferedReader(reader)) { diff --git a/core/src/main/java/edu/wpi/grip/core/util/SafeShutdown.java b/core/src/main/java/edu/wpi/grip/core/util/SafeShutdown.java index 3aa326aeb5..d1b063e2b9 100644 --- a/core/src/main/java/edu/wpi/grip/core/util/SafeShutdown.java +++ b/core/src/main/java/edu/wpi/grip/core/util/SafeShutdown.java @@ -14,6 +14,40 @@ public final class SafeShutdown { public static volatile boolean stopping = false; + /** + * Exit codes used by the GRIP application. + */ + public enum ExitCode { + /** + * Clean shutdown. + */ + SAFE_SHUTDOWN(0x00), + /** + * An unknown exception was thrown and uncaught. + */ + MISC_ERROR(0x01), + + /** + * The HTTP server cannot start (typically due to the port already being in use). + */ + HTTP_SERVER_COULD_NOT_START(0x02), + /** + * CUDA is required by OpenCV but no compatible runtime is available on the system. + */ + CUDA_UNAVAILABLE(0x04), + ; + + private final int code; + + ExitCode(int code) { + this.code = code; + } + + public int getCode() { + return code; + } + } + static { /* * Shutdown hook run order is non-deterministic but this increases our likelihood of @@ -40,7 +74,7 @@ public void run() { * has been flagged true. This is nullable. * @see System#exit(int) */ - public static void exit(int statusCode, @Nullable PreSystemExitHook hook) { + public static void exit(ExitCode statusCode, @Nullable PreSystemExitHook hook) { flagStopping(); try { if (hook != null) { @@ -48,7 +82,7 @@ public static void exit(int statusCode, @Nullable PreSystemExitHook hook) { } } finally { Logger.getLogger(SafeShutdown.class.getName()).info("Exiting GRIP"); - System.exit(statusCode); + System.exit(statusCode.getCode()); } } @@ -57,9 +91,9 @@ public static void exit(int statusCode, @Nullable PreSystemExitHook hook) { * Helper method that passes null as the PreSystemExitHook. * * @param statusCode exit status. - * @see #exit(int) + * @see #exit(ExitCode, PreSystemExitHook) */ - public static void exit(int statusCode) { + public static void exit(ExitCode statusCode) { exit(statusCode, null); } diff --git a/core/src/test/java/edu/wpi/grip/core/AddOperation.java b/core/src/test/java/edu/wpi/grip/core/AddOperation.java index e86c453294..7d1775ae69 100644 --- a/core/src/test/java/edu/wpi/grip/core/AddOperation.java +++ b/core/src/test/java/edu/wpi/grip/core/AddOperation.java @@ -11,7 +11,6 @@ import com.google.common.eventbus.EventBus; import org.bytedeco.javacpp.opencv_core; -import org.bytedeco.javacpp.opencv_core.Mat; import java.util.List; @@ -22,13 +21,13 @@ public class AddOperation implements Operation { public static final OperationDescription DESCRIPTION = OperationDescription .builder().name("OpenCV Add").summary("Compute the per-pixel sum of two images.").build(); - private final SocketHint aHint = SocketHints.Inputs.createMatSocketHint("a", false); - private final SocketHint bHint = SocketHints.Inputs.createMatSocketHint("b", false); - private final SocketHint sumHint = SocketHints.Inputs.createMatSocketHint("sum", true); + private final SocketHint aHint = SocketHints.createImageSocketHint("a"); + private final SocketHint bHint = SocketHints.createImageSocketHint("b"); + private final SocketHint sumHint = SocketHints.createImageSocketHint("sum"); - private InputSocket a; - private InputSocket b; - private OutputSocket sum; + private InputSocket a; + private InputSocket b; + private OutputSocket sum; public AddOperation(EventBus eventBus) { this(new MockInputSocketFactory(eventBus), new MockOutputSocketFactory(eventBus)); @@ -56,6 +55,8 @@ public List getOutputSockets() { @Override public void perform() { - opencv_core.add(a.getValue().get(), b.getValue().get(), sum.getValue().get()); + opencv_core.add(a.getValue().get().getCpu(), + b.getValue().get().getCpu(), + sum.getValue().get().rawCpu()); } } diff --git a/core/src/test/java/edu/wpi/grip/core/CoreSanityTest.java b/core/src/test/java/edu/wpi/grip/core/CoreSanityTest.java index fa4be39545..b52c95e84e 100644 --- a/core/src/test/java/edu/wpi/grip/core/CoreSanityTest.java +++ b/core/src/test/java/edu/wpi/grip/core/CoreSanityTest.java @@ -28,7 +28,8 @@ public CoreSanityTest() { ManualPipelineRunner.class, SubtractionOperation.class, Main.class, - CoreCommandLineHelper.class + CoreCommandLineHelper.class, + OperationDescription.class ).contains(c)); setDefault(OutputSocket.class, new MockOutputSocket("Mock Out")); setDefault(InputSocket.class, new MockInputSocket("Mock In")); diff --git a/core/src/test/java/edu/wpi/grip/core/PipelineRunnerTest.java b/core/src/test/java/edu/wpi/grip/core/PipelineRunnerTest.java index 49dd26f583..48ae07d2b6 100644 --- a/core/src/test/java/edu/wpi/grip/core/PipelineRunnerTest.java +++ b/core/src/test/java/edu/wpi/grip/core/PipelineRunnerTest.java @@ -19,6 +19,7 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import net.jodah.concurrentunit.Waiter; + import org.junit.After; import org.junit.Before; import org.junit.Test; diff --git a/core/src/test/java/edu/wpi/grip/core/cuda/CudaVerifierTest.java b/core/src/test/java/edu/wpi/grip/core/cuda/CudaVerifierTest.java new file mode 100644 index 0000000000..020bc26c2e --- /dev/null +++ b/core/src/test/java/edu/wpi/grip/core/cuda/CudaVerifierTest.java @@ -0,0 +1,51 @@ +package edu.wpi.grip.core.cuda; + +import org.junit.Test; + +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class CudaVerifierTest { + + @Test + public void testVerifyNonCudaRuntime() { + CudaVerifier verifier = new NonExitingVerifier(() -> false, () -> false); + assertTrue("Not using CUDA should always pass verification", verifier.verify()); + } + + @Test + public void testUsingCudaWithNoAvailableRuntime() { + CudaVerifier verifier = new NonExitingVerifier(() -> true, () -> false); + assertFalse("Using CUDA but no available runtime should fail verification", verifier.verify()); + } + + @Test + public void testExitsWhenFailsVerification() { + NonExitingVerifier verifier = new NonExitingVerifier(() -> true, () -> false); + + verifier.verifyCuda(); + assertEquals(1, verifier.getExitCount()); + } + + private static final class NonExitingVerifier extends CudaVerifier { + + private int exitCount = 0; + + public NonExitingVerifier(AccelerationMode accelerationMode, CudaDetector cudaDetector) { + super(accelerationMode, cudaDetector, new Properties()); + } + + @Override + void exit() { + exitCount++; + } + + public int getExitCount() { + return exitCount; + } + } + +} diff --git a/core/src/test/java/edu/wpi/grip/core/operations/OperationsUtil.java b/core/src/test/java/edu/wpi/grip/core/operations/OperationsUtil.java index 8920b3cbdc..6923b5b802 100644 --- a/core/src/test/java/edu/wpi/grip/core/operations/OperationsUtil.java +++ b/core/src/test/java/edu/wpi/grip/core/operations/OperationsUtil.java @@ -13,9 +13,14 @@ public class OperationsUtil { @Inject private CVOperations cvOperations; + @Inject + private Operations operations; public ImmutableList operations() { - return cvOperations.operations(); + return ImmutableList.builder() + .addAll(cvOperations.operations()) + .addAll(operations.operations()) + .build(); } public OperationMetaData getMetaData(String opName) { diff --git a/core/src/test/java/edu/wpi/grip/core/operations/opencv/AddOperationTest.java b/core/src/test/java/edu/wpi/grip/core/operations/opencv/AddOperationTest.java index 4946278eaf..3b89f0ccbd 100644 --- a/core/src/test/java/edu/wpi/grip/core/operations/opencv/AddOperationTest.java +++ b/core/src/test/java/edu/wpi/grip/core/operations/opencv/AddOperationTest.java @@ -1,6 +1,7 @@ package edu.wpi.grip.core.operations.opencv; import edu.wpi.grip.core.AddOperation; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -55,20 +56,20 @@ public void testAddMatrixOfOnesToMatrixOfTwosEqualsMatrixOfThrees() { // Given List inputs = addition.getInputSockets(); List outputs = addition.getOutputSockets(); - InputSocket a = inputs.get(0); - InputSocket b = inputs.get(1); - OutputSocket c = outputs.get(0); + InputSocket a = inputs.get(0); + InputSocket b = inputs.get(1); + OutputSocket c = outputs.get(0); int[] sz = {256, 256}; - a.setValue(new Mat(2, sz, opencv_core.CV_8U, Scalar.all(1))); - b.setValue(new Mat(2, sz, opencv_core.CV_8U, Scalar.all(2))); + a.getValue().ifPresent(m -> m.set(new Mat(2, sz, opencv_core.CV_8U, Scalar.all(1)))); + b.getValue().ifPresent(m -> m.set(new Mat(2, sz, opencv_core.CV_8U, Scalar.all(2)))); for (int i = 0; i < 1000; i++) { addition.perform(); } Mat expectedResult = new Mat(2, sz, opencv_core.CV_8U, Scalar.all(3)); - assertTrue(isMatEqual((Mat) c.getValue().get(), expectedResult)); + assertTrue(isMatEqual(c.getValue().get().getCpu(), expectedResult)); } } diff --git a/core/src/test/java/edu/wpi/grip/core/serialization/ProjectTest.java b/core/src/test/java/edu/wpi/grip/core/serialization/ProjectTest.java index b119685574..01313abe33 100644 --- a/core/src/test/java/edu/wpi/grip/core/serialization/ProjectTest.java +++ b/core/src/test/java/edu/wpi/grip/core/serialization/ProjectTest.java @@ -4,6 +4,7 @@ import edu.wpi.grip.core.AdditionOperation; import edu.wpi.grip.core.Connection; import edu.wpi.grip.core.ManualPipelineRunner; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.OperationMetaData; import edu.wpi.grip.core.Pipeline; import edu.wpi.grip.core.PipelineRunner; @@ -274,19 +275,20 @@ public void testPerformSerializedPipelineWithMats() throws Exception { serializeAndDeserialize(); Step step1 = pipeline.getSteps().get(0); - InputSocket a = (InputSocket) step1.getInputSockets().get(0); - InputSocket b = (InputSocket) step1.getInputSockets().get(1); - OutputSocket sum = (OutputSocket) step1.getOutputSockets().get(0); + InputSocket a = (InputSocket) step1.getInputSockets().get(0); + InputSocket b = (InputSocket) step1.getInputSockets().get(1); + OutputSocket sum = (OutputSocket) step1.getOutputSockets().get(0); - a.setValue(new Mat(1, 1, CV_32F, new Scalar(1234.5))); - b.setValue(new Mat(1, 1, CV_32F, new Scalar(6789.0))); + a.getValue().get().set(new Mat(1, 1, CV_32F, new Scalar(1234.5))); + b.getValue().get().set(new Mat(1, 1, CV_32F, new Scalar(6789.0))); + a.flagChanged(); pipelineRunner.runPipeline(); Mat diff = new Mat(); Mat expected = new Mat(1, 1, CV_32F, new Scalar(1234.5 + 6789.0)); - compare(expected, sum.getValue().get(), diff, CMP_NE); + compare(expected, sum.getValue().get().getCpu(), diff, CMP_NE); assertEquals("Deserialized pipeline with Mat operations did not produce the expected sum.", 0, countNonZero(diff)); } @@ -301,8 +303,8 @@ public void testSerializePipelineWithSource() throws Exception { serializeAndDeserialize(); final ImageFileSource sourceDeserialized = (ImageFileSource) pipeline.getSources().get(0); - Files.gompeiJpegFile.assertSameImage((Mat) sourceDeserialized.createOutputSockets().get(0) - .getValue().get()); + Files.gompeiJpegFile.assertSameImage( + (MatWrapper) sourceDeserialized.createOutputSockets().get(0).getValue().get()); } @Test diff --git a/core/src/test/java/edu/wpi/grip/core/sockets/MockInputSocketFactory.java b/core/src/test/java/edu/wpi/grip/core/sockets/MockInputSocketFactory.java index bd3f0931a6..1d0b0cbe24 100644 --- a/core/src/test/java/edu/wpi/grip/core/sockets/MockInputSocketFactory.java +++ b/core/src/test/java/edu/wpi/grip/core/sockets/MockInputSocketFactory.java @@ -1,11 +1,13 @@ package edu.wpi.grip.core.sockets; +import edu.wpi.grip.core.cuda.NullCudaDetector; + import com.google.common.eventbus.EventBus; public class MockInputSocketFactory extends InputSocketImpl.FactoryImpl { public MockInputSocketFactory(EventBus eventBus) { - super(eventBus); + super(eventBus, new NullCudaDetector()); } } diff --git a/core/src/test/java/edu/wpi/grip/core/sources/CameraSourceTest.java b/core/src/test/java/edu/wpi/grip/core/sources/CameraSourceTest.java index 29447211f4..2eaac028cc 100644 --- a/core/src/test/java/edu/wpi/grip/core/sources/CameraSourceTest.java +++ b/core/src/test/java/edu/wpi/grip/core/sources/CameraSourceTest.java @@ -281,7 +281,7 @@ static class MockFrameGrabber extends FrameGrabber { for (int y = 0; y < frameIdx.rows(); y++) { for (int x = 0; x < frameIdx.cols(); x++) { for (int z = 0; z < frameIdx.channels(); z++) { - frameIdx.putDouble(new int[]{y, x, z}, y + x + z); + frameIdx.putDouble(new long[]{y, x, z}, y + x + z); } } } diff --git a/core/src/test/java/edu/wpi/grip/core/sources/HttpSourceTest.java b/core/src/test/java/edu/wpi/grip/core/sources/HttpSourceTest.java index 8a81abc172..159c256124 100644 --- a/core/src/test/java/edu/wpi/grip/core/sources/HttpSourceTest.java +++ b/core/src/test/java/edu/wpi/grip/core/sources/HttpSourceTest.java @@ -1,5 +1,6 @@ package edu.wpi.grip.core.sources; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Pipeline; import edu.wpi.grip.core.http.ContextStore; import edu.wpi.grip.core.http.GripServer; @@ -17,7 +18,6 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; -import org.bytedeco.javacpp.opencv_core.Mat; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -62,7 +62,7 @@ public void setUp() throws URISyntaxException { @Test public void testPostImage() throws IOException, InterruptedException { - OutputSocket imageSource = source.getOutputSockets().get(0); + OutputSocket imageSource = source.getOutputSockets().get(0); // We have to manually update the output sockets to get the image source.updateOutputSockets(); diff --git a/core/src/test/java/edu/wpi/grip/core/sources/ImageFileSourceTest.java b/core/src/test/java/edu/wpi/grip/core/sources/ImageFileSourceTest.java index 7d87e6b8e7..77d8840607 100644 --- a/core/src/test/java/edu/wpi/grip/core/sources/ImageFileSourceTest.java +++ b/core/src/test/java/edu/wpi/grip/core/sources/ImageFileSourceTest.java @@ -1,5 +1,6 @@ package edu.wpi.grip.core.sources; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.sockets.MockOutputSocketFactory; import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.util.Files; @@ -7,7 +8,6 @@ import com.google.common.eventbus.EventBus; -import org.bytedeco.javacpp.opencv_core.Mat; import org.junit.Before; import org.junit.Test; @@ -40,7 +40,7 @@ public void testLoadImageToMat() throws IOException { final ImageFileSource fileSource = new ImageFileSource(osf, origin -> null, this .imageFile.file); fileSource.initialize(); - OutputSocket outputSocket = fileSource.getOutputSockets().get(0); + OutputSocket outputSocket = fileSource.getOutputSockets().get(0); // Then assertTrue("The output socket's value was empty.", outputSocket.getValue().isPresent()); @@ -53,7 +53,7 @@ public void testReadInTextFile() throws IOException { final ImageFileSource fileSource = new ImageFileSource(osf, origin -> null, this .textFile); fileSource.initialize(); - OutputSocket outputSocket = fileSource.getOutputSockets().get(0); + OutputSocket outputSocket = fileSource.getOutputSockets().get(0); assertTrue("No matrix should have been returned.", outputSocket.getValue().get().empty()); } @@ -71,7 +71,7 @@ public void testCallingInitializeAfterGetOutputSocketUpdatesOutputSocket() throw final ImageFileSource source = new ImageFileSource(osf, origin -> null, this .imageFile.file); // Calling this before loading the image should throw an exception - final OutputSocket imageSource = source.getOutputSockets().get(0); + final OutputSocket imageSource = source.getOutputSockets().get(0); assertTrue("The value should not be present if the source hasn't been initialized", imageSource.getValue().get().empty()); source.initialize(); diff --git a/core/src/test/java/edu/wpi/grip/core/sources/MultiImageFileSourceTest.java b/core/src/test/java/edu/wpi/grip/core/sources/MultiImageFileSourceTest.java index 914e7e1805..8aee0d99f5 100644 --- a/core/src/test/java/edu/wpi/grip/core/sources/MultiImageFileSourceTest.java +++ b/core/src/test/java/edu/wpi/grip/core/sources/MultiImageFileSourceTest.java @@ -1,5 +1,6 @@ package edu.wpi.grip.core.sources; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.sockets.MockOutputSocketFactory; import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.util.Files; @@ -7,7 +8,6 @@ import com.google.common.eventbus.EventBus; -import org.bytedeco.javacpp.opencv_core.Mat; import org.junit.Before; import org.junit.Test; @@ -54,7 +54,7 @@ public void createMultiImageFileSourceWithTextFile() throws IOException { @Test public void testNextValue() throws Exception { source.next(); - OutputSocket outputSocket = source.getOutputSockets().get(0); + OutputSocket outputSocket = source.getOutputSockets().get(0); source.updateOutputSockets(); gompeiJpegFile.assertSameImage(outputSocket.getValue().get()); } @@ -63,7 +63,7 @@ public void testNextValue() throws Exception { public void testPreviousValue() throws Exception { source.previous(); - OutputSocket outputSocket = source.getOutputSockets().get(0); + OutputSocket outputSocket = source.getOutputSockets().get(0); source.updateOutputSockets(); gompeiJpegFile.assertSameImage(outputSocket.getValue().get()); } @@ -71,7 +71,7 @@ public void testPreviousValue() throws Exception { @Test public void testConstructedWithIndex() { sourceWithIndexSet.updateOutputSockets(); - OutputSocket outputSocket = sourceWithIndexSet.getOutputSockets().get(0); + OutputSocket outputSocket = sourceWithIndexSet.getOutputSockets().get(0); gompeiJpegFile.assertSameImage(outputSocket.getValue().get()); } @@ -85,7 +85,7 @@ public void testLoadFromProperties() throws Exception { properties); newSource.initialize(); newSource.updateOutputSockets(); - OutputSocket outputSocket = newSource.getOutputSockets().get(0); + OutputSocket outputSocket = newSource.getOutputSockets().get(0); gompeiJpegFile.assertSameImage(outputSocket.getValue().get()); } } diff --git a/core/src/test/java/edu/wpi/grip/core/util/MetaInfReaderTest.java b/core/src/test/java/edu/wpi/grip/core/util/MetaInfReaderTest.java index e5513cabf2..a49affe19f 100644 --- a/core/src/test/java/edu/wpi/grip/core/util/MetaInfReaderTest.java +++ b/core/src/test/java/edu/wpi/grip/core/util/MetaInfReaderTest.java @@ -34,7 +34,7 @@ public void testRead() throws IOException { String input = "# Leading comment\na\nb\n# Comment in the middle\n\nc\nd\ne # Trailing comment"; List expected = ImmutableList.of("a", "b", "c", "d", "e"); InputStream s = new StringBufferInputStream(input); - List out = MetaInfReader.read(s) + List out = MetaInfReader.readLines(s) .collect(Collectors.toList()); assertEquals("Unexpected line result", expected, out); } diff --git a/core/src/test/java/edu/wpi/grip/core/util/SafeShutdownTest.java b/core/src/test/java/edu/wpi/grip/core/util/SafeShutdownTest.java index d8b196009c..0e3aed0af4 100644 --- a/core/src/test/java/edu/wpi/grip/core/util/SafeShutdownTest.java +++ b/core/src/test/java/edu/wpi/grip/core/util/SafeShutdownTest.java @@ -41,7 +41,7 @@ public void tearDown() { @Test public void testSafeShutdownShutsDownIfHandlerThrowsError() throws Exception { try { - SafeShutdown.exit(0, () -> { + SafeShutdown.exit(SafeShutdown.ExitCode.SAFE_SHUTDOWN, () -> { throw new AssertionError("This should not be the exception that appears"); }); } catch (IllegalStateException e) { diff --git a/core/src/test/java/edu/wpi/grip/util/ImageWithData.java b/core/src/test/java/edu/wpi/grip/util/ImageWithData.java index 261a387d62..40b66ff3ef 100644 --- a/core/src/test/java/edu/wpi/grip/util/ImageWithData.java +++ b/core/src/test/java/edu/wpi/grip/util/ImageWithData.java @@ -1,6 +1,7 @@ package edu.wpi.grip.util; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.util.ImageLoadingUtility; import org.bytedeco.javacpp.opencv_core.Mat; @@ -35,6 +36,10 @@ public Mat createMat() { } } + public void assertSameImage(MatWrapper image) { + assertSameImage(image.getCpu()); + } + public void assertSameImage(final Mat image) { // Check that the image that is read in is 2 dimensional assertEquals("Matrix from loaded image did not have expected number of rows.", this.rows, diff --git a/ui/src/main/java/edu/wpi/grip/ui/DeployController.java b/ui/src/main/java/edu/wpi/grip/ui/DeployController.java index 6c0401a8be..387fd4bed3 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/DeployController.java +++ b/ui/src/main/java/edu/wpi/grip/ui/DeployController.java @@ -36,6 +36,7 @@ import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; + import javafx.application.Platform; import javafx.beans.binding.Bindings; import javafx.beans.property.BooleanProperty; @@ -45,6 +46,7 @@ import javafx.scene.control.ProgressIndicator; import javafx.scene.control.TextField; import javafx.scene.control.ToggleButton; + import javax.inject.Inject; /** diff --git a/ui/src/main/java/edu/wpi/grip/ui/Main.java b/ui/src/main/java/edu/wpi/grip/ui/Main.java index b797176548..2975d965cb 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/Main.java +++ b/ui/src/main/java/edu/wpi/grip/ui/Main.java @@ -1,8 +1,11 @@ package edu.wpi.grip.ui; import edu.wpi.grip.core.GripCoreModule; +import edu.wpi.grip.core.GripCudaModule; import edu.wpi.grip.core.GripFileModule; +import edu.wpi.grip.core.Loggers; import edu.wpi.grip.core.PipelineRunner; +import edu.wpi.grip.core.cuda.CudaVerifier; import edu.wpi.grip.core.events.UnexpectedThrowableEvent; import edu.wpi.grip.core.exception.GripServerException; import edu.wpi.grip.core.http.GripServer; @@ -76,22 +79,37 @@ public static void main(String[] args) { @Override public void init() throws IOException { + Loggers.setupLoggers(); parsedArgs = commandLineHelper.parse(getParameters().getRaw()); + // Verify CUDA before using the core module, since that will cause OpenCV to be loaded, + // which will crash the app if we use CUDA and it's not available + GripCudaModule cudaModule = new GripCudaModule(); + CudaVerifier cudaVerifier = Guice.createInjector(cudaModule).getInstance(CudaVerifier.class); + cudaVerifier.verifyCuda(); + if (parsedArgs.hasOption(UICommandLineHelper.HEADLESS_OPTION)) { // If --headless was specified on the command line, // run in headless mode (only use the core module) logger.info("Launching GRIP in headless mode"); - injector = Guice.createInjector(Modules.override(new GripCoreModule(), new GripFileModule(), - new GripSourcesHardwareModule()).with(new GripNetworkModule())); + injector = Guice.createInjector( + Modules.override( + new GripCoreModule(), + new GripFileModule(), + new GripSourcesHardwareModule() + ).with(new GripNetworkModule(), cudaModule)); injector.injectMembers(this); headless = true; } else { // Otherwise, run with both the core and UI modules, and show the JavaFX stage logger.info("Launching GRIP in UI mode"); - injector = Guice.createInjector(Modules.override(new GripCoreModule(), new GripFileModule(), - new GripSourcesHardwareModule()).with(new GripNetworkModule(), new GripUiModule())); + injector = Guice.createInjector( + Modules.override( + new GripCoreModule(), + new GripFileModule(), + new GripSourcesHardwareModule() + ).with(new GripNetworkModule(), new GripUiModule(), cudaModule)); injector.injectMembers(this); notifyPreloader(new Preloader.ProgressNotification(0.15)); @@ -160,7 +178,9 @@ public void start(Stage stage) throws IOException { + "HTTP sources and operations will not work until GRIP is restarted. " + "Continue without HTTP functionality anyway?" ); - alert.showAndWait().filter(ButtonType.NO::equals).ifPresent(bt -> SafeShutdown.exit(1)); + alert.showAndWait() + .filter(ButtonType.NO::equals) + .ifPresent(bt -> SafeShutdown.exit(SafeShutdown.ExitCode.HTTP_SERVER_COULD_NOT_START)); } } @@ -195,8 +215,8 @@ public final void onUnexpectedThrowableEvent(UnexpectedThrowableEvent event) { try { logger.log(Level.SEVERE, "Failed to show exception alert", e); } finally { - SafeShutdown.exit(1); // Ensure we shut down the application if we get an - // exception + // Ensure we shut down the application if we get an exception + SafeShutdown.exit(SafeShutdown.ExitCode.MISC_ERROR); } } } diff --git a/ui/src/main/java/edu/wpi/grip/ui/MainWindowController.java b/ui/src/main/java/edu/wpi/grip/ui/MainWindowController.java index ed5a1bedd7..00eb1ea063 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/MainWindowController.java +++ b/ui/src/main/java/edu/wpi/grip/ui/MainWindowController.java @@ -279,7 +279,7 @@ protected void showProjectAboutDialog() throws IOException { protected boolean quit() { if (showConfirmationDialogAndWait()) { pipelineRunner.stopAsync(); - SafeShutdown.exit(0); + SafeShutdown.exit(SafeShutdown.ExitCode.SAFE_SHUTDOWN); return true; } return false; diff --git a/ui/src/main/java/edu/wpi/grip/ui/codegeneration/data/TPipeline.java b/ui/src/main/java/edu/wpi/grip/ui/codegeneration/data/TPipeline.java index 176a63006b..c504e3b539 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/codegeneration/data/TPipeline.java +++ b/ui/src/main/java/edu/wpi/grip/ui/codegeneration/data/TPipeline.java @@ -2,6 +2,7 @@ import edu.wpi.grip.core.Connection; import edu.wpi.grip.core.Step; +import edu.wpi.grip.core.sockets.CudaSocket; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.ui.codegeneration.TemplateMethods; @@ -97,6 +98,10 @@ private void set(List pipeSteps) { for (int i = 0; i < pipeSteps.size(); i++) { TStep tStep = this.steps.get(i); for (InputSocket input : pipeSteps.get(i).getInputSockets()) { + // Skip CudaSockets entirely - generated code is purely CPU + if (input instanceof CudaSocket) { + continue; + } TInput tInput; String type = TemplateMethods.parseSocketType(input); if ("Type".equals(type)) { diff --git a/ui/src/main/java/edu/wpi/grip/ui/components/PreviousNextButtons.java b/ui/src/main/java/edu/wpi/grip/ui/components/PreviousNextButtons.java index 23a3c170dc..d1cd86fbca 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/components/PreviousNextButtons.java +++ b/ui/src/main/java/edu/wpi/grip/ui/components/PreviousNextButtons.java @@ -6,6 +6,7 @@ import org.controlsfx.control.SegmentedButton; import java.util.function.Consumer; + import javafx.scene.Node; import javafx.scene.control.ToggleButton; import javafx.scene.control.Tooltip; diff --git a/ui/src/main/java/edu/wpi/grip/ui/pipeline/OutputSocketController.java b/ui/src/main/java/edu/wpi/grip/ui/pipeline/OutputSocketController.java index f5c5cce7fd..56f3315fd6 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/pipeline/OutputSocketController.java +++ b/ui/src/main/java/edu/wpi/grip/ui/pipeline/OutputSocketController.java @@ -1,5 +1,6 @@ package edu.wpi.grip.ui.pipeline; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.events.BenchmarkEvent; import edu.wpi.grip.core.events.SocketChangedEvent; import edu.wpi.grip.core.events.SocketPreviewChangedEvent; @@ -25,7 +26,6 @@ import javafx.scene.layout.StackPane; import static com.google.common.base.Preconditions.checkNotNull; -import static org.bytedeco.javacpp.opencv_core.Mat; /** * A JavaFX control that renders an {@link OutputSocket} that is the output of a step. It shows a @@ -96,13 +96,13 @@ public void onSocketChanged(SocketChangedEvent event) { if (!this.socket.getValue().isPresent()) { // No value handlePreview(false); - } else if (!(this.socket.getValue().get() instanceof Mat)) { + } else if (!(this.socket.getValue().get() instanceof MatWrapper)) { // There is a non-image value, which can always be previewed handlePreview(true); } else { // Only allow the image to be previewed if it's previewable boolean previewable = this.socket.getValue() - .map(Mat.class::cast) + .map(MatWrapper.class::cast) .map(ImageBasedPreviewView::isPreviewable) .get(); handlePreview(previewable); diff --git a/ui/src/main/java/edu/wpi/grip/ui/pipeline/PipelineController.java b/ui/src/main/java/edu/wpi/grip/ui/pipeline/PipelineController.java index 09f83e9710..d2bdd99d2e 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/pipeline/PipelineController.java +++ b/ui/src/main/java/edu/wpi/grip/ui/pipeline/PipelineController.java @@ -31,6 +31,7 @@ import java.util.NavigableMap; import java.util.TreeMap; import java.util.stream.Collectors; + import javafx.beans.InvalidationListener; import javafx.beans.property.ReadOnlyObjectProperty; import javafx.collections.ObservableList; @@ -43,6 +44,7 @@ import javafx.scene.input.TransferMode; import javafx.scene.layout.HBox; import javafx.scene.layout.VBox; + import javax.annotation.Nullable; import javax.inject.Inject; diff --git a/ui/src/main/java/edu/wpi/grip/ui/pipeline/input/CheckboxInputSocketController.java b/ui/src/main/java/edu/wpi/grip/ui/pipeline/input/CheckboxInputSocketController.java index 7329810a45..65cdd9d76e 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/pipeline/input/CheckboxInputSocketController.java +++ b/ui/src/main/java/edu/wpi/grip/ui/pipeline/input/CheckboxInputSocketController.java @@ -1,6 +1,9 @@ package edu.wpi.grip.ui.pipeline.input; +import edu.wpi.grip.core.cuda.AccelerationMode; +import edu.wpi.grip.core.cuda.CudaDetector; import edu.wpi.grip.core.events.SocketChangedEvent; +import edu.wpi.grip.core.sockets.CudaSocket; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.ui.pipeline.SocketHandleView; import edu.wpi.grip.ui.util.GripPlatform; @@ -23,12 +26,17 @@ public class CheckboxInputSocketController extends InputSocketController socket) { + CheckboxInputSocketController(SocketHandleView.Factory socketHandleViewFactory, + GripPlatform platform, + AccelerationMode accelerationMode, + CudaDetector cudaDetector, + @Assisted InputSocket socket) { super(socketHandleViewFactory, socket); this.platform = platform; + isCudaAvailable = accelerationMode.isUsingCuda() && cudaDetector.isCompatibleCudaInstalled(); this.checkBox = new CheckBox(); } @@ -42,6 +50,11 @@ public void initialize() { this.getIdentifier().setContentDisplay(ContentDisplay.RIGHT); this.setContent(checkBox); + // Disable if controlling a CUDA socket and CUDA acceleration is unavailable + if (getSocket() instanceof CudaSocket && !isCudaAvailable) { + this.checkBox.setDisable(true); + } + assignSocketValue(getSocket().getValue()); // Add the listener after so that setting the initial value doesn't trigger it. this.checkBox.selectedProperty().addListener(o -> this.getSocket().setValue(this.checkBox diff --git a/ui/src/main/java/edu/wpi/grip/ui/preview/BlobsSocketPreviewView.java b/ui/src/main/java/edu/wpi/grip/ui/preview/BlobsSocketPreviewView.java index 75b8ecaca2..00d060858b 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/preview/BlobsSocketPreviewView.java +++ b/ui/src/main/java/edu/wpi/grip/ui/preview/BlobsSocketPreviewView.java @@ -58,7 +58,7 @@ public BlobsSocketPreviewView(GripPlatform platform, OutputSocket s protected void convertImage() { synchronized (this) { final BlobsReport blobsReport = this.getSocket().getValue().get(); - final Mat input = blobsReport.getInput(); + final Mat input = blobsReport.getInput().getCpu(); if (input.channels() == 3) { input.copyTo(tmp); diff --git a/ui/src/main/java/edu/wpi/grip/ui/preview/ImageBasedPreviewView.java b/ui/src/main/java/edu/wpi/grip/ui/preview/ImageBasedPreviewView.java index 50a751baba..0531625ab2 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/preview/ImageBasedPreviewView.java +++ b/ui/src/main/java/edu/wpi/grip/ui/preview/ImageBasedPreviewView.java @@ -1,5 +1,6 @@ package edu.wpi.grip.ui.preview; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.events.RenderEvent; import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.ui.util.ImageConverter; @@ -11,7 +12,6 @@ import static org.bytedeco.javacpp.opencv_core.CV_8S; import static org.bytedeco.javacpp.opencv_core.CV_8U; -import static org.bytedeco.javacpp.opencv_core.Mat; /** * Base class for image previews. @@ -58,7 +58,7 @@ protected final int getImageHeight() { * * @return true if the image can be previewed, false if it can't */ - public static boolean isPreviewable(Mat image) { + public static boolean isPreviewable(MatWrapper image) { return (image.channels() == 1) || (image.channels() == 3) && (image.depth() == CV_8U || image.depth() == CV_8S); } diff --git a/ui/src/main/java/edu/wpi/grip/ui/preview/ImageSocketPreviewView.java b/ui/src/main/java/edu/wpi/grip/ui/preview/ImageSocketPreviewView.java index 647cae478e..fb59a3cae7 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/preview/ImageSocketPreviewView.java +++ b/ui/src/main/java/edu/wpi/grip/ui/preview/ImageSocketPreviewView.java @@ -1,23 +1,22 @@ package edu.wpi.grip.ui.preview; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.ui.util.GripPlatform; import javafx.scene.image.Image; -import static org.bytedeco.javacpp.opencv_core.Mat; - /** * A SocketPreviewView that previews sockets containing OpenCV Mats. */ -public class ImageSocketPreviewView extends ImageBasedPreviewView { +public class ImageSocketPreviewView extends ImageBasedPreviewView { private final GripPlatform platform; /** * @param socket An output socket to preview. */ - ImageSocketPreviewView(GripPlatform platform, OutputSocket socket) { + ImageSocketPreviewView(GripPlatform platform, OutputSocket socket) { super(socket); this.platform = platform; this.setContent(imageView); @@ -28,9 +27,9 @@ protected void convertImage() { synchronized (this) { this.getSocket().getValue() .filter(ImageBasedPreviewView::isPreviewable) - .ifPresent(mat -> { + .ifPresent(m -> { platform.runAsSoonAsPossible(() -> { - Image image = imageConverter.convert(mat, getImageHeight()); + Image image = imageConverter.convert(m.getCpu(), getImageHeight()); imageView.setImage(image); }); }); diff --git a/ui/src/main/java/edu/wpi/grip/ui/preview/LinesSocketPreviewView.java b/ui/src/main/java/edu/wpi/grip/ui/preview/LinesSocketPreviewView.java index 8b70e3db3a..603e05417e 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/preview/LinesSocketPreviewView.java +++ b/ui/src/main/java/edu/wpi/grip/ui/preview/LinesSocketPreviewView.java @@ -69,7 +69,7 @@ protected void convertImage() { synchronized (this) { final LinesReport linesReport = this.getSocket().getValue().get(); final List lines = linesReport.getLines(); - Mat input = linesReport.getInput(); + Mat input = linesReport.getInput().getCpu(); // If there were lines found, draw them on the image before displaying it if (!linesReport.getLines().isEmpty()) { diff --git a/ui/src/main/java/edu/wpi/grip/ui/preview/RectangleSocketPreviewView.java b/ui/src/main/java/edu/wpi/grip/ui/preview/RectangleSocketPreviewView.java index faf0677240..c10f20c496 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/preview/RectangleSocketPreviewView.java +++ b/ui/src/main/java/edu/wpi/grip/ui/preview/RectangleSocketPreviewView.java @@ -58,7 +58,7 @@ protected void convertImage() { synchronized (this) { final RectsReport report = this.getSocket().getValue().get(); final List rectangles = report.getRectangles(); - Mat input = report.getImage(); + Mat input = report.getImage().getCpu(); if (input.channels() == 3) { input.copyTo(tmp); diff --git a/ui/src/main/java/edu/wpi/grip/ui/preview/SocketPreviewViewFactory.java b/ui/src/main/java/edu/wpi/grip/ui/preview/SocketPreviewViewFactory.java index 2d7017e98f..38a6b677d7 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/preview/SocketPreviewViewFactory.java +++ b/ui/src/main/java/edu/wpi/grip/ui/preview/SocketPreviewViewFactory.java @@ -1,5 +1,6 @@ package edu.wpi.grip.ui.preview; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.operations.composite.BlobsReport; import edu.wpi.grip.core.operations.composite.ContoursReport; import edu.wpi.grip.core.operations.composite.LinesReport; @@ -11,7 +12,6 @@ import com.google.inject.Inject; import com.google.inject.Singleton; -import static org.bytedeco.javacpp.opencv_core.Mat; import static org.bytedeco.javacpp.opencv_core.Point; import static org.bytedeco.javacpp.opencv_core.Size; @@ -39,9 +39,9 @@ public class SocketPreviewViewFactory { @SuppressWarnings("unchecked") public SocketPreviewView create(OutputSocket socket) { final SocketPreviewView previewView; - if (socket.getSocketHint().getType() == Mat.class) { - previewView = (SocketPreviewView) new ImageSocketPreviewView(platform, (OutputSocket) - socket); + if (socket.getSocketHint().getType() == MatWrapper.class) { + previewView = (SocketPreviewView) new ImageSocketPreviewView(platform, + (OutputSocket) socket); } else if (socket.getSocketHint().getType() == Point.class || socket.getSocketHint().getType() == Size.class) { previewView = (SocketPreviewView) new PointSizeSocketPreviewView(platform, socket); diff --git a/ui/src/main/java/edu/wpi/grip/ui/util/ImageConverter.java b/ui/src/main/java/edu/wpi/grip/ui/util/ImageConverter.java index 736a31897e..c5e19d7f36 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/util/ImageConverter.java +++ b/ui/src/main/java/edu/wpi/grip/ui/util/ImageConverter.java @@ -1,5 +1,7 @@ package edu.wpi.grip.ui.util; +import edu.wpi.grip.core.MatWrapper; + import com.google.common.primitives.UnsignedBytes; import org.bytedeco.javacpp.opencv_core.Mat; @@ -25,6 +27,9 @@ public final class ImageConverter { private WritableImage image; private IntBuffer pixels; + public Image convert(MatWrapper wrapper, int desiredHeight) { + return convert(wrapper.getCpu(), desiredHeight); + } /** * Convert a BGR-formatted OpenCV {@link Mat} into a JavaFX {@link Image}. JavaFX understands ARGB diff --git a/ui/src/test/java/edu/wpi/grip/ui/codegeneration/tools/HelperTools.java b/ui/src/test/java/edu/wpi/grip/ui/codegeneration/tools/HelperTools.java index d28209e13c..aa9a918fcc 100644 --- a/ui/src/test/java/edu/wpi/grip/ui/codegeneration/tools/HelperTools.java +++ b/ui/src/test/java/edu/wpi/grip/ui/codegeneration/tools/HelperTools.java @@ -4,7 +4,6 @@ import org.bytedeco.javacpp.indexer.UByteIndexer; import org.opencv.core.Core; -import org.opencv.core.CvType; import org.opencv.core.Mat; import org.opencv.core.MatOfByte; import org.opencv.imgcodecs.Imgcodecs; @@ -23,6 +22,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.opencv.core.CvType.CV_8UC; public class HelperTools { private static final Logger logger = Logger.getLogger(HelperTools.class.getName()); @@ -68,7 +68,7 @@ public static double matAvgDiff(Mat mat1, Mat mat2) { */ public static Mat bytedecoMatToCVMat(org.bytedeco.javacpp.opencv_core.Mat input) { UByteIndexer idxer = input.createIndexer(); - Mat out = new Mat(idxer.rows(), idxer.cols(), CvType.CV_8UC(idxer.channels())); + Mat out = new Mat((int) idxer.rows(), (int) idxer.cols(), CV_8UC((int) idxer.channels())); //Mat out = new Mat(idxer.rows(),idxer.cols(),input.type()); for (int row = 0; row < idxer.rows(); row++) { for (int col = 0; col < idxer.cols(); col++) { diff --git a/ui/src/test/java/edu/wpi/grip/ui/preview/ImageSocketPreviewViewTest.java b/ui/src/test/java/edu/wpi/grip/ui/preview/ImageSocketPreviewViewTest.java index e866ddf8e8..a1b06f1b36 100644 --- a/ui/src/test/java/edu/wpi/grip/ui/preview/ImageSocketPreviewViewTest.java +++ b/ui/src/test/java/edu/wpi/grip/ui/preview/ImageSocketPreviewViewTest.java @@ -1,5 +1,6 @@ package edu.wpi.grip.ui.preview; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.operations.network.MockGripNetworkModule; import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.core.sockets.SocketHint; @@ -13,7 +14,6 @@ import com.google.inject.Injector; import com.google.inject.util.Modules; -import org.bytedeco.javacpp.opencv_core.Mat; import org.junit.After; import org.junit.Test; import org.testfx.framework.junit.ApplicationTest; @@ -39,9 +39,9 @@ public void start(Stage stage) { final ImageSocketPreviewView imageSocketPreviewView = new ImageSocketPreviewView(new MockGripPlatform(new EventBus()), injector.getInstance(OutputSocket.Factory.class) - .create(new SocketHint.Builder<>(Mat.class) + .create(new SocketHint.Builder<>(MatWrapper.class) .identifier(identifier) - .initialValueSupplier(Files.gompeiJpegFile::createMat) + .initialValueSupplier(() -> MatWrapper.wrap(Files.gompeiJpegFile.createMat())) .build())); final Scene scene = new Scene(imageSocketPreviewView); stage.setScene(scene); diff --git a/ui/ui.gradle.kts b/ui/ui.gradle.kts index b982dec998..95016c7d23 100644 --- a/ui/ui.gradle.kts +++ b/ui/ui.gradle.kts @@ -17,6 +17,12 @@ if (!(project.hasProperty("generation") || project.hasProperty("genonly"))) { } } +val withCuda = project.hasProperty("cuda") || project.hasProperty("WITH_CUDA") + +if (withCuda) { + version = "$version-cuda" +} + dependencies { compile(project(":core")) compile(project(":ui:preloader"))