Merge branch 'dev'

README.md

@@ -4,24 +4,24 @@
 
 **OpenEM** is a library that provides advanced video analytics for 
 fisheries electronic monitoring (EM) data.  It currently supports detection,
-classification and measurement of fish during landing or discard.  This
-functionality is currently only available via a deployment library with
+classification, counting and measurement of fish during landing or discard.  
+This functionality is currently only available via a deployment library with
 pretrained models available in our example data (see tutorial).  The base
 library is written in C++, with bindings available for both Python and C#.
 Examples are included for all three languages.
 
 There are immediate plans to develop a training library so that users
-can build their own models on their own data, and to add counting 
-functionality.  Currently builds have only been tested on Windows.  We plan
-to support both Ubuntu and macOS in the future.
+can build their own models on their own data.  Currently builds have only 
+been tested on Windows.  We plan to support both Ubuntu and macOS in the 
+future.
 
 Click the image below to see a video of OpenEM in action:
 
 [![OpenEM detection example](https://img.youtube.com/vi/EZ1Xyg_mnhM/0.jpg)](https://youtu.be/EZ1Xyg_mnhM)
 
 ### Contents
 
-* [Building and Testing](doc/build.md)
+* [Building](doc/build.md)
 * [Tutorial](doc/tutorial.md)
 * [Data Collection Guidelines](doc/data_collection.md)
 * [Annotation Guidelines](doc/annotation.md)

deploy/bindings/openem.i

@@ -11,9 +11,10 @@
 #include "find_ruler.h"
 #include "detect.h"
 #include "classify.h"
+#include "count.h"
 %}
 
-%include "pointer.i"
+%include "cpointer.i"
 %include "stdint.i"
 %include "std_array.i"
 %include "std_string.i"
@@ -22,6 +23,7 @@
 namespace std {
   %template(VectorDouble) vector<double>;
   %template(VectorFloat) vector<float>;
+  %template(VectorInt) vector<int>;
   %template(VectorVectorFloat) vector<vector<float>>;
   %template(VectorVectorVectorFloat) vector<vector<vector<float>>>;
   %template(VectorUint8) vector<uint8_t>;
@@ -31,6 +33,11 @@ namespace std {
   %template(VectorRect) vector<array<int, 4>>;
   %template(VectorVectorRect) vector<vector<array<int, 4>>>;
   %template(VectorImage) vector<openem::Image>;
+  %template(VectorDetection) vector<openem::detect::Detection>;
+  %template(VectorVectorDetection) vector<vector<openem::detect::Detection>>;
+  %template(ArrayFloat3) array<float, 3>;
+  %template(VectorClassification) vector<openem::classify::Classification>;
+  %template(VectorVectorClassification) vector<vector<openem::classify::Classification>>;
 };
 
 %include "error_codes.h"
@@ -39,4 +46,5 @@ namespace std {
 %include "find_ruler.h"
 %include "detect.h"
 %include "classify.h"
+%include "count.h"
 

deploy/include/classify.h

@@ -28,6 +28,22 @@
 namespace openem {
 namespace classify {
 
+/// Contains classification results.
+struct Classification {
+  /// Species scores.
+  ///
+  /// Elements correspond to the species used to train the loaded model.
+  std::vector<float> species;
+
+  /// Cover scores.
+  ///
+  /// Elements correspond to the following:
+  /// * No fish in the image.
+  /// * Fish is covered by a hand.
+  /// * Fish is not covered.
+  std::array<float, 3> cover;
+};
+
 /// Class for determining fish species and whether the image is covered
 /// by a hand, clear, or not a fish.
 class Classifier {
@@ -63,17 +79,11 @@ class Classifier {
   /// @return Error code.
   ErrorCode AddImage(const Image& image);
 
-  /// Determines fish species and whether the fish is covered by a hand,
-  /// clear, or not a fish.
-  /// @param scores Vector of double vectors.  Each double vector
-  /// corresponds to one of the images in the image queue.  The first
-  /// three numbers in the double vector correspond to:
-  /// * No fish in the image.
-  /// * Fish is covered by a hand.
-  /// * Fish is not covered.
-  /// The remaining vector elements correspond to the species used to train
-  /// the loaded model.
-  ErrorCode Process(std::vector<std::vector<float>>* scores);
+  /// Performs classification on each image that was added to the 
+  /// processing queue with AddImage.
+  /// @param results Vector of classification results.
+  /// @return Error code.
+  ErrorCode Process(std::vector<Classification>* results);
  private:
   /// Forward declaration of implementation class.
   class ClassifierImpl;

deploy/include/count.h

@@ -0,0 +1,80 @@
+/// @file
+/// @brief Interface for counting fish.
+/// @copyright Copyright (C) 2018 CVision AI.
+/// @license This file is part of OpenEM, released under GPLv3.
+//  OpenEM is free software: you can redistribute it and/or modify
+//  it under the terms of the GNU General Public License as published by
+//  the Free Software Foundation, either version 3 of the License, or
+//  any later version.
+//
+//  This program is distributed in the hope that it will be useful,
+//  but WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+//  GNU General Public License for more details.
+//
+//  You should have received a copy of the GNU General Public License
+//  along with OpenEM.  If not, see <http://www.gnu.org/licenses/>.
+
+#ifndef OPENEM_DEPLOY_COUNT_H_
+#define OPENEM_DEPLOY_COUNT_H_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "detect.h"
+#include "classify.h"
+#include "error_codes.h"
+
+namespace openem {
+namespace count {
+
+/// Class for finding keyframes.
+class KeyframeFinder {
+ public:
+  /// Constructor.
+  KeyframeFinder();
+
+  /// Destructor.
+  ~KeyframeFinder();
+
+  /// Initializes the keyframe finder.
+  /// @param model_path Path to protobuf file containing model.
+  /// @param img_width Width of image input to detector.
+  /// @param img_height Height of image input to detector.
+  /// @param gpu_fraction Fraction of GPU memory that may be allocated to
+  /// this object.
+  /// @return Error code.
+  ErrorCode Init(
+      const std::string& model_path, 
+      int img_width,
+      int img_height,
+      double gpu_fraction=1.0);
+
+  /// Finds keyframes in a given sequence.
+  /// @param classifications Sequence of outputs from classifier.  Outer
+  /// vector corresponds to frames, inner corresponds to detections in each
+  /// frame.
+  /// @param detections Sequence of outputs from detector.  Outer vector
+  /// corresponds to frames, inner corresponds to detections in each frame.
+  /// @param keyframes Vector of keyframes in the sequence.  The length of
+  /// this vector is the number of fish in the sequence.  The values 
+  /// are the indices of the classification and detection vectors.
+  /// @return Error code.
+  ErrorCode Process(
+      const std::vector<std::vector<classify::Classification>>& classifications,
+      const std::vector<std::vector<detect::Detection>>& detections,
+      std::vector<int>* keyframes);
+ private:
+  /// Forward declaration of implementation class.
+  class KeyframeFinderImpl;
+
+  /// Pointer to implementation.
+  std::unique_ptr<KeyframeFinderImpl> impl_;
+};
+
+} // namespace count
+} // namespace openem
+
+#endif // OPENEM_DEPLOY_COUNT_H_
+

deploy/include/detail/model.h

@@ -36,6 +36,53 @@ class Model {
   /// Constructor.
   Model();
 
+  /// Destructor.
+  ~Model();
+
+  /// Loads a model from a protobuf file and initializes the tensorflow
+  /// session.
+  /// @param model_path Path to protobuf file containing the model.
+  /// @param gpu_fraction Fraction fo GPU allowed to be used by this object.
+  /// @return Error code.
+  ErrorCode Init(const std::string& model_path, double gpu_fraction);
+
+  /// Returns input size.
+  /// @return Input size.
+  std::vector<int> InputSize();
+
+  /// Returns whether the model has been initialized.
+  /// @retur True if initialized.
+  bool Initialized();
+
+  /// Processes the model on the current batch.
+  /// @param input Input tensor.
+  /// @param input_name Name of input tensor.
+  /// @param output_names Name of output tensors.
+  /// @param outputs Output of the model.
+  /// @return Error code.
+  ErrorCode Process(
+      const tensorflow::Tensor& input,
+      const std::string& input_name,
+      const std::vector<std::string>& output_names,
+      std::vector<tensorflow::Tensor>* outputs);
+ private:
+  /// Forward declaration of class implementation.
+  class ModelImpl;
+
+  /// Pointer to implementation.
+  std::unique_ptr<ModelImpl> impl_;
+};
+
+/// Model that accepts image data as input.  This class is intended to be 
+/// an implementation detail only.
+class ImageModel {
+ public:
+  /// Constructor.
+  ImageModel();
+
+  /// Destructor.
+  ~ImageModel(); 
+
   /// Loads a model from a protobuf file and initializes the tensorflow
   /// session.
   /// @param model_path Path to protobuf file containing the model.
@@ -58,32 +105,20 @@ class Model {
       std::function<tensorflow::Tensor(const cv::Mat&, int, int)> preprocess);
 
   /// Processes the model on the current batch.
-  /// @param outputs Output of the model.
   /// @param input_name Name of input tensor.
   /// @param output_names Name of output tensors.
+  /// @param outputs Output of the model.
   /// @return Error code.
   ErrorCode Process(
-      std::vector<tensorflow::Tensor>* outputs, 
       const std::string& input_name,
-      const std::vector<std::string>& output_names);
+      const std::vector<std::string>& output_names,
+      std::vector<tensorflow::Tensor>* outputs);
  private:
-  /// Tensorflow session.
-  std::unique_ptr<tensorflow::Session> session_;
-
-  /// Input image width.
-  int width_;
-
-  /// Input image height.
-  int height_;
-
-  /// Indicates whether the model has been initialized.
-  bool initialized_;
-
-  /// Queue of futures containing preprocessed images.
-  std::queue<std::future<tensorflow::Tensor>> preprocessed_;
+  /// Forward declaration of class implementation.
+  class ImageModelImpl;
 
-  /// Mutex for handling concurrent access to image queue.
-  std::mutex mutex_;
+  /// Pointer to implementation.
+  std::unique_ptr<ImageModelImpl> impl_;
 };
 
 } // namespace detail

deploy/include/detail/util.h

@@ -48,11 +48,19 @@ ErrorCode GetSession(tensorflow::Session** session, double gpu_fraction);
 
 /// Gets graph input size.
 /// @param graph_def Graph definition.
+/// @param input_size Input size vector.
+/// @return Error code.
+ErrorCode InputSize(
+    const tensorflow::GraphDef& graph_def,
+    std::vector<int>* input_size);
+
+/// Gets image size from graph.
+/// @param input_size Input dimensions output from InputSize.
 /// @param width Width dimension of input layer.
 /// @param height Height dimension of input layer.
 /// @return Error code.
-ErrorCode InputSize(
-    const tensorflow::GraphDef& graph_def, 
+ErrorCode ImageSize(
+    const std::vector<int>& input_size,
     int* width, 
     int* height);
 

deploy/include/detect.h

@@ -28,6 +28,18 @@
 namespace openem {
 namespace detect {
 
+/// Contains information for a single detection.
+struct Detection {
+  /// Location of the detection in the frame.
+  Rect location;
+
+  /// Confidence score.
+  float confidence;
+
+  /// Species index based on highest confidence.
+  int species;
+};
+
 /// Class for detecting fish in images.
 class Detector {
  public:
@@ -64,7 +76,9 @@ class Detector {
 
   /// Finds fish in batched images by performing object detection
   /// with Single Shot Detector (SSD).
-  ErrorCode Process(std::vector<std::vector<Rect>>* detections);
+  /// @param detections Vector of detections for each image.
+  /// @return Error code.
+  ErrorCode Process(std::vector<std::vector<Detection>>* detections);
  private:
   /// Forward declaration of implementation class.
   class DetectorImpl;

deploy/include/error_codes.h

@@ -36,7 +36,10 @@ enum ErrorCode {
   kErrorNumChann,      ///< Invalid number of image channels.
   kErrorVidFileOpen,   ///< Failed to open video file.
   kErrorVidNotOpen,    ///< Tried to read from unopened video.
-  kErrorVidFrame       ///< Failed to read video frame.
+  kErrorVidFrame,      ///< Failed to read video frame.
+  kErrorLenMismatch,   ///< Mismatch in sequence lengths.
+  kErrorNumInputDims,  ///< Unexpected number of input dimensions.
+  kErrorBadSeqLength   ///< Wrong sequence length for input.
 };
 
 } // namespace openem

deploy/src/CMakeLists.txt

@@ -3,6 +3,7 @@ add_library(openem
   find_ruler.cc 
   detect.cc
   classify.cc
+  count.cc
   util.cc 
   model.cc
   video.cc