[jsk_perception] Support chainercv mask rcnn

.travis.yml

@@ -39,7 +39,7 @@ before_script:
 script:
   - if [ "${CHECK_PYTHON3_COMPILE}" == "true" ]; then python3 -m compileall .; exit $?; fi
   # use https://github.com/ros-infrastructure/rosdep/pull/694 to respect version_lt for python pip, some package requries python3
-  - export BEFORE_SCRIPT="sudo pip install numpy==1.16.6; sudo pip install fcn chainercv chainer==6.7.0 cupy-cuda91; $BEFORE_SCRIPT"
+  - export BEFORE_SCRIPT="sudo pip install numpy==1.16.6; sudo pip install fcn chainercv chainer-mask-rcnn==0.3.0 chainer==6.7.0 cupy-cuda91; $BEFORE_SCRIPT"
   - if [[ "$ROS_DISTRO" == "kinetic" && "$ROS_DISTRO" == "melodic" ]]; then export BEFORE_SCRIPT="sudo apt-get install -y patchutils; curl -sL https://patch-diff.githubusercontent.com/raw/ros-infrastructure/rosdep/pull/753.diff | sudo patch -d /usr/lib/python2.7/dist-packages/ -p2; curl -sL https://patch-diff.githubusercontent.com/raw/ros-infrastructure/rosdep/pull/694.diff | filterdiff --exclude='a/test/*' | sudo patch -d /usr/lib/python2.7/dist-packages/ -p2; $BEFORE_SCRIPT"; fi
   - .travis/travis.sh
   - (cd $TRAVIS_BUILD_DIR/doc && source setup.sh && make html)

doc/jsk_perception/nodes/mask_rcnn_instance_segmentation.rst

@@ -55,18 +55,27 @@ Parameters
 
   GPU id.
 
+* ``~model_name`` (String, ``mask_rcnn_resnet50``)
+
+  Model name.
+  ``mask_rcnn_resnet50``, ``mask_rcnn_fpn_resnet50`` and  ``mask_rcnn_fpn_resnet101`` are supported.
+
 * ``~score_thresh`` (Float, default: ``0.7``)
 
   Score threshold of detections.
 
-* ``~fg_class_names`` (List of String, required)
+* ``~fg_class_names`` (List of String, ``None``)
 
   Foreground class names that is used to identify number of class.
   It is also used for the ``name`` field of ``~output/labels``.
+  When ``pretrained_model: coco`` is set, default COCO ``pretrained_model`` and ``fg_class_names`` will be loaded.
+  When ``pretrained_model: voc`` is set, default VOC ``pretrained_model`` and ``fg_class_names`` will be loaded.
 
 * ``~pretrained_model`` (String, required)
 
   Pretrained model path.
+  When ``pretrained_model: coco`` is set, default COCO ``pretrained_model`` and ``fg_class_names`` will be loaded.
+  When ``pretrained_model: voc`` is set, default VOC ``pretrained_model`` and ``fg_class_names`` will be loaded.
 
 * ``~classifier_name`` (String, default: ``rospy.get_name()``)
 

jsk_perception/node_scripts/mask_rcnn_instance_segmentation.py

@@ -26,7 +26,21 @@
 ''', file=sys.stderr)
    sys.exit(1)
 import chainer
+from chainercv.datasets.coco.coco_utils \
+    import coco_instance_segmentation_label_names
+try:
+    from chainercv.links import MaskRCNNFPNResNet101
+    from chainercv.links import MaskRCNNFPNResNet50
+except ImportError:
+    print('''If you want to use chainercv mask_rcnn, please upgrade chainercv
+
+    sudo pip install chainercv>=0.13.0
+
+''', file=sys.stdout)
+
+from chainercv.utils import mask_to_bbox
 import numpy as np
+import yaml
 
 try:
     import chainer_mask_rcnn
@@ -49,6 +63,7 @@
 from jsk_recognition_msgs.msg import ClassificationResult
 from jsk_topic_tools import ConnectionBasedTransport
 from pcl_msgs.msg import PointIndices
+import rospkg
 import rospy
 from sensor_msgs.msg import Image
 
@@ -65,7 +80,7 @@ def __init__(self):
         chainer.global_config.train = False
         chainer.global_config.enable_backprop = False
 
-        fg_class_names = rospy.get_param('~fg_class_names')
+        fg_class_names = rospy.get_param('~fg_class_names', None)
         if isinstance(fg_class_names, str) and os.path.exists(fg_class_names):
             rospy.loginfo('Loading class names from file: {}'.format(fg_class_names))
             with open(fg_class_names, 'r') as f:
@@ -75,16 +90,56 @@ def __init__(self):
         pretrained_model = rospy.get_param('~pretrained_model')
         self.classifier_name = rospy.get_param(
             "~classifier_name", rospy.get_name())
-
-        n_fg_class = len(self.fg_class_names)
-        self.model = chainer_mask_rcnn.models.MaskRCNNResNet(
-            n_layers=50,
-            n_fg_class=n_fg_class,
-            pretrained_model=pretrained_model,
-            anchor_scales=rospy.get_param('~anchor_scales', [4, 8, 16, 32]),
-            min_size=rospy.get_param('~min_size', 600),
-            max_size=rospy.get_param('~max_size', 1000),
-        )
+        self.model_name = rospy.get_param('~model_name', 'mask_rcnn_resnet50')
+        rospack = rospkg.RosPack()
+
+        if self.model_name == 'mask_rcnn_resnet50':
+            if pretrained_model == 'coco':
+                pretrained_model = os.path.join(
+                    rospack.get_path('jsk_perception'),
+                    'trained_data/mask_rcnn_resnet50_coco_20180730.npz')
+                if self.fg_class_names is None:
+                    yaml_path = os.path.join(
+                        rospack.get_path('jsk_perception'),
+                        'sample/config/coco_class_names.yaml')
+                    with open(yaml_path) as yaml_f:
+                        self.fg_class_names = yaml.load(yaml_f)
+            elif pretrained_model == 'voc':
+                pretrained_model = os.path.join(
+                    rospack.get_path('jsk_perception'),
+                    'trained_data/mask_rcnn_resnet50_voc_20180516.npz')
+                if self.fg_class_names is None:
+                    yaml_path = os.path.join(
+                        rospack.get_path('jsk_perception'),
+                        'sample/config/voc_class_names.yaml')
+                    with open(yaml_path) as yaml_f:
+                        self.fg_class_names = yaml.load(yaml_f)
+
+            self.model = chainer_mask_rcnn.models.MaskRCNNResNet(
+                n_layers=50,
+                n_fg_class=len(self.fg_class_names),
+                pretrained_model=pretrained_model,
+                anchor_scales=rospy.get_param('~anchor_scales', [4, 8, 16, 32]),
+                min_size=rospy.get_param('~min_size', 600),
+                max_size=rospy.get_param('~max_size', 1000),
+            )
+        elif self.model_name == 'mask_rcnn_fpn_resnet50':
+            if pretrained_model == 'coco':
+                self.fg_class_names = coco_instance_segmentation_label_names
+            self.model = MaskRCNNFPNResNet50(
+                n_fg_class=len(self.fg_class_names),
+                pretrained_model=pretrained_model)
+            self.model.use_preset('visualize')
+        elif self.model_name == 'mask_rcnn_fpn_resnet101':
+            if pretrained_model == 'coco':
+                self.fg_class_names = coco_instance_segmentation_label_names
+            self.model = MaskRCNNFPNResNet101(
+                n_fg_class=len(self.fg_class_names),
+                pretrained_model=pretrained_model)
+            self.model.use_preset('visualize')
+        else:
+            rospy.logerr('Unsupported model_name: {}'.format(self.model_name))
+        self.model.score_thresh = rospy.get_param('~score_thresh', 0.7)
         if self.gpu >= 0:
             self.model.to_gpu(self.gpu)
 
@@ -121,16 +176,23 @@ def config_callback(self, config, level):
     def callback(self, imgmsg):
         bridge = cv_bridge.CvBridge()
         img = bridge.imgmsg_to_cv2(imgmsg, desired_encoding='rgb8')
-        img_chw = img.transpose(2, 0, 1)  # C, H, W
+        img_chw = img.transpose((2, 0, 1))  # C, H, W
 
         if self.gpu >= 0:
             chainer.cuda.get_device_from_id(self.gpu).use()
-        bboxes, masks, labels, scores = self.model.predict([img_chw])
-
-        bboxes = bboxes[0]
-        masks = masks[0]
-        labels = labels[0]
-        scores = scores[0]
+        if self.model_name == 'mask_rcnn_resnet50':
+            bboxes, masks, labels, scores = self.model.predict([img_chw])
+            bboxes = bboxes[0]
+            masks = masks[0]
+            labels = labels[0]
+            scores = scores[0]
+        else:
+            img_chw = img_chw.astype(np.float32)
+            masks, labels, scores = self.model.predict([img_chw])
+            masks = masks[0]
+            labels = labels[0]
+            scores = scores[0]
+            bboxes = mask_to_bbox(masks)
 
         msg_indices = ClusterPointIndices(header=imgmsg.header)
         msg_labels = LabelArray(header=imgmsg.header)

jsk_perception/package.xml

@@ -81,6 +81,7 @@
   <exec_depend version_lt="7.0.0">python-chainer-pip</exec_depend>
   <!-- python-cahiner-pip depends: [cython, python-numpy] in rosdep/python.yaml, which is not installable on python3 environment -->
   <exec_depend condition="$ROS_PYTHON_VERSION == 2">python-chainercv-pip</exec_depend>
+  <exec_depend condition="$ROS_PYTHON_VERSION == 2">python-chainercv-mask-rcnn-pip</exec_depend>
   <!-- }} install chainer -->
   <exec_depend>python-dlib</exec_depend>  <!-- pip -->
   <!-- install fcn {{ -->