From 5fb7ea8b35a15f9f1c43a77622f91d0a7e65d045 Mon Sep 17 00:00:00 2001 From: Yongxuanzhang <44033547+Yongxuanzhang@users.noreply.github.com> Date: Wed, 10 Feb 2021 02:56:46 -0500 Subject: [PATCH] docs(ndarray): update docstring for ndarray (#1920) * docs(ndarray): update docstring for ndarray * docs: minor fix Co-authored-by: Nan Wang --- jina/types/ndarray/__init__.py | 19 ++++++++--------- jina/types/ndarray/dense/numpy.py | 28 ++++++++++++------------- jina/types/ndarray/sparse/__init__.py | 10 +++++---- jina/types/ndarray/sparse/numpy.py | 3 ++- jina/types/ndarray/sparse/pytorch.py | 16 +++++++------- jina/types/ndarray/sparse/scipy.py | 12 +++++------ jina/types/ndarray/sparse/tensorflow.py | 4 ++-- 7 files changed, 45 insertions(+), 47 deletions(-) diff --git a/jina/types/ndarray/__init__.py b/jina/types/ndarray/__init__.py index 987aeb7d2e8f7..5ddf7d551bf03 100644 --- a/jina/types/ndarray/__init__.py +++ b/jina/types/ndarray/__init__.py @@ -11,18 +11,17 @@ class BaseNdArray(ProtoTypeMixin): - """A base class for containing the protobuf message of NdArray. It defines interfaces - for easier get/set value. + """ + A base class for containing the protobuf message of NdArray. It defines interfaces for easier get/set value. Do not use this class directly. Subclass should be used. + + :param proto: the protobuf message, when not given then create a new one via :meth:`get_null_proto` """ def __init__(self, proto: Union['PbMessageType', AnyNdArray] = None, *args, **kwargs): - """ - - :param proto: the protobuf message, when not given then create a new one via :meth:`get_null_proto` - """ + """Set the constructor method.""" if proto is not None and isinstance(type(proto), PbMessageType): self._pb_body = proto # a weak ref/copy else: @@ -33,20 +32,20 @@ def __init__(self, proto: Union['PbMessageType', AnyNdArray] = None, @property def null_proto(self) -> 'PbMessageType': - """Get the new protobuf representation""" + """Get the new protobuf representation.""" raise NotImplementedError @property def value(self) -> AnyNdArray: - """Return the value of the ndarray, in numpy, scipy, tensorflow, pytorch type""" + """Return the value of the ndarray, in numpy, scipy, tensorflow, pytorch type.""" raise NotImplementedError @value.setter def value(self, value: AnyNdArray): - """Set the value from numpy, scipy, tensorflow, pytorch type to protobuf""" + """Set the value from numpy, scipy, tensorflow, pytorch type to protobuf.""" raise NotImplementedError def copy_to(self, proto: 'PbMessageType') -> 'BaseNdArray': - """Copy itself to another protobuf message, return a view of the copied message""" + """Copy itself to another protobuf message, return a view of the copied message.""" proto.CopyFrom(self._pb_body) return self.__class__(proto) diff --git a/jina/types/ndarray/dense/numpy.py b/jina/types/ndarray/dense/numpy.py index 41766dfd6921e..40fe19f47ade8 100644 --- a/jina/types/ndarray/dense/numpy.py +++ b/jina/types/ndarray/dense/numpy.py @@ -13,26 +13,24 @@ class DenseNdArray(BaseDenseNdArray): Dense NdArray powered by numpy, supports quantization method. Most of the cases you don't want use this class directly, use :class:`NdArray` instead. - """ - def __init__(self, proto: 'jina_pb2.NdArrayProto' = None, quantize: str = None, *args, **kwargs): - """ + :param proto: the protobuf message, when not given then create a new one + :param quantize: the quantization method used when converting to protobuf. + Availables are ``fp16``, ``uint8``, default is None. - :param proto: the protobuf message, when not given then create a new one - :param quantize: the quantization method used when converting to protobuf. - Availables are ``fp16``, ``uint8``, default is None. + .. note:: + Remarks on quantization: + The quantization only works when ``x`` is in ``float32`` or ``float64``. The motivation is to + save the network bandwidth by using less bits to store the numpy array in the protobuf. - .. note:: - Remarks on quantization: - The quantization only works when ``x`` is in ``float32`` or ``float64``. The motivation is to - save the network bandwidth by using less bits to store the numpy array in the protobuf. + - ``fp16`` quantization is lossless, can be used widely. Each float is represented by 16 bits. + - ``uint8`` quantization is lossy. Each float is represented by 8 bits. + The algorithm behind is standard scaling. - - ``fp16`` quantization is lossless, can be used widely. Each float is represented by 16 bits. - - ``uint8`` quantization is lossy. Each float is represented by 8 bits. - The algorithm behind is standard scaling. + the quantize type is stored and the blob is self-contained to recover the original numpy array + """ - the quantize type is stored and the blob is self-contained to recover the original numpy array - """ + def __init__(self, proto: 'jina_pb2.NdArrayProto' = None, quantize: str = None, *args, **kwargs): super().__init__(proto, *args, **kwargs) self.quantize = os.environ.get('JINA_ARRAY_QUANT', quantize) diff --git a/jina/types/ndarray/sparse/__init__.py b/jina/types/ndarray/sparse/__init__.py index 38ce8d1cf3515..58a2b7de6f311 100644 --- a/jina/types/ndarray/sparse/__init__.py +++ b/jina/types/ndarray/sparse/__init__.py @@ -14,7 +14,7 @@ class BaseSparseNdArray(BaseNdArray): """ - The base class for SparseNdArray. + The base class for :class:`SparseNdArray`. Do not use this class directly. Subclass should be used. @@ -28,17 +28,19 @@ def null_proto(self): return jina_pb2.SparseNdArrayProto() def sparse_constructor(self, indices: 'np.ndarray', values: 'np.ndarray', shape: List[int]) -> AnySparseNdArray: - """ Sparse NdArray constructor, must be implemented by subclass + """ + Sparse NdArray constructor, must be implemented by subclass. :param indices: the indices of the sparse array :param values: the values of the sparse array :param shape: the shape of the dense array - :return: + :return: Sparse NdArray """ raise NotImplementedError def sparse_parser(self, value: AnySparseNdArray) -> Dict[str, Union['np.ndarray', List[int]]]: - """ Parsing a Sparse NdArray to indices, values and shape, must be implemented by subclass + """ + Parse a Sparse NdArray to indices, values and shape, must be implemented by subclass. :param value: the sparse ndarray :return: a Dict with three entries {'indices': ..., 'values':..., 'shape':...} diff --git a/jina/types/ndarray/sparse/numpy.py b/jina/types/ndarray/sparse/numpy.py index 4af0d1074d95a..38ae0854dcd3b 100644 --- a/jina/types/ndarray/sparse/numpy.py +++ b/jina/types/ndarray/sparse/numpy.py @@ -8,7 +8,8 @@ class SparseNdArray(BaseSparseNdArray): - """Numpy powered sparse ndarray, it uses nonzero. + """ + Numpy powered sparse ndarray, it uses nonzero. .. note:: This always take a dense :class:`np.ndarray` and return a :class:`np.ndarray`. diff --git a/jina/types/ndarray/sparse/pytorch.py b/jina/types/ndarray/sparse/pytorch.py index c35fb6da8e136..86561939a8a72 100644 --- a/jina/types/ndarray/sparse/pytorch.py +++ b/jina/types/ndarray/sparse/pytorch.py @@ -12,20 +12,20 @@ class SparseNdArray(BaseSparseNdArray): - """Pytorch powered sparse ndarray, i.e. FloatTensor + """ + Pytorch powered sparse ndarray, i.e. FloatTensor. .. seealso:: https://pytorch.org/docs/stable/sparse.html + + :param transpose_indices: in torch, the input to LongTensor is NOT a list of index tuples. + If you want to write your indices this way, you should transpose before passing them to the sparse constructor + + .. note:: + To comply with Tensorflow, `transpose_indices` is set to True by default """ def __init__(self, transpose_indices: bool = True, *args, **kwargs): - """ - :param transpose_indices: in torch, the input to LongTensor is NOT a list of index tuples. - If you want to write your indices this way, you should transpose before passing them to the sparse constructor - - .. note:: - To comply with Tensorflow, `transpose_indices` is set to True by default - """ super().__init__(*args, **kwargs) self.transpose_indices = transpose_indices diff --git a/jina/types/ndarray/sparse/scipy.py b/jina/types/ndarray/sparse/scipy.py index 239b53c91bd7b..b5e0e09bda513 100644 --- a/jina/types/ndarray/sparse/scipy.py +++ b/jina/types/ndarray/sparse/scipy.py @@ -12,22 +12,20 @@ class SparseNdArray(BaseSparseNdArray): - """Scipy powered sparse ndarray + """ + Scipy powered sparse ndarray. .. warning:: scipy only supports ndim=2 .. seealso:: https://docs.scipy.org/doc/scipy/reference/sparse.html + + :param proto: the protobuf message, when not given then create a new one via :meth:`get_null_proto` + :param sp_format: the sparse format of the scipy matrix. one of 'coo', 'bsr', 'csc', 'csr'. """ def __init__(self, proto: 'jina_pb2.SparseNdArrayProto' = None, sp_format: str = 'coo', *args, **kwargs): - """ - - :param sp_format: the sparse format of the scipy matrix. one of 'coo', 'bsr', 'csc', 'csr' - :param args: - :param kwargs: - """ import scipy.sparse super().__init__(proto, *args, **kwargs) support_fmt = {'coo', 'bsr', 'csc', 'csr'} diff --git a/jina/types/ndarray/sparse/tensorflow.py b/jina/types/ndarray/sparse/tensorflow.py index b7319542c0559..b740b28f7d31f 100644 --- a/jina/types/ndarray/sparse/tensorflow.py +++ b/jina/types/ndarray/sparse/tensorflow.py @@ -10,11 +10,11 @@ class SparseNdArray(BaseSparseNdArray): - """Tensorflow powered sparse ndarray, i.e. SparseTensor + """ + Tensorflow powered sparse ndarray, i.e. SparseTensor. .. seealso:: https://www.tensorflow.org/api_docs/python/tf/sparse/SparseTensor - """ def sparse_constructor(self, indices: 'np.ndarray', values: 'np.ndarray', shape: List[int]) -> 'SparseTensor':