Skip to content

Commit

Permalink
docs(ndarray): update docstring for ndarray (#1920)
Browse files Browse the repository at this point in the history
* docs(ndarray): update docstring for ndarray

* docs: minor fix

Co-authored-by: Nan Wang <[email protected]>
  • Loading branch information
Yongxuanzhang and nan-wang authored Feb 10, 2021
1 parent 768c9a8 commit 5fb7ea8
Show file tree
Hide file tree
Showing 7 changed files with 45 additions and 47 deletions.
19 changes: 9 additions & 10 deletions jina/types/ndarray/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,17 @@


class BaseNdArray(ProtoTypeMixin):
"""A base class for containing the protobuf message of NdArray. It defines interfaces
for easier get/set value.
"""
A base class for containing the protobuf message of NdArray. It defines interfaces for easier get/set value.
Do not use this class directly. Subclass should be used.
:param proto: the protobuf message, when not given then create a new one via :meth:`get_null_proto`
"""

def __init__(self, proto: Union['PbMessageType', AnyNdArray] = None,
*args, **kwargs):
"""
:param proto: the protobuf message, when not given then create a new one via :meth:`get_null_proto`
"""
"""Set the constructor method."""
if proto is not None and isinstance(type(proto), PbMessageType):
self._pb_body = proto # a weak ref/copy
else:
Expand All @@ -33,20 +32,20 @@ def __init__(self, proto: Union['PbMessageType', AnyNdArray] = None,

@property
def null_proto(self) -> 'PbMessageType':
"""Get the new protobuf representation"""
"""Get the new protobuf representation."""
raise NotImplementedError

@property
def value(self) -> AnyNdArray:
"""Return the value of the ndarray, in numpy, scipy, tensorflow, pytorch type"""
"""Return the value of the ndarray, in numpy, scipy, tensorflow, pytorch type."""
raise NotImplementedError

@value.setter
def value(self, value: AnyNdArray):
"""Set the value from numpy, scipy, tensorflow, pytorch type to protobuf"""
"""Set the value from numpy, scipy, tensorflow, pytorch type to protobuf."""
raise NotImplementedError

def copy_to(self, proto: 'PbMessageType') -> 'BaseNdArray':
"""Copy itself to another protobuf message, return a view of the copied message"""
"""Copy itself to another protobuf message, return a view of the copied message."""
proto.CopyFrom(self._pb_body)
return self.__class__(proto)
28 changes: 13 additions & 15 deletions jina/types/ndarray/dense/numpy.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,26 +13,24 @@ class DenseNdArray(BaseDenseNdArray):
Dense NdArray powered by numpy, supports quantization method.
Most of the cases you don't want use this class directly, use :class:`NdArray` instead.
"""
def __init__(self, proto: 'jina_pb2.NdArrayProto' = None, quantize: str = None, *args, **kwargs):
"""
:param proto: the protobuf message, when not given then create a new one
:param quantize: the quantization method used when converting to protobuf.
Availables are ``fp16``, ``uint8``, default is None.
:param proto: the protobuf message, when not given then create a new one
:param quantize: the quantization method used when converting to protobuf.
Availables are ``fp16``, ``uint8``, default is None.
.. note::
Remarks on quantization:
The quantization only works when ``x`` is in ``float32`` or ``float64``. The motivation is to
save the network bandwidth by using less bits to store the numpy array in the protobuf.
.. note::
Remarks on quantization:
The quantization only works when ``x`` is in ``float32`` or ``float64``. The motivation is to
save the network bandwidth by using less bits to store the numpy array in the protobuf.
- ``fp16`` quantization is lossless, can be used widely. Each float is represented by 16 bits.
- ``uint8`` quantization is lossy. Each float is represented by 8 bits.
The algorithm behind is standard scaling.
- ``fp16`` quantization is lossless, can be used widely. Each float is represented by 16 bits.
- ``uint8`` quantization is lossy. Each float is represented by 8 bits.
The algorithm behind is standard scaling.
the quantize type is stored and the blob is self-contained to recover the original numpy array
"""

the quantize type is stored and the blob is self-contained to recover the original numpy array
"""
def __init__(self, proto: 'jina_pb2.NdArrayProto' = None, quantize: str = None, *args, **kwargs):
super().__init__(proto, *args, **kwargs)
self.quantize = os.environ.get('JINA_ARRAY_QUANT', quantize)

Expand Down
10 changes: 6 additions & 4 deletions jina/types/ndarray/sparse/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

class BaseSparseNdArray(BaseNdArray):
"""
The base class for SparseNdArray.
The base class for :class:`SparseNdArray`.
Do not use this class directly. Subclass should be used.
Expand All @@ -28,17 +28,19 @@ def null_proto(self):
return jina_pb2.SparseNdArrayProto()

def sparse_constructor(self, indices: 'np.ndarray', values: 'np.ndarray', shape: List[int]) -> AnySparseNdArray:
""" Sparse NdArray constructor, must be implemented by subclass
"""
Sparse NdArray constructor, must be implemented by subclass.
:param indices: the indices of the sparse array
:param values: the values of the sparse array
:param shape: the shape of the dense array
:return:
:return: Sparse NdArray
"""
raise NotImplementedError

def sparse_parser(self, value: AnySparseNdArray) -> Dict[str, Union['np.ndarray', List[int]]]:
""" Parsing a Sparse NdArray to indices, values and shape, must be implemented by subclass
"""
Parse a Sparse NdArray to indices, values and shape, must be implemented by subclass.
:param value: the sparse ndarray
:return: a Dict with three entries {'indices': ..., 'values':..., 'shape':...}
Expand Down
3 changes: 2 additions & 1 deletion jina/types/ndarray/sparse/numpy.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@


class SparseNdArray(BaseSparseNdArray):
"""Numpy powered sparse ndarray, it uses nonzero.
"""
Numpy powered sparse ndarray, it uses nonzero.
.. note::
This always take a dense :class:`np.ndarray` and return a :class:`np.ndarray`.
Expand Down
16 changes: 8 additions & 8 deletions jina/types/ndarray/sparse/pytorch.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,20 +12,20 @@


class SparseNdArray(BaseSparseNdArray):
"""Pytorch powered sparse ndarray, i.e. FloatTensor
"""
Pytorch powered sparse ndarray, i.e. FloatTensor.
.. seealso::
https://pytorch.org/docs/stable/sparse.html
:param transpose_indices: in torch, the input to LongTensor is NOT a list of index tuples.
If you want to write your indices this way, you should transpose before passing them to the sparse constructor
.. note::
To comply with Tensorflow, `transpose_indices` is set to True by default
"""

def __init__(self, transpose_indices: bool = True, *args, **kwargs):
"""
:param transpose_indices: in torch, the input to LongTensor is NOT a list of index tuples.
If you want to write your indices this way, you should transpose before passing them to the sparse constructor
.. note::
To comply with Tensorflow, `transpose_indices` is set to True by default
"""
super().__init__(*args, **kwargs)
self.transpose_indices = transpose_indices

Expand Down
12 changes: 5 additions & 7 deletions jina/types/ndarray/sparse/scipy.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,22 +12,20 @@


class SparseNdArray(BaseSparseNdArray):
"""Scipy powered sparse ndarray
"""
Scipy powered sparse ndarray.
.. warning::
scipy only supports ndim=2
.. seealso::
https://docs.scipy.org/doc/scipy/reference/sparse.html
:param proto: the protobuf message, when not given then create a new one via :meth:`get_null_proto`
:param sp_format: the sparse format of the scipy matrix. one of 'coo', 'bsr', 'csc', 'csr'.
"""

def __init__(self, proto: 'jina_pb2.SparseNdArrayProto' = None, sp_format: str = 'coo', *args, **kwargs):
"""
:param sp_format: the sparse format of the scipy matrix. one of 'coo', 'bsr', 'csc', 'csr'
:param args:
:param kwargs:
"""
import scipy.sparse
super().__init__(proto, *args, **kwargs)
support_fmt = {'coo', 'bsr', 'csc', 'csr'}
Expand Down
4 changes: 2 additions & 2 deletions jina/types/ndarray/sparse/tensorflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@


class SparseNdArray(BaseSparseNdArray):
"""Tensorflow powered sparse ndarray, i.e. SparseTensor
"""
Tensorflow powered sparse ndarray, i.e. SparseTensor.
.. seealso::
https://www.tensorflow.org/api_docs/python/tf/sparse/SparseTensor
"""

def sparse_constructor(self, indices: 'np.ndarray', values: 'np.ndarray', shape: List[int]) -> 'SparseTensor':
Expand Down

0 comments on commit 5fb7ea8

Please sign in to comment.