feat: add Python 3.9 support (#1802)

* feat: add Python 3.9 support

.github/workflows/cd.yml

@@ -98,7 +98,7 @@ jobs:
       fail-fast: false
       matrix:
         tag_stage: ["", "-devel", "-daemon"]
-        tag_pyversion: ["-py37", "-py38"]
+        tag_pyversion: ["-py37", "-py38", "-py39"]
     steps:
       - uses: actions/checkout@v2
         with:
@@ -111,6 +111,8 @@ jobs:
 
           if [[ "${{ matrix.tag_pyversion }}" == "-py38" ]]; then
             echo "PY_VERSION=3.8" >> $GITHUB_ENV
+          elif [[ "${{ matrix.tag_pyversion }}" == "-py39" ]]; then
+            echo "PY_VERSION=3.9" >> $GITHUB_ENV
           else
             echo "PY_VERSION=3.7" >> $GITHUB_ENV
             echo "TAG_ALIAS=jinaai/jina:master${{ matrix.tag_stage }}" >> $GITHUB_ENV

Dockerfiles/debianx.Dockerfile

@@ -10,6 +10,7 @@ ARG PY_VERSION=3.7
 FROM python:${PY_VERSION}-slim AS jina_base
 
 ARG VCS_REF
+ARG PY_VERSION
 ARG BUILD_DATE
 ARG JINA_VERSION
 ARG PIP_TAG
@@ -27,8 +28,8 @@ LABEL org.opencontainers.image.created=${BUILD_DATE} \
       org.opencontainers.image.title="Jina" \
       org.opencontainers.image.description="Jina is the cloud-native neural search solution powered by state-of-the-art AI and deep learning technology"
 
-ENV JINA_BUILD_BASE_DEP="python3-grpcio" \
-    JINA_BUILD_DEVEL_DEP="build-essential gcc libc-dev python3-gevent libmagic1" \
+ENV JINA_COMPILERS="gcc libc-dev make" \
+    JINA_BUILD_DEVEL_DEP="python3-gevent libmagic1" \
     PYTHONPATH=$PYTHONPATH:/usr/lib/python${PY_VERSION}/dist-packages:/usr/local/lib/python${PY_VERSION}/site-packages:/usr/lib/python3/dist-packages:/usr/local/lib/python3/site-packages \
     JINA_VERSION=${JINA_VERSION} \
     JINA_VCS_VERSION=${VCS_REF} \
@@ -39,12 +40,13 @@ ENV JINA_BUILD_BASE_DEP="python3-grpcio" \
 
 COPY . /jina/
 
-RUN apt-get update && apt-get install --no-install-recommends -y ${JINA_BUILD_BASE_DEP} && \
-    apt-get autoremove && apt-get clean && rm -rf /var/lib/apt/lists/* && \
-    ln -s locale.h /usr/include/xlocale.h && \
+RUN ln -s locale.h /usr/include/xlocale.h && \
+    if [ "${PY_VERSION}" = "3.9" ]; then apt-get update && apt-get install --no-install-recommends -y ${JINA_COMPILERS}; fi && \
     cd /jina && \
     pip install . --compile --extra-index-url ${PIP_EXTRA_INDEX_URL} && \
     if [ -n "${PIP_TAG}" ]; then pip install ".[${PIP_TAG}]" --compile --extra-index-url $PIP_EXTRA_INDEX_URL; fi && \
+    if [ "${PY_VERSION}" = "3.9" ]; then apt-get remove -y --auto-remove ${JINA_COMPILERS}; fi && \
+    apt-get autoremove && apt-get clean && rm -rf /var/lib/apt/lists/* && \
     rm -rf /tmp/* && rm -rf /jina && rm /usr/include/xlocale.h
 
 ENTRYPOINT ["jina"]
@@ -54,10 +56,12 @@ FROM jina_base AS jina_devel
 COPY . /jina/
 
 RUN apt-get update && apt-get install --no-install-recommends -y ruby-dev build-essential && \
+    if [ "${PY_VERSION}" = "3.9" ]; then apt-get update && apt-get install --no-install-recommends -y ${JINA_COMPILERS}; fi && \
     apt-get install --no-install-recommends -y ${JINA_BUILD_DEVEL_DEP} && \
-    apt-get autoremove && apt-get clean && rm -rf /var/lib/apt/lists/* && \
     ln -s locale.h /usr/include/xlocale.h && cd /jina && \
     pip install .[devel] --compile --extra-index-url ${PIP_EXTRA_INDEX_URL} && \
+    if [ "${PY_VERSION}" = "3.9" ]; then apt-get remove -y --auto-remove ${JINA_COMPILERS}; fi && \
+    apt-get autoremove && apt-get clean && rm -rf /var/lib/apt/lists/* && \
     rm -rf /tmp/* && rm -rf /jina && rm /usr/include/xlocale.h
 
 ENTRYPOINT ["jina"]
@@ -66,11 +70,13 @@ FROM jina_base AS jina_daemon
 
 COPY . /jina/
 
-RUN apt-get update && apt-get install --no-install-recommends -y ruby-dev build-essential && \
-    apt-get autoremove && apt-get clean && rm -rf /var/lib/apt/lists/* && \
+RUN apt-get update && apt-get install --no-install-recommends -y ruby-dev ${JINA_COMPILERS} && \
+    if [ "${PY_VERSION}" = "3.9" ]; then apt-get update && apt-get install --no-install-recommends -y ${JINA_COMPILERS}; fi && \
     gem install fluentd --no-doc && \
     ln -s locale.h /usr/include/xlocale.h && cd /jina && \
     pip install .[daemon] --compile --extra-index-url ${PIP_EXTRA_INDEX_URL} && \
+    if [ "${PY_VERSION}" = "3.9" ]; then apt-get remove -y --auto-remove ruby-dev ${JINA_COMPILERS}; fi && \
+    apt-get autoremove && apt-get clean && rm -rf /var/lib/apt/lists/* && \
     rm -rf /tmp/* && rm -rf /jina && rm /usr/include/xlocale.h
 
 ENTRYPOINT ["jinad"]

README.md

@@ -6,7 +6,7 @@ An easier way to build neural search in the cloud
 </p>
 <p align=center>
 <a href="#license"><img src="https://github.com/jina-ai/jina/blob/master/.github/badges/license-badge.svg?raw=true" alt="Jina" title="Jina is licensed under Apache-2.0"></a>
-<a href="https://pypi.org/project/jina/"><img src="https://github.com/jina-ai/jina/blob/master/.github/badges/python-badge.svg?raw=true" alt="Python 3.7 3.8" title="Jina supports Python 3.7 and above"></a>
+<a href="https://pypi.org/project/jina/"><img src="https://github.com/jina-ai/jina/blob/master/.github/badges/python-badge.svg?raw=true" alt="Python 3.7 3.8 3.9" title="Jina supports Python 3.7 and above"></a>
 <a href="https://pypi.org/project/jina/"><img src="https://img.shields.io/pypi/v/jina?color=%23099cec&amp;label=PyPI&amp;logo=pypi&amp;logoColor=white" alt="PyPI"></a>
 <a href="https://hub.docker.com/r/jinaai/jina/tags"><img src="https://img.shields.io/docker/v/jinaai/jina?color=%23099cec&amp;label=Docker&amp;logo=docker&amp;logoColor=white&amp;sort=semver" alt="Docker Image Version (latest semver)"></a>
 <a href="https://github.com/jina-ai/jina/actions?query=workflow%3ACI"><img src="https://github.com/jina-ai/jina/workflows/CI/badge.svg" alt="CI"></a>
@@ -38,7 +38,7 @@ Jina is a deep learning-powered search framework for building <strong>cross-/mul
 
 ## Installation
 
-| 📦<br><sub><sup>x86/64,arm/v6,v7,[v8 (Apple M1)](https://github.com/jina-ai/jina/issues/1781)</sup></sub> | On Linux/macOS & Python 3.7/3.8 | Docker Users|
+| 📦<br><sub><sup>x86/64,arm/v6,v7,[v8 (Apple M1)](https://github.com/jina-ai/jina/issues/1781)</sup></sub> | On Linux/macOS & Python 3.7/3.8/[3.9](https://github.com/jina-ai/jina/issues/1801) | Docker Users|
 | --- | --- | --- |
 | Standard | `pip install -U jina` | `docker run jinaai/jina:latest` |
 | <sub>Daemon</sub> | <sub>`pip install -U "jina[daemon]"`</sub> | <sub>`docker run --network=host jinaai/jina:latest-daemon`</sub> |

RELEASE.md

@@ -68,6 +68,7 @@ jinaai/jina:{version}{python_version}{extra}
 - `{python_version}`: The Python version of the image. Possible values: 
     - ` `, `-py37`: Python 3.7;
     - `-py38` for Python 3.8;
+    - `-py39` for Python 3.9;
 - `{extra}`: the extra dependency installed along with Jina. Possible values:
     - ` `: Jina is installed inside the image via `pip install jina`;
     - `-devel`: Jina is installed inside the image via `pip install jina[devel]`;

docs/chapters/install/os/on-x.md

@@ -10,7 +10,7 @@ Yes! Since `0.8.6` Jina runs smoothly on Jupyter Lab/Notebook. For better experi
 
 ## Google Colab
 
-Jina requires Python 3.7 or 3.8. At the time of writing Google Colab runs Python 3.6.7 so Jina can't be installed via `pip`.
+Jina requires Python 3.7/3.8/3.9. At the time of writing Google Colab runs Python 3.6.7 so Jina can't be installed via `pip`.
 
 ## Fedora
 

jina/__init__.py

@@ -16,8 +16,24 @@
 import sys as _sys
 import types as _types
 
-if _sys.version_info < (3, 7, 0):
-    raise OSError(f'Jina requires Python 3.7 and above, but yours is {_sys.version_info}')
+from google.protobuf.internal import api_implementation as _api_implementation
+
+if _api_implementation._default_implementation_type != 'cpp':
+    import warnings as _warnings
+
+    _warnings.warn('''
+    You are using Python protobuf backend, not the C++ version, which is much faster.
+    
+    This is often due to C++ implementation failed to compile while installing Protobuf
+    - You are using in Python 3.9 (https://github.com/jina-ai/jina/issues/1801)
+    - You are using on architecture other than x86_64/armv6/armv7
+    - You installation is broken, try `pip install --force protobuf`
+    - You have C++ backend but you shut it down, try `export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp`
+    
+    ''', RuntimeWarning)
+
+if _sys.version_info < (3, 7, 0) or _sys.version_info >= (3, 10, 0):
+    raise OSError(f'Jina requires Python 3.7/3.8/3.9, but yours is {_sys.version_info}')
 
 if _sys.version_info >= (3, 8, 0) and _platform.system() == 'Darwin':
     # temporary fix for python 3.8 on macos where the default start is set to "spawn"

jina/types/sets/document.py

@@ -3,7 +3,13 @@
 from typing import Union, Sequence, Iterable, Tuple
 
 import numpy as np
-from google.protobuf.pyext._message import RepeatedCompositeContainer
+
+try:
+    # when protobuf using Cpp backend
+    from google.protobuf.pyext._message import RepeatedCompositeContainer as RepeatedContainer
+except:
+    # when protobuf using Python backend
+    from google.protobuf.internal.containers import RepeatedCompositeFieldContainer as RepeatedContainer
 
 from ...proto.jina_pb2 import DocumentProto
 
@@ -19,7 +25,7 @@ class DocumentSet(MutableSequence):
     a generator but ALSO modify it, count it, get item, or union two 'DocumentSet's using the '+' and '+=' operators.
     """
 
-    def __init__(self, docs_proto: Union['RepeatedCompositeContainer', Sequence['Document']]):
+    def __init__(self, docs_proto: Union['RepeatedContainer', Sequence['Document']]):
         super().__init__()
         self._docs_proto = docs_proto
         self._docs_map = {}
@@ -91,7 +97,7 @@ def clear(self):
 
     def reverse(self):
         """In-place reverse the sequence """
-        if isinstance(self._docs_proto, RepeatedCompositeContainer):
+        if isinstance(self._docs_proto, RepeatedContainer):
             size = len(self._docs_proto)
             hi_idx = size - 1
             for i in range(int(size / 2)):

jina/types/sets/querylang.py

@@ -1,7 +1,14 @@
 from collections.abc import MutableSequence
 from typing import Iterable, Union, Dict
 
-from google.protobuf.pyext._message import RepeatedCompositeContainer
+
+try:
+    # when protobuf using Cpp backend
+    from google.protobuf.pyext._message import RepeatedCompositeContainer as RepeatedContainer
+except:
+    # when protobuf using Python backend
+    from google.protobuf.internal.containers import RepeatedCompositeFieldContainer as RepeatedContainer
+
 
 from ..querylang import QueryLang
 from ...helper import typename
@@ -18,7 +25,7 @@ class QueryLangSet(MutableSequence):
     a generator but ALSO modify it, count it, get item.
     """
 
-    def __init__(self, querylang_protos: 'RepeatedCompositeContainer'):
+    def __init__(self, querylang_protos: 'RepeatedContainer'):
         super().__init__()
         self._querylangs_proto = querylang_protos
         self._querylangs_map = {}

setup.py

@@ -8,13 +8,16 @@
 
 PY37 = 'py37'
 PY38 = 'py38'
+PY39 = 'py39'
 
-if sys.version_info >= (3, 8, 0):
+if sys.version_info >= (3, 10, 0) or sys.version_info < (3, 7, 0):
+    raise OSError(f'Jina requires Python 3.7/3.8/3.9, but yours is {sys.version}')
+elif sys.version_info >= (3, 9, 0):
+    py_tag = PY39
+elif sys.version_info >= (3, 8, 0):
     py_tag = PY38
 elif sys.version_info >= (3, 7, 0):
     py_tag = PY37
-else:
-    raise OSError(f'Jina requires Python 3.7 and above, but yours is {sys.version}')
 
 try:
     pkg_name = 'jina'