Merge pull request #888 from datajoint/release013

Add release 0.13 features and bug fixes

Author: dimitri-yatsenko

.coveragerc

@@ -3,3 +3,4 @@ branch = False
 source = datajoint
 
 [report]
+show_missing = True

.github/workflows/development.yaml

@@ -21,8 +21,6 @@ jobs:
             mysql_ver: "5.7"
           - py_ver: "3.6"
             mysql_ver: "5.7"
-          - py_ver: "3.5"
-            mysql_ver: "5.7"
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{matrix.py_ver}}
@@ -38,7 +36,7 @@ jobs:
       - name: Run primary tests
         env:
           UID: "1001"
-          GID: "116"
+          GID: "121"
           PY_VER: ${{matrix.py_ver}}
           MYSQL_VER: ${{matrix.mysql_ver}}
           ALPINE_VER: "3.10"
@@ -50,4 +48,4 @@ jobs:
       - name: Run style tests
         run: |
           flake8 --ignore=E121,E123,E126,E226,E24,E704,W503,W504,E722,F401,W605 datajoint \
-                 --count --max-complexity=62 --max-line-length=127 --statistics
+                 --count --max-complexity=62 --max-line-length=127 --statistics

.gitignore

@@ -24,4 +24,5 @@ notebook
 .vscode
 __main__.py
 jupyter_custom.js
-apk_requirements.txt
+apk_requirements.txt
+.eggs

CHANGELOG.md

@@ -1,5 +1,22 @@
 ## Release notes
 
+### 0.13.0 -- Mar 24, 2021
+* Re-implement query transpilation into SQL, fixing issues (#386, #449, #450, #484). PR #754
+* Re-implement cascading deletes for better performance. PR #839.
+* Add table method `.update1` to update a row in the table with new values PR #763
+* Python datatypes are now enabled by default in blobs (#761). PR #785
+* Added permissive join and restriction operators `@` and `^` (#785) PR #754
+* Support DataJoint datatype and connection plugins (#715, #729) PR 730, #735
+* Add `dj.key_hash` alias to `dj.hash.key_hash`
+* Default enable_python_native_blobs to True
+* Bugfix - Regression error on joins with same attribute name (#857) PR #878
+* Bugfix - Error when `fetch1('KEY')` when `dj.config['fetch_format']='frame'` set (#876) PR #880, #878
+* Bugfix - Error when cascading deletes in tables with many, complex keys (#883, #886) PR #839
+* Add deprecation warning for `_update`. PR #889
+* Add `purge_query_cache` utility. PR #889
+* Add tests for query caching and permissive join and restriction. PR #889
+* Drop support for Python 3.5
+
 ### 0.12.9 -- Mar 12, 2021
 * Fix bug with fetch1 with `dj.config['fetch_format']="frame"`. (#876) PR #880
 

LNX-docker-compose.yml

@@ -1,4 +1,4 @@
-# docker-compose -f LNX-docker-compose.yml --env-file LNX.env up --build --exit-code-from app
+# docker-compose -f LNX-docker-compose.yml --env-file LNX.env up --exit-code-from app --build
 version: '2.2'
 x-net: &net
   networks:
@@ -32,7 +32,7 @@ services:
       interval: 1s
   fakeservices.datajoint.io:
     <<: *net
-    image: raphaelguzman/nginx:v0.0.13
+    image: datajoint/nginx:v0.0.16
     environment:
     - ADD_db_TYPE=DATABASE
     - ADD_db_ENDPOINT=db:3306
@@ -72,14 +72,17 @@ services:
       - COVERALLS_SERVICE_NAME
       - COVERALLS_REPO_TOKEN
     working_dir: /src
-    command: >
-      /bin/sh -c
-       "
-        pip install --user nose nose-cov coveralls .;
-        pip freeze | grep datajoint;
-        nosetests -vsw tests --with-coverage --cover-package=datajoint && coveralls;
-        # jupyter notebook;
-       "
+    command:
+      - sh
+      - -c
+      - |
+        set -e
+        pip install --user -r test_requirements.txt
+        pip install -e .
+        pip freeze | grep datajoint
+        nosetests -vsw tests --with-coverage --cover-package=datajoint
+        coveralls
+        # jupyter notebook
     # ports:
     #   - "8888:8888"
     user: ${UID}:${GID}

README.md

@@ -32,9 +32,10 @@ Some Python datatypes such as dicts were coerced into numpy recarrays and then f
 However, since some Python types were coerced into MATLAB types, old blobs and new blobs may now be fetched as different types of objects even if they were inserted the same way. 
 For example, new `dict` objects will be returned as `dict` while the same types of objects inserted with `datajoint 0.11` will be recarrays.
 
-Since this is a big change, we chose to disable full blob support by default as a temporary precaution, which will be removed in version 0.13.
+Since this is a big change, we chose to temporarily disable this feature by default in DataJoint for Python 0.12.x, allowing users to adjust their code if necessary. 
+From 13.x, the flag will default to True (on), and will ultimately be removed when corresponding decode support for the new format is added to datajoint-matlab (see: datajoint-matlab #222, datajoint-python #765).
 
-You may enable it by setting the `enable_python_native_blobs` flag in `dj.config`. 
+The flag is configured by setting the `enable_python_native_blobs` flag in `dj.config`. 
 
 ```python
 import datajoint as dj
@@ -68,7 +69,7 @@ as structured arrays, whereas new record inserted in DataJoint 0.12 with
 appropriate native python type (dict, etc).  
 Furthermore, DataJoint for MATLAB does not yet support unpacking native Python datatypes.
 
-With `dj.config["enable_python_native_blobs"]` set to `False` (default), 
+With `dj.config["enable_python_native_blobs"]` set to `False`, 
 any attempt to insert any datatype other than a numpy array will result in an exception.
 This is meant to get users to read this message in order to allow proper testing
 and migration of pre-0.12 pipelines to 0.12 in a safe manner.

datajoint.pub

@@ -0,0 +1,6 @@
+-----BEGIN PUBLIC KEY-----
+MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDUMOo2U7YQ1uOrKU/IreM3AQP2
+AXJC3au+S9W+dilxHcJ3e98bRVqrFeOofcGeRPoNc38fiLmLDUiBskJeVrpm29Wo
+AkH6yhZWk1o8NvGMhK4DLsJYlsH6tZuOx9NITKzJuOOH6X1I5Ucs7NOSKnmu7g5g
+WTT5kCgF5QAe5JN8WQIDAQAB
+-----END PUBLIC KEY-----

datajoint/__init__.py

@@ -15,7 +15,7 @@
 """
 
 __author__ = "DataJoint Contributors"
-__date__ = "February 7, 2019"
+__date__ = "November 7, 2020"
 __all__ = ['__author__', '__version__',
            'config', 'conn', 'Connection',
            'Schema', 'schema', 'VirtualModule', 'create_virtual_module',
@@ -24,7 +24,7 @@
            'Not', 'AndList', 'U', 'Diagram', 'Di', 'ERD',
            'set_password', 'kill',
            'MatCell', 'MatStruct', 'AttributeAdapter',
-           'errors', 'DataJointError', 'key']
+           'errors', 'DataJointError', 'key', 'key_hash']
 
 from .version import __version__
 from .settings import config
@@ -38,6 +38,7 @@
 from .admin import set_password, kill
 from .blob import MatCell, MatStruct
 from .fetch import key
+from .hash import key_hash
 from .attribute_adapter import AttributeAdapter
 from . import errors
 from .errors import DataJointError

datajoint/attribute_adapter.py

@@ -1,5 +1,6 @@
 import re
 from .errors import DataJointError, _support_adapted_types
+from .plugin import type_plugins
 
 
 class AttributeAdapter:
@@ -38,10 +39,11 @@ def get_adapter(context, adapter_name):
         raise DataJointError('Support for Adapted Attribute types is disabled.')
     adapter_name = adapter_name.lstrip('<').rstrip('>')
     try:
-        adapter = context[adapter_name]
+        adapter = (context[adapter_name] if adapter_name in context
+                   else type_plugins[adapter_name]['object'].load())
     except KeyError:
         raise DataJointError(
-            "Attribute adapter '{adapter_name}' is not defined.".format(adapter_name=adapter_name)) from None
+            "Attribute adapter '{adapter_name}' is not defined.".format(adapter_name=adapter_name))
     if not isinstance(adapter, AttributeAdapter):
         raise DataJointError(
             "Attribute adapter '{adapter_name}' must be an instance of datajoint.AttributeAdapter".format(

datajoint/autopopulate.py

@@ -7,7 +7,6 @@
 from tqdm import tqdm
 from .expression import QueryExpression, AndList
 from .errors import DataJointError, LostConnectionError
-from .table import FreeTable
 import signal
 
 # noinspection PyExceptionInherit,PyCallingNonCallable
@@ -41,7 +40,7 @@ def _rename_attributes(table, props):
             parents = self.target.parents(primary=True, as_objects=True, foreign_key_info=True)
             if not parents:
                 raise DataJointError(
-                    'A relation must have primary dependencies for auto-populate to work') from None
+                    'A relation must have primary dependencies for auto-populate to work')
             self._key_source = _rename_attributes(*parents[0])
             for q in parents[1:]:
                 self._key_source *= _rename_attributes(*q)
@@ -58,15 +57,16 @@ def make(self, key):
     @property
     def target(self):
         """
-        relation to be populated.
-        Typically, AutoPopulate are mixed into a Relation object and the target is self.
+        :return: table to be populated.
+        In the typical case, dj.AutoPopulate is mixed into a dj.Table class by inheritance and the target is self.
         """
         return self
 
     def _job_key(self, key):
         """
         :param key:  they key returned for the job from the key source
         :return: the dict to use to generate the job reservation hash
+        This method allows subclasses to control the job reservation granularity.
         """
         return key
 
@@ -136,7 +136,7 @@ def handler(signum, frame):
 
         make = self._make_tuples if hasattr(self, '_make_tuples') else self.make
 
-        for key in (tqdm(keys) if display_progress else keys):
+        for key in (tqdm(keys, desc=self.__class__.__name__) if display_progress else keys):
             if max_calls is not None and call_count >= max_calls:
                 break
             if not reserve_jobs or jobs.reserve(self.target.table_name, self._job_key(key)):

datajoint/blob.py

@@ -11,11 +11,10 @@
 import uuid
 import numpy as np
 from .errors import DataJointError
-from .utils import OrderedDict
 from .settings import config
 
 
-mxClassID = OrderedDict((
+mxClassID = dict((
     # see http://www.mathworks.com/help/techdoc/apiref/mxclassid.html
     ('mxUNKNOWN_CLASS', None),
     ('mxCELL_CLASS', None),
@@ -346,8 +345,8 @@ def pack_set(self, t):
             len_u64(it) + it for it in (self.pack_blob(i) for i in t))
 
     def read_dict(self):
-        return OrderedDict((self.read_blob(self.read_value()), self.read_blob(self.read_value()))
-                           for _ in range(self.read_value()))
+        return dict((self.read_blob(self.read_value()), self.read_blob(self.read_value()))
+                    for _ in range(self.read_value()))
 
     def pack_dict(self, d):
         return b"\4" + len_u64(d) + b"".join(