Merge upstream

.github/workflows/python-publish.yml

@@ -1,9 +1,9 @@
 name: Upload to PIP
 
-# Controls when the action will run. 
+# Controls when the action will run.
 on:
   # Triggers the workflow when a release is created
-  release: 
+  release:
     types: [created]
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
@@ -18,10 +18,10 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       # Sets up python
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
           python-version: 3.9
 

.github/workflows/python-test.yml

@@ -16,12 +16,12 @@ jobs:
       fail-fast: false
       matrix:
         # Lint on smallest and largest active versions
-        python-version: [3.6, 3.9]
+        python-version: ["3.9", "3.10"]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
@@ -45,12 +45,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9, pypy3]
+        python-version: ["3.9", "3.10", "pypy3.10"]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
@@ -72,12 +72,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.9]
+        python-version: ["3.9"]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

CHANGELOG.md

@@ -1,6 +1,18 @@
 Change Log
 ==========
 
+v2.3.0
+------
+Merges upstream changes:
+
+- Fix pagination for models with alias columns
+- Add `QuerySet.model` to support django-rest-framework 3
+- Improve support of ClickHouse v21.9 (mangototango)
+- Ignore non-numeric parts in ClickHouse version (mangototango)
+- Fix precedence of ~ operator in Q objects	(mangototango)
+- Support for adding a column to the beginning of a table (meanmail)
+- Add stddevPop and stddevSamp functions (k.peskov)
+
 v2.2.2
 ------
 - Unpined requirements to enhance compatability
@@ -216,5 +228,3 @@ v0.7.0
 v0.6.3
 ------
 - Python 3 support
-
-

clickhouse_orm/database.py

@@ -55,6 +55,14 @@ def __init__(self, message):
         """,
             re.VERBOSE | re.DOTALL,
         ),
+        # ClickHouse v21+
+        re.compile(
+            r"""
+            Code:\ (?P<code>\d+).
+            \ (?P<type1>[^ \n]+):\ (?P<msg>.+)
+        """,
+            re.VERBOSE | re.DOTALL,
+        ),
     )
 
     @classmethod
@@ -124,14 +132,20 @@ def __init__(
         self.db_exists = self._is_existing_database()
         if readonly:
             if not self.db_exists:
-                raise DatabaseException("Database does not exist, and cannot be created under readonly connection")
+                raise DatabaseException(
+                    "Database does not exist, and cannot be created under readonly connection"
+                )
             self.connection_readonly = self._is_connection_readonly()
             self.readonly = True
         elif autocreate and not self.db_exists:
             self.create_database()
         self.server_version = self._get_server_version()
         # Versions 1.1.53981 and below don't have timezone function
-        self.server_timezone = self._get_server_timezone() if self.server_version > (1, 1, 53981) else pytz.utc
+        self.server_timezone = (
+            self._get_server_timezone()
+            if self.server_version > (1, 1, 53981)
+            else pytz.utc
+        )
         # Versions 19.1.16 and above support codec compression
         self.has_codec_support = self.server_version >= (19, 1, 16)
         # Version 19.0 and above support LowCardinality
@@ -158,7 +172,9 @@ def create_table(self, model_class):
         if model_class.is_system_model():
             raise DatabaseException("You can't create system table")
         if model_class.engine is None:
-            raise DatabaseException("%s class must define an engine" % model_class.__name__)
+            raise DatabaseException(
+                "%s class must define an engine" % model_class.__name__
+            )
         self._send(model_class.create_table_sql(self))
 
     def drop_table(self, model_class):
@@ -229,7 +245,9 @@ def insert(self, model_instances, batch_size=1000):
         if first_instance.is_read_only() or first_instance.is_system_model():
             raise DatabaseException("You can't insert into read only and system tables")
 
-        fields_list = ",".join(["`%s`" % name for name in first_instance.fields(writable=True)])
+        fields_list = ",".join(
+            ["`%s`" % name for name in first_instance.fields(writable=True)]
+        )
         fmt = "TSKV" if model_class.has_funcs_as_defaults() else "TabSeparated"
         query = "INSERT INTO $table (%s) FORMAT %s\n" % (fields_list, fmt)
 
@@ -289,11 +307,15 @@ def select(self, query, model_class=None, settings=None):
         lines = r.iter_lines()
         field_names = parse_tsv(next(lines))
         field_types = parse_tsv(next(lines))
-        model_class = model_class or ModelBase.create_ad_hoc_model(zip(field_names, field_types))
+        model_class = model_class or ModelBase.create_ad_hoc_model(
+            zip(field_names, field_types)
+        )
         for line in lines:
             # skip blank line left by WITH TOTALS modifier
             if line:
-                yield model_class.from_tsv(line, field_names, self.server_timezone, self)
+                yield model_class.from_tsv(
+                    line, field_names, self.server_timezone, self
+                )
 
     def raw(self, query, settings=None, stream=False):
         """
@@ -306,7 +328,15 @@ def raw(self, query, settings=None, stream=False):
         query = self._substitute(query, None)
         return self._send(query, settings=settings, stream=stream).text
 
-    def paginate(self, model_class, order_by, page_num=1, page_size=100, conditions=None, settings=None):
+    def paginate(
+        self,
+        model_class,
+        order_by,
+        page_num=1,
+        page_size=100,
+        conditions=None,
+        settings=None,
+    ):
         """
         Selects records and returns a single page of model instances.
 
@@ -330,7 +360,8 @@ def paginate(self, model_class, order_by, page_num=1, page_size=100, conditions=
         elif page_num < 1:
             raise ValueError("Invalid page number: %d" % page_num)
         offset = (page_num - 1) * page_size
-        query = "SELECT * FROM $table"
+        query = "SELECT {} FROM $table".format(", ".join(model_class.fields().keys()))
+
         if conditions:
             if isinstance(conditions, Q):
                 conditions = conditions.to_sql(model_class)
@@ -367,7 +398,9 @@ def migrate(self, migrations_package_name, up_to=9999):
             self.insert(
                 [
                     MigrationHistory(
-                        package_name=migrations_package_name, module_name=name, applied=datetime.date.today()
+                        package_name=migrations_package_name,
+                        module_name=name,
+                        applied=datetime.date.today(),
                     )
                 ]
             )
@@ -378,7 +411,10 @@ def _get_applied_migrations(self, migrations_package_name):
         from .migrations import MigrationHistory
 
         self.create_table(MigrationHistory)
-        query = "SELECT module_name from $table WHERE package_name = '%s'" % migrations_package_name
+        query = (
+            "SELECT module_name from $table WHERE package_name = '%s'"
+            % migrations_package_name
+        )
         query = self._substitute(query, MigrationHistory)
         return set(obj.module_name for obj in self.select(query))
 
@@ -388,7 +424,9 @@ def _send(self, data, settings=None, stream=False):
             if self.log_statements:
                 logger.info(data)
         params = self._build_params(settings)
-        r = self.request_session.post(self.db_url, params=params, data=data, stream=stream, timeout=self.timeout)
+        r = self.request_session.post(
+            self.db_url, params=params, data=data, stream=stream, timeout=self.timeout
+        )
         if r.status_code != 200:
             raise ServerError(r.text)
         return r
@@ -413,7 +451,10 @@ def _substitute(self, query, model_class=None):
                 if model_class.is_system_model():
                     mapping["table"] = "`system`.`%s`" % model_class.table_name()
                 else:
-                    mapping["table"] = "`%s`.`%s`" % (self.db_name, model_class.table_name())
+                    mapping["table"] = "`%s`.`%s`" % (
+                        self.db_name,
+                        model_class.table_name(),
+                    )
             query = Template(query).safe_substitute(mapping)
         return query
 
@@ -432,10 +473,12 @@ def _get_server_version(self, as_tuple=True):
         except ServerError as e:
             logger.exception("Cannot determine server version (%s), assuming 1.1.0", e)
             ver = "1.1.0"
-        return tuple(int(n) for n in ver.split(".")) if as_tuple else ver
+        return tuple(int(n) for n in ver.split(".") if n.isdigit()) if as_tuple else ver
 
     def _is_existing_database(self):
-        r = self._send("SELECT count() FROM system.databases WHERE name = '%s'" % self.db_name)
+        r = self._send(
+            "SELECT count() FROM system.databases WHERE name = '%s'" % self.db_name
+        )
         return r.text.strip() == "1"
 
     def _is_connection_readonly(self):