Merge pull request #148 from Altinity/customizations/22.3

Pre-release PR 22.3.6

cmake/autogenerated_versions.txt

@@ -2,11 +2,11 @@
 
 # NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION,
 # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
-SET(VERSION_REVISION 54460)
+SET(VERSION_REVISION 6)
 SET(VERSION_MAJOR 22)
 SET(VERSION_MINOR 3)
 SET(VERSION_PATCH 6)
 SET(VERSION_GITHASH 438b4a81f77c98b0dd959e19cfa4b62cabf549d6)
-SET(VERSION_DESCRIBE v22.3.6.5-lts)
-SET(VERSION_STRING 22.3.6.5)
+SET(VERSION_DESCRIBE v22.3.6.6.altinitystable)
+SET(VERSION_STRING 22.3.6.6.altinitystable)
 # end of autochange

cmake/version.cmake

@@ -19,5 +19,5 @@ set (VERSION_STRING_SHORT "${VERSION_MAJOR}.${VERSION_MINOR}")
 math (EXPR VERSION_INTEGER "${VERSION_PATCH} + ${VERSION_MINOR}*1000 + ${VERSION_MAJOR}*1000000")
 
 if(CLICKHOUSE_OFFICIAL_BUILD)
-    set(VERSION_OFFICIAL " (official build)")
+    set(VERSION_OFFICIAL " (Altinity build)")
 endif()

docker/images.json

@@ -1,164 +1,164 @@
 {
     "docker/packager/deb": {
-        "name": "clickhouse/deb-builder",
+        "name": "altinityinfra/deb-builder",
         "dependent": []
     },
     "docker/packager/binary": {
-        "name": "clickhouse/binary-builder",
+        "name": "altinityinfra/binary-builder",
         "dependent": [
             "docker/test/split_build_smoke_test",
             "docker/test/codebrowser"
         ]
     },
     "docker/test/compatibility/centos": {
-        "name": "clickhouse/test-old-centos",
+        "name": "altinityinfra/test-old-centos",
         "dependent": []
     },
     "docker/test/compatibility/ubuntu": {
-        "name": "clickhouse/test-old-ubuntu",
+        "name": "altinityinfra/test-old-ubuntu",
         "dependent": []
     },
     "docker/test/integration/base": {
-        "name": "clickhouse/integration-test",
+        "name": "altinityinfra/integration-test",
         "dependent": []
     },
     "docker/test/fuzzer": {
-        "name": "clickhouse/fuzzer",
+        "name": "altinityinfra/fuzzer",
         "dependent": []
     },
     "docker/test/performance-comparison": {
-        "name": "clickhouse/performance-comparison",
+        "name": "altinityinfra/performance-comparison",
         "dependent": []
     },
     "docker/test/util": {
-        "name": "clickhouse/test-util",
+        "name": "altinityinfra/test-util",
         "dependent": [
             "docker/test/base",
             "docker/test/fasttest"
         ]
     },
     "docker/test/stateless": {
-        "name": "clickhouse/stateless-test",
+        "name": "altinityinfra/stateless-test",
         "dependent": [
             "docker/test/stateful",
             "docker/test/unit"
         ]
     },
     "docker/test/stateful": {
-        "name": "clickhouse/stateful-test",
+        "name": "altinityinfra/stateful-test",
         "dependent": [
             "docker/test/stress"
         ]
     },
     "docker/test/unit": {
-        "name": "clickhouse/unit-test",
+        "name": "altinityinfra/unit-test",
         "dependent": []
     },
     "docker/test/stress": {
-        "name": "clickhouse/stress-test",
+        "name": "altinityinfra/stress-test",
         "dependent": []
     },
     "docker/test/split_build_smoke_test": {
-        "name": "clickhouse/split-build-smoke-test",
+        "name": "altinityinfra/split-build-smoke-test",
         "dependent": []
     },
     "docker/test/codebrowser": {
-        "name": "clickhouse/codebrowser",
+        "name": "altinityinfra/codebrowser",
         "dependent": []
     },
     "docker/test/integration/runner": {
         "only_amd64": true,
-        "name": "clickhouse/integration-tests-runner",
+        "name": "altinityinfra/integration-tests-runner",
         "dependent": []
     },
     "docker/test/testflows/runner": {
-        "name": "clickhouse/testflows-runner",
+        "name": "altinityinfra/testflows-runner",
         "dependent": []
     },
     "docker/test/fasttest": {
-        "name": "clickhouse/fasttest",
+        "name": "altinityinfra/fasttest",
         "dependent": []
     },
     "docker/test/style": {
-        "name": "clickhouse/style-test",
+        "name": "altinityinfra/style-test",
         "dependent": []
     },
     "docker/test/integration/s3_proxy": {
-        "name": "clickhouse/s3-proxy",
+        "name": "altinityinfra/s3-proxy",
         "dependent": []
     },
     "docker/test/integration/resolver": {
-        "name": "clickhouse/python-bottle",
+        "name": "altinityinfra/python-bottle",
         "dependent": []
     },
     "docker/test/integration/helper_container": {
-        "name": "clickhouse/integration-helper",
+        "name": "altinityinfra/integration-helper",
         "dependent": []
     },
     "docker/test/integration/mysql_golang_client": {
-        "name": "clickhouse/mysql-golang-client",
+        "name": "altinityinfra/mysql-golang-client",
         "dependent": []
     },
     "docker/test/integration/dotnet_client": {
-        "name": "clickhouse/dotnet-client",
+        "name": "altinityinfra/dotnet-client",
         "dependent": []
     },
     "docker/test/integration/mysql_java_client": {
-        "name": "clickhouse/mysql-java-client",
+        "name": "altinityinfra/mysql-java-client",
         "dependent": []
     },
     "docker/test/integration/mysql_js_client": {
-        "name": "clickhouse/mysql-js-client",
+        "name": "altinityinfra/mysql-js-client",
         "dependent": []
     },
     "docker/test/integration/mysql_php_client": {
-        "name": "clickhouse/mysql-php-client",
+        "name": "altinityinfra/mysql-php-client",
         "dependent": []
     },
     "docker/test/integration/postgresql_java_client": {
-        "name": "clickhouse/postgresql-java-client",
+        "name": "altinityinfra/postgresql-java-client",
         "dependent": []
     },
     "docker/test/integration/kerberos_kdc": {
         "only_amd64": true,
-        "name": "clickhouse/kerberos-kdc",
+        "name": "altinityinfra/kerberos-kdc",
         "dependent": []
     },
     "docker/test/base": {
-         "name": "clickhouse/test-base",
-         "dependent": [
+        "name": "altinityinfra/test-base",
+        "dependent": [
             "docker/test/stateless",
             "docker/test/integration/base",
             "docker/test/fuzzer",
             "docker/test/keeper-jepsen"
-         ]
+        ]
     },
     "docker/test/integration/kerberized_hadoop": {
         "only_amd64": true,
-        "name": "clickhouse/kerberized-hadoop",
+        "name": "altinityinfra/kerberized-hadoop",
         "dependent": []
     },
     "docker/test/sqlancer": {
-        "name": "clickhouse/sqlancer-test",
+        "name": "altinityinfra/sqlancer-test",
         "dependent": []
     },
     "docker/test/keeper-jepsen": {
-        "name": "clickhouse/keeper-jepsen-test",
+        "name": "altinityinfra/keeper-jepsen-test",
         "dependent": []
     },
     "docker/docs/builder": {
-        "name": "clickhouse/docs-builder",
+        "name": "altinityinfra/docs-builder",
         "dependent": [
             "docker/docs/check",
             "docker/docs/release"
         ]
     },
     "docker/docs/check": {
-        "name": "clickhouse/docs-check",
+        "name": "altinityinfra/docs-check",
         "dependent": []
     },
     "docker/docs/release": {
-        "name": "clickhouse/docs-release",
+        "name": "altinityinfra/docs-release",
         "dependent": []
     }
 }

docker/packager/binary/build.sh

@@ -19,6 +19,9 @@ ln -sf darwin-x86_64 build/cmake/toolchain/darwin-aarch64
 # export CCACHE_LOGFILE=/build/ccache.log
 # export CCACHE_DEBUG=1
 
+# https://stackoverflow.com/a/71940133
+git config --global --add safe.directory '*'
+
 mkdir -p build/build_docker
 cd build/build_docker
 rm -f CMakeCache.txt

docker/packager/packager

@@ -331,7 +331,7 @@ if __name__ == "__main__":
         args.output_dir = os.path.abspath(os.path.join(os.getcwd(), args.output_dir))
 
     image_type = "binary" if args.package_type == "performance" else args.package_type
-    image_name = "clickhouse/binary-builder"
+    image_name = "altinityinfra/binary-builder"
 
     if not os.path.isabs(args.clickhouse_repo_path):
         ch_root = os.path.abspath(os.path.join(os.getcwd(), args.clickhouse_repo_path))

docker/test/stateful/Dockerfile

@@ -1,6 +1,7 @@
 # rebuild in #33610
 # docker build -t clickhouse/stateful-test .
 ARG FROM_TAG=latest
+# TODO consider replacing clickhouse with altinityinfra dockerhub account
 FROM clickhouse/stateless-test:$FROM_TAG
 
 RUN apt-get update -y \

packages/clickhouse-client.yaml

@@ -4,8 +4,8 @@ name: "clickhouse-client"
 arch: "all"
 platform: "linux"
 version: "${CLICKHOUSE_VERSION_STRING}"
-vendor: "ClickHouse Inc."
-homepage: "https://clickhouse.com"
+vendor: "Altinity Inc."
+homepage: "https://altinity.com/"
 license: "Apache"
 section: "database"
 priority: "optional"

packages/clickhouse-common-static-dbg.yaml

@@ -4,8 +4,8 @@ name: "clickhouse-common-static-dbg"
 arch: "${DEB_ARCH}" # amd64, arm64
 platform: "linux"
 version: "${CLICKHOUSE_VERSION_STRING}"
-vendor: "ClickHouse Inc."
-homepage: "https://clickhouse.com"
+vendor: "Altinity Inc."
+homepage: "https://altinity.com/"
 license: "Apache"
 section: "database"
 priority: "optional"

packages/clickhouse-common-static.yaml

@@ -4,8 +4,8 @@ name: "clickhouse-common-static"
 arch: "${DEB_ARCH}" # amd64, arm64
 platform: "linux"
 version: "${CLICKHOUSE_VERSION_STRING}"
-vendor: "ClickHouse Inc."
-homepage: "https://clickhouse.com"
+vendor: "Altinity Inc."
+homepage: "https://altinity.com/"
 license: "Apache"
 section: "database"
 priority: "optional"

packages/clickhouse-keeper-dbg.yaml

@@ -4,8 +4,8 @@ name: "clickhouse-keeper-dbg"
 arch: "${DEB_ARCH}" # amd64, arm64
 platform: "linux"
 version: "${CLICKHOUSE_VERSION_STRING}"
-vendor: "ClickHouse Inc."
-homepage: "https://clickhouse.com"
+vendor: "Altinity Inc."
+homepage: "https://altinity.com/"
 license: "Apache"
 section: "database"
 priority: "optional"

packages/clickhouse-keeper.yaml

@@ -4,8 +4,8 @@ name: "clickhouse-keeper"
 arch: "${DEB_ARCH}" # amd64, arm64
 platform: "linux"
 version: "${CLICKHOUSE_VERSION_STRING}"
-vendor: "ClickHouse Inc."
-homepage: "https://clickhouse.com"
+vendor: "Altinity Inc."
+homepage: "https://altinity.com/"
 license: "Apache"
 section: "database"
 priority: "optional"

packages/clickhouse-server.yaml

@@ -4,8 +4,8 @@ name: "clickhouse-server"
 arch: "all"
 platform: "linux"
 version: "${CLICKHOUSE_VERSION_STRING}"
-vendor: "ClickHouse Inc."
-homepage: "https://clickhouse.com"
+vendor: "Altinity Inc."
+homepage: "https://altinity.com/"
 license: "Apache"
 section: "database"
 priority: "optional"

tests/ci/build_check.py

@@ -8,7 +8,7 @@
 import time
 from typing import List, Optional, Tuple
 
-from env_helper import REPO_COPY, TEMP_PATH, CACHES_PATH, IMAGES_PATH
+from env_helper import REPO_COPY, TEMP_PATH, CACHES_PATH, IMAGES_PATH, S3_BUILDS_BUCKET, CLICKHOUSE_STABLE_VERSION_SUFFIX
 from s3_helper import S3Helper
 from pr_info import PRInfo
 from version_helper import (
@@ -21,7 +21,7 @@
 from docker_pull_helper import get_image_with_version
 from tee_popen import TeePopen
 
-IMAGE_NAME = "clickhouse/binary-builder"
+IMAGE_NAME = "altinityinfra/binary-builder"
 
 
 def get_build_config(build_check_name: str, build_name: str) -> BuildConfig:
@@ -229,12 +229,12 @@ def main():
         log_url = ""
         for url in build_results:
             if "build_log.log" in url:
-                log_url = "https://s3.amazonaws.com/clickhouse-builds/" + url.replace(
+                log_url = f"https://s3.amazonaws.com/{S3_BUILDS_BUCKET}/" + url.replace(
                     "+", "%2B"
                 ).replace(" ", "%20")
             else:
                 build_urls.append(
-                    "https://s3.amazonaws.com/clickhouse-builds/"
+                    f"https://s3.amazonaws.com/{S3_BUILDS_BUCKET}/"
                     + url.replace("+", "%2B").replace(" ", "%20")
                 )
         success = len(build_urls) > 0
@@ -258,15 +258,19 @@ def main():
 
     logging.info("Got version from repo %s", version.string)
 
-    official_flag = pr_info.number == 0
-    version_type = "testing"
-    if "release" in pr_info.labels or "release-lts" in pr_info.labels:
-        version_type = "stable"
-        official_flag = True
+    official_flag = True
+    version._flavour = version_type = CLICKHOUSE_STABLE_VERSION_SUFFIX
+    # TODO (vnemkov): right now we'll use simplified version management:
+    # only update git hash and explicitly set stable version suffix.
+    # official_flag = pr_info.number == 0
+    # version_type = "testing"
+    # if "release" in pr_info.labels or "release-lts" in pr_info.labels:
+    #     version_type = CLICKHOUSE_STABLE_VERSION_SUFFIX
+    #     official_flag = True
 
     update_version_local(version, version_type)
 
-    logging.info("Updated local files with version")
+    logging.info(f"Updated local files with version : {version.string} / {version.describe}")
 
     logging.info("Build short name %s", build_name)
 

tests/ci/ccache_utils.py

@@ -5,6 +5,7 @@
 import sys
 import os
 import shutil
+from env_helper import S3_BUILDS_BUCKET
 from pathlib import Path
 
 import requests
@@ -71,7 +72,7 @@ def get_ccache_if_not_exists(
         for obj in objects:
             if ccache_name in obj:
                 logging.info("Found ccache on path %s", obj)
-                url = "https://s3.amazonaws.com/clickhouse-builds/" + obj
+                url = f"https://s3.amazonaws.com/{S3_BUILDS_BUCKET}/" + obj
                 compressed_cache = os.path.join(temp_path, os.path.basename(obj))
                 dowload_file_with_progress(url, compressed_cache)