From bf2e39a098469f379ff3aecacf5aef47074c4034 Mon Sep 17 00:00:00 2001 From: Mathias Fussenegger Date: Mon, 9 Sep 2024 11:02:58 +0200 Subject: [PATCH 1/6] Add some more type annotations to clients.py --- cr8/clients.py | 25 ++++++++++++++----------- setup.py | 3 ++- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/cr8/clients.py b/cr8/clients.py index 0c6de01..b301d38 100644 --- a/cr8/clients.py +++ b/cr8/clients.py @@ -7,17 +7,17 @@ import time from urllib.parse import urlparse, parse_qs, urlunparse from datetime import datetime, date -from typing import List, Union, Iterable, Dict +from typing import List, Union, Iterable, Dict, Optional, Any from decimal import Decimal from cr8.aio import asyncio # import via aio for uvloop setup try: import asyncpg except ImportError: - asyncpg = None + asyncpg = None # type: ignore try: - import simdjson + import simdjson # type: ignore dumps = simdjson.dumps except ImportError: dumps = json.dumps @@ -137,7 +137,7 @@ def _plain_or_callable(obj): return obj -def _date_or_none(d: str) -> str: +def _date_or_none(d: str) -> Optional[str]: """Return a date as if, if valid, otherwise None >>> _date_or_none('2017-02-27') @@ -152,7 +152,7 @@ def _date_or_none(d: str) -> str: return None -def _to_dsn(hosts): +def _to_dsn(hosts: str) -> str: """Convert a host URI into a dsn for aiopg. >>> _to_dsn('aiopg://myhostname:4242/mydb') @@ -177,7 +177,7 @@ def _to_dsn(hosts): host, port = netloc.split(':', maxsplit=1) except ValueError: host = netloc - port = 5432 + port = "5432" dbname = p.path[1:] if p.path else 'doc' dsn = f'postgres://{user_and_pw}@{host}:{port}/{dbname}' if p.query: @@ -185,7 +185,7 @@ def _to_dsn(hosts): return dsn -def _to_boolean(v): +def _to_boolean(v: str) -> bool: if str(v).lower() in ("true"): return True elif str(v).lower() in ("false"): @@ -194,7 +194,7 @@ def _to_boolean(v): raise ValueError('not a boolean value') -def _verify_ssl_from_first(hosts): +def _verify_ssl_from_first(hosts: List[str]) -> bool: """Check if SSL validation parameter is passed in URI >>> _verify_ssl_from_first(['https://myhost:4200/?verify_ssl=false']) @@ -295,7 +295,7 @@ def __exit__(self, *exs): self.close() -def _append_sql(host): +def _append_sql(host: str) -> str: """ Append `/_sql` to the host, dropping any query parameters. >>> _append_sql('http://n1:4200') @@ -316,12 +316,15 @@ def _append_sql(host): class HttpClient: - def __init__(self, hosts, conn_pool_limit=25, session_settings=None): + def __init__(self, + hosts: List[str], + conn_pool_limit=25, + session_settings: Optional[Dict[str, Any]] = None): self.hosts = hosts self.urls = itertools.cycle(list(map(_append_sql, hosts))) self.conn_pool_limit = conn_pool_limit self.is_cratedb = True - self._pools = {} + self._pools: Dict[str, asyncio.Queue] = {} self.session_settings = session_settings or {} async def _session(self, url): diff --git a/setup.py b/setup.py index 641f583..cdfae51 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,8 @@ 'asyncpg' ], extras_require={ - 'extra': ['uvloop', 'pysimdjson'] + 'extra': ['uvloop', 'pysimdjson'], + "dev": ["asyncpg-stubs", "mypy"] }, python_requires='>=3.7', classifiers=[ From f85e0943fb2046c56a2a9a43d42df595059a55e8 Mon Sep 17 00:00:00 2001 From: Mathias Fussenegger Date: Mon, 9 Sep 2024 11:04:22 +0200 Subject: [PATCH 2/6] Add editorconfig --- .editorconfig | 6 ++++++ cr8/engine.py | 2 +- specs/count_countries.json | 4 ++-- 3 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..b9e9010 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,6 @@ +root = true + +[*] +charset = utf-8 +indent_style = space +indent_size = 4 diff --git a/cr8/engine.py b/cr8/engine.py index fbe8170..a88cbe0 100644 --- a/cr8/engine.py +++ b/cr8/engine.py @@ -5,7 +5,7 @@ from cr8 import aio from cr8.metrics import Stats, get_sampler -from cr8.clients import client, HttpClient +from cr8.clients import client TimedStats = namedtuple('TimedStats', ['started', 'ended', 'stats']) diff --git a/specs/count_countries.json b/specs/count_countries.json index 1d21d91..271fa0a 100644 --- a/specs/count_countries.json +++ b/specs/count_countries.json @@ -15,8 +15,8 @@ ] }, "session_settings": { - "application_name": "my_app", - "timezone": "UTC" + "application_name": "my_app", + "timezone": "UTC" }, "queries": [{ "iterations": 1000, From ce439941184d6b4e74e572ada59e1db93e6ecd5f Mon Sep 17 00:00:00 2001 From: Mathias Fussenegger Date: Mon, 9 Sep 2024 11:25:33 +0200 Subject: [PATCH 3/6] Update CrateDB versions in test_reindex to work with newer JDKs --- tests/test_reindex.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/test_reindex.py b/tests/test_reindex.py index 49e8990..17366a7 100644 --- a/tests/test_reindex.py +++ b/tests/test_reindex.py @@ -19,21 +19,21 @@ def setUp(self): 'http.port': '44200-44250' } - def teardown(self): + def tearDown(self): for node in self._to_stop: node.stop() self._to_stop = [] shutil.rmtree(self.data_path, ignore_errors=True) def test_reindex(self): - crate_v3 = CrateNode( - crate_dir=get_crate('3.x.x'), + crate_v4 = CrateNode( + crate_dir=get_crate('4.x.x'), keep_data=True, settings=self.crate_settings ) - self._to_stop.append(crate_v3) - crate_v3.start() - with client(crate_v3.http_url) as c: + self._to_stop.append(crate_v4) + crate_v4.start() + with client(crate_v4.http_url) as c: aio.run(c.execute, "create table t (x int)") args = ( (1,), @@ -41,21 +41,21 @@ def test_reindex(self): (3,), ) aio.run(c.execute_many, "insert into t (x) values (?)", args) - crate_v3.stop() - self._to_stop.remove(crate_v3) + crate_v4.stop() + self._to_stop.remove(crate_v4) - crate_v4 = CrateNode( - crate_dir=get_crate('4.0.3'), + crate_v5 = CrateNode( + crate_dir=get_crate('5.0.3'), keep_data=True, settings=self.crate_settings ) - self._to_stop.append(crate_v4) - crate_v4.start() - reindex(hosts=crate_v4.http_url) - with client(crate_v4.http_url) as c: + self._to_stop.append(crate_v5) + crate_v5.start() + reindex(hosts=crate_v5.http_url) + with client(crate_v5.http_url) as c: result = aio.run(c.execute, "SELECT version FROM information_schema.tables WHERE table_name = 't'") version = result['rows'][0][0] - self.assertEqual(version, {'upgraded': None, 'created': '4.0.3'}) + self.assertEqual(version, {'upgraded': None, 'created': '5.0.3'}) cnt = aio.run(c.execute, 'SELECT count(*) FROM t')['rows'][0][0] self.assertEqual(cnt, 3) From 22c5aa445ce77ef1c876036d58ac71b1bd1019fa Mon Sep 17 00:00:00 2001 From: Mathias Fussenegger Date: Mon, 9 Sep 2024 11:37:33 +0200 Subject: [PATCH 4/6] Add debug configurations for run-crate and run-spec --- .vscode/launch.json | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..35d83df --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://raw.githubusercontent.com/mfussenegger/dapconfig-schema/master/dapconfig-schema.json", + "version": "0.2.0", + "configurations": [ + { + "type": "debugpy", + "request": "launch", + "name": "run_crate", + "module": "cr8.run_crate", + "args": ["latest-nightly"], + "console": "integratedTerminal" + }, + { + "type": "debugpy", + "request": "launch", + "name": "run_spec", + "module": "cr8.run_spec", + "args": ["specs/sample.toml", "localhost:4200"], + "console": "integratedTerminal" + } + ] +} From 0a2e14af02c0bff2979c7cdfd1d2b9b2704f46c6 Mon Sep 17 00:00:00 2001 From: Mathias Fussenegger Date: Mon, 9 Sep 2024 11:43:30 +0200 Subject: [PATCH 5/6] Update install instructions to account for PEP 668 --- README.rst | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index aea9d2e..54581d2 100644 --- a/README.rst +++ b/README.rst @@ -75,13 +75,10 @@ Python >= 3.7 is required to use the command line tools. Install them using `pip `_:: - python3.7 -m pip install --user cr8 - -This will install ``cr8`` into ``~/.local/bin``. Either use -``~/.local/bin/cr8`` to launch it or add ``~/.local/bin`` to your ``$PATH`` -environment variable. - + python3 -m venv venv + venv/bin/python -m pip install cr8 +This will install ``cr8`` into ``venv/bin`` An alternative is to download a single ``zipapp`` file from the `releases page `_. From 83433f257ce9994f2f0ecffb7356cb131715b829 Mon Sep 17 00:00:00 2001 From: Mathias Fussenegger Date: Mon, 9 Sep 2024 11:54:22 +0200 Subject: [PATCH 6/6] Update setup-python/java in GHA workflow --- .editorconfig | 3 +++ .github/workflows/main.yml | 17 +++++++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/.editorconfig b/.editorconfig index b9e9010..cb5cd42 100644 --- a/.editorconfig +++ b/.editorconfig @@ -4,3 +4,6 @@ root = true charset = utf-8 indent_style = space indent_size = 4 + +[yaml] +indent_size = 2 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f01b5ba..05eb093 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,15 +16,18 @@ jobs: python-version: [3.7, 3.8, 3.9, '3.10', '3.11', '3.12'] steps: - - uses: actions/checkout@master + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + cache: "pip" + cache-dependency-path: setup.py - name: Set up Java - uses: actions/setup-java@v1 + uses: actions/setup-java@v4 with: + distribution: "temurin" java-version: 11 - name: Install dependencies @@ -43,19 +46,21 @@ jobs: env: WACKLIG_TOKEN: ${{ secrets.WACKLIG_TOKEN }} run: | - curl -s https://raw.githubusercontent.com/pipifein/wacklig-uploader/master/wacklig.py | python - --token $WACKLIG_TOKEN || echo "Upload to wacklig failed" + curl -s https://raw.githubusercontent.com/pipifein/wacklig-uploader/master/wacklig.py | python - --token "$WACKLIG_TOKEN" || echo "Upload to wacklig failed" publish: name: Build & publish package to pypi needs: test runs-on: ubuntu-latest steps: - - uses: actions/checkout@master + - uses: actions/checkout@v4 - name: Set up python 3.7 - uses: actions/setup-python@v1 + uses: actions/setup-python@v5 with: python-version: 3.7 + cache: "pip" + cache-dependency-path: setup.py - name: Build package run: |