-
Notifications
You must be signed in to change notification settings - Fork 1.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: add support for coordinator schemas (#28031)
Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>
- Loading branch information
1 parent
f7563e3
commit 9b30f8d
Showing
20 changed files
with
292 additions
and
73 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| <clickhouse> | ||
| <tcp_port>9001</tcp_port> | ||
| <remote_servers> | ||
| <posthog> | ||
| <shard> | ||
| <replica> | ||
| <host>clickhouse</host> | ||
| <port>9000</port> | ||
| </replica> | ||
| </shard> | ||
| </posthog> | ||
| <posthog_single_shard> | ||
| <shard> | ||
| <replica> | ||
| <host>clickhouse</host> | ||
| <port>9000</port> | ||
| </replica> | ||
| </shard> | ||
| </posthog_single_shard> | ||
| <posthog_migrations> | ||
| <shard> | ||
| <replica> | ||
| <host>clickhouse</host> | ||
| <port>9000</port> | ||
| </replica> | ||
| </shard> | ||
| <shard> | ||
| <replica> | ||
| <host>clickhouse-coordinator</host> | ||
| <port>9001</port> | ||
| </replica> | ||
| </shard> | ||
| </posthog_migrations> | ||
| </remote_servers> | ||
|
|
||
| <macros> | ||
| <shard>02</shard> | ||
| <replica>coord</replica> | ||
| <hostClusterType>online</hostClusterType> | ||
| <hostClusterRole>coordinator</hostClusterRole> | ||
| </macros> | ||
| </clickhouse> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| <clickhouse> | ||
| <tcp_port>9000</tcp_port> | ||
|
|
||
| <remote_servers> | ||
| <posthog> | ||
| <shard> | ||
| <replica> | ||
| <host>localhost</host> | ||
| <port>9000</port> | ||
| </replica> | ||
| </shard> | ||
| </posthog> | ||
| <posthog_single_shard> | ||
| <shard> | ||
| <replica> | ||
| <host>localhost</host> | ||
| <port>9000</port> | ||
| </replica> | ||
| </shard> | ||
| </posthog_single_shard> | ||
| <posthog_migrations> | ||
| <shard> | ||
| <replica> | ||
| <host>localhost</host> | ||
| <port>9000</port> | ||
| </replica> | ||
| </shard> | ||
| </posthog_migrations> | ||
| </remote_servers> | ||
|
|
||
| <macros> | ||
| <shard>01</shard> | ||
| <replica>ch1</replica> | ||
| <hostClusterType>online</hostClusterType> | ||
| <hostClusterRole>worker</hostClusterRole> | ||
| </macros> | ||
| </clickhouse> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| <clickhouse> | ||
| <tcp_port>9000</tcp_port> | ||
| <remote_servers> | ||
| <posthog> | ||
| <shard> | ||
| <replica> | ||
| <host>clickhouse</host> | ||
| <port>9000</port> | ||
| </replica> | ||
| </shard> | ||
| </posthog> | ||
| <posthog_single_shard> | ||
| <shard> | ||
| <replica> | ||
| <host>clickhouse</host> | ||
| <port>9000</port> | ||
| </replica> | ||
| </shard> | ||
| </posthog_single_shard> | ||
| <posthog_migrations> | ||
| <shard> | ||
| <replica> | ||
| <host>clickhouse</host> | ||
| <port>9000</port> | ||
| </replica> | ||
| </shard> | ||
| <shard> | ||
| <replica> | ||
| <host>clickhouse-coordinator</host> | ||
| <port>9001</port> | ||
| </replica> | ||
| </shard> | ||
| </posthog_migrations> | ||
| </remote_servers> | ||
|
|
||
| <macros> | ||
| <shard>01</shard> | ||
| <replica>ch1</replica> | ||
| <hostClusterType>online</hostClusterType> | ||
| <hostClusterRole>worker</hostClusterRole> | ||
| </macros> | ||
| </clickhouse> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,23 +1,28 @@ | ||
| from typing import Union | ||
| from collections.abc import Callable | ||
| import logging | ||
|
|
||
| from infi.clickhouse_orm import migrations | ||
|
|
||
| from posthog.clickhouse.client.execute import sync_execute | ||
| from posthog.clickhouse.client.connection import NodeRole | ||
| from posthog.clickhouse.cluster import get_cluster | ||
| from posthog.settings.data_stores import CLICKHOUSE_MIGRATIONS_CLUSTER | ||
|
|
||
| logger = logging.getLogger("migrations") | ||
|
|
||
| def run_sql_with_exceptions(sql: Union[str, Callable[[], str]], settings=None): | ||
|
|
||
| def run_sql_with_exceptions(sql: str, settings=None, node_role: NodeRole = NodeRole.WORKER): | ||
| """ | ||
| migrations.RunSQL does not raise exceptions, so we need to wrap it in a function that does. | ||
| node_role is set to WORKER by default to keep compatibility with the old migrations. | ||
| """ | ||
|
|
||
| if settings is None: | ||
| settings = {} | ||
| cluster = get_cluster(client_settings=settings, cluster=CLICKHOUSE_MIGRATIONS_CLUSTER) | ||
|
|
||
| def run_sql(database): | ||
| nonlocal sql | ||
| if callable(sql): | ||
| sql = sql() | ||
| sync_execute(sql, settings=settings) | ||
| def run_migration(): | ||
| if node_role == NodeRole.ALL: | ||
| logger.info(" Running migration on coordinators and workers") | ||
| return cluster.map_all_hosts(lambda client: client.execute(sql)).result() | ||
| else: | ||
| logger.info(f" Running migration on {node_role.value.lower()}s") | ||
| return cluster.map_hosts_by_role(lambda client: client.execute(sql), node_role=node_role).result() | ||
|
|
||
| return migrations.RunPython(run_sql) | ||
| return migrations.RunPython(lambda _: run_migration()) |
Oops, something went wrong.