Implement native calls from Python Driver to Rust Driver (#304)

## What is the goal of this PR?

We replace the direct implementation of the client side of the TypeDB
protocol in Python with native calls to the Rust client over FFI.

## What are the changes implemented in this PR?

- merge `TypeDBClient` and `ClusterClient` into single `Client`,
following the protocol merge;
- remove `Remote` variants of `Concept`s; Concept API calls now require
a `Transaction` to be provided;
- make `Value` "generic" over its value type, which allows us to
collapse all the `Attribute.<Type>`s into a single `Attribute` class;
- associate each top-level class (`Client`, `Session`, `Concept`, etc.)
with its native representation on the Rust side;
- replace Python method implementations with calls using the native
representation over FFI.

Author: dmikhalin

.bazelversion

@@ -1 +1 @@
-5.1.1
+6.2.0

BUILD

@@ -34,14 +34,27 @@ load("@vaticle_dependencies//distribution:deployment.bzl", "deployment")
 load(":deployment.bzl", github_deployment = "deployment")
 
 
+genrule(
+    name = "native-client-wrapper",
+    outs = ["typedb/native_client_wrapper.py"],
+    srcs = ["@vaticle_typedb_driver_java//rust:native_client_python_wrapper"],
+    cmd = "cp $< $@",
+    visibility = ["//visibility:public"]
+)
+
+genrule(
+    name = "native-client-binary",
+    outs = ["typedb/native_client_python.so"],
+    srcs = ["@vaticle_typedb_driver_java//rust:native_client_python"],
+    cmd = "cp $< $@",
+    visibility = ["//visibility:public"]
+)
+
 py_library(
     name = "client_python",
-    srcs = glob(["typedb/**/*.py"]),
-    deps = [
-        vaticle_typedb_client_python_requirement("typedb-protocol"),
-        vaticle_typedb_client_python_requirement("protobuf"),
-        vaticle_typedb_client_python_requirement("grpcio"),
-    ],
+    srcs = glob(["typedb/**/*.py"]) + [":native-client-wrapper"],
+    data = [":native-client-binary"],
+    deps = ["@vaticle_typedb_driver_java//rust:native_client_python_wrapper"],
     visibility = ["//visibility:public"]
 )
 
@@ -80,7 +93,7 @@ assemble_pip(
         "Environment :: Console",
         "Topic :: Database :: Front-Ends"
     ],
-    url = "https://github.com/vaticle/typedb-client-python/",
+    url = "https://github.com/vaticle/typedb-driver-python/",
     author = "Vaticle",
     author_email = "[email protected]",
     license = "Apache-2.0",

WORKSPACE

@@ -28,6 +28,13 @@ workspace(name = "vaticle_typedb_client_python")
 load("//dependencies/vaticle:repositories.bzl", "vaticle_dependencies")
 vaticle_dependencies()
 
+# Load //builder/python
+load("@vaticle_dependencies//builder/python:deps.bzl", python_deps = "deps")
+python_deps()
+load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains")
+py_repositories()
+python_register_toolchains(name = "python39", python_version = "3.9")
+
 # Load //builder/bazel for RBE
 load("@vaticle_dependencies//builder/bazel:deps.bzl", "bazel_toolchain")
 bazel_toolchain()
@@ -44,26 +51,43 @@ kotlin_repositories()
 load("@io_bazel_rules_kotlin//kotlin:core.bzl", "kt_register_toolchains")
 kt_register_toolchains()
 
-# Load //builder/python
-load("@vaticle_dependencies//builder/python:deps.bzl", python_deps = "deps")
-python_deps()
+# Load //builder/rust
+load("@vaticle_dependencies//builder/rust:deps.bzl", rust_deps = "deps")
+rust_deps()
+
+load("@rules_rust//rust:repositories.bzl", "rules_rust_dependencies", "rust_register_toolchains")
+rules_rust_dependencies()
+rust_register_toolchains(edition = "2021", include_rustc_srcs = True)
+
+load("@vaticle_dependencies//library/crates:crates.bzl", "fetch_crates")
+fetch_crates()
+load("@crates//:defs.bzl", "crate_repositories")
+crate_repositories()
 
 # Load //builder/grpc
 load("@vaticle_dependencies//builder/grpc:deps.bzl", grpc_deps = "deps")
 grpc_deps()
-load("@com_github_grpc_grpc//bazel:grpc_deps.bzl",
-com_github_grpc_grpc_deps = "grpc_deps")
+
+load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", com_github_grpc_grpc_deps = "grpc_deps")
 com_github_grpc_grpc_deps()
+load("@stackb_rules_proto//java:deps.bzl", "java_grpc_compile")
+java_grpc_compile()
 
 # Load //tool/common
 load("@vaticle_dependencies//tool/common:deps.bzl", "vaticle_dependencies_ci_pip",
-    vaticle_dependencies_tool_maven_artifacts = "maven_artifacts")
+     vaticle_dependencies_tool_maven_artifacts = "maven_artifacts")
 vaticle_dependencies_ci_pip()
+load("@vaticle_dependencies_ci_pip//:requirements.bzl", "install_deps")
+install_deps()
 
 # Load //tool/checkstyle
 load("@vaticle_dependencies//tool/checkstyle:deps.bzl", checkstyle_deps = "deps")
 checkstyle_deps()
 
+# Load //tool/swig
+load("@vaticle_dependencies//tool/swig:deps.bzl", swig_deps = "deps")
+swig_deps()
+
 ######################################
 # Load @vaticle_bazel_distribution #
 ######################################
@@ -80,33 +104,40 @@ rules_pkg_dependencies()
 # Load //pip
 load("@vaticle_bazel_distribution//pip:deps.bzl", pip_deps = "deps")
 pip_deps()
+load("@vaticle_bazel_distribution_pip//:requirements.bzl", "install_deps")
+install_deps()
 
 # Load //github
 load("@vaticle_bazel_distribution//github:deps.bzl", github_deps = "deps")
 github_deps()
 
-################################
+##############################
 # Load @vaticle dependencies #
-################################
+##############################
 
-load("//dependencies/vaticle:repositories.bzl", "vaticle_typedb_common", "vaticle_typedb_behaviour")
-vaticle_typedb_common()
+load("//dependencies/vaticle:repositories.bzl", "vaticle_typedb_behaviour", "vaticle_typedb_common", "vaticle_typedb_driver_java", "vaticle_typedb_protocol", "vaticle_typeql")
 vaticle_typedb_behaviour()
+vaticle_typedb_common()
+vaticle_typedb_driver_java()
+vaticle_typedb_protocol()
+vaticle_typeql()
 
 # Load artifacts
 load("//dependencies/vaticle:artifacts.bzl", "vaticle_typedb_artifacts", "vaticle_typedb_cluster_artifacts")
 vaticle_typedb_artifacts()
 vaticle_typedb_cluster_artifacts()
 
-#################################
+######################################
 # Load @vaticle_typedb_client_python #
-#################################
+######################################
 
-load("@rules_python//python:pip.bzl", "pip_install")
-pip_install(
+load("@rules_python//python:pip.bzl", "pip_parse")
+pip_parse(
     name = "vaticle_typedb_client_python_pip",
-    requirements = "//:requirements_dev.txt",
+    requirements_lock = "//:requirements_dev.txt",
 )
+load("@vaticle_typedb_client_python_pip//:requirements.bzl", "install_deps")
+install_deps()
 
 ############################
 # Load @maven dependencies #
@@ -115,9 +146,9 @@ pip_install(
 load("@vaticle_dependencies//library/maven:rules.bzl", "maven")
 maven(vaticle_dependencies_tool_maven_artifacts)
 
-##################################################
+#######################################################
 # Create @vaticle_typedb_client_python_workspace_refs #
-##################################################
+#######################################################
 load("@vaticle_bazel_distribution//common:rules.bzl", "workspace_refs")
 workspace_refs(
     name = "vaticle_typedb_client_python_workspace_refs"

dependencies/vaticle/artifacts.bzl

@@ -29,7 +29,7 @@ def vaticle_typedb_artifacts():
         artifact_name = "typedb-server-{platform}-{version}.{ext}",
         tag_source = deployment["artifact.release"],
         commit_source = deployment["artifact.snapshot"],
-        tag = "2.19.0",
+        commit = "7de728a6cc51273299a041608fd589bb1f1ff0ff",
     )
 
 def vaticle_typedb_cluster_artifacts():
@@ -39,5 +39,5 @@ def vaticle_typedb_cluster_artifacts():
         artifact_name = "typedb-cluster-all-{platform}-{version}.{ext}",
         tag_source = deployment_private["artifact.release"],
         commit_source = deployment_private["artifact.snapshot"],
-        tag = "2.19.0",
+        commit = "9f26c8967c2e1a7fcbd72b819b48c1f7f67c78e8",
     )

dependencies/vaticle/repositories.bzl

@@ -25,7 +25,7 @@ def vaticle_dependencies():
     git_repository(
         name = "vaticle_dependencies",
         remote = "https://github.com/vaticle/dependencies",
-        commit = "51fdda8c935d5f9fc34bfdf41b0a6ac1ca77e2ee", # sync-marker: do not remove this comment, this is used for sync-dependencies by @vaticle_dependencies
+        commit = "cbdaf0134a33eaa71bfe0edb9e4878250ec9cdc0", # sync-marker: do not remove this comment, this is used for sync-dependencies by @vaticle_dependencies
     )
 
 def vaticle_typedb_common():
@@ -35,10 +35,37 @@ def vaticle_typedb_common():
         tag = "2.18.0" # sync-marker: do not remove this comment, this is used for sync-dependencies by @vaticle_typedb_common
     )
 
+def vaticle_typeql():
+    git_repository(
+        name = "vaticle_typeql",
+        remote = "https://github.com/dmikhalin/typeql",
+        commit = "c3ab12f7e96079bb5dc6d7e962e5270fec4d976e",  # sync-marker: do not remove this comment, this is used for sync-dependencies by @vaticle_typeql
+    )
+
+def vaticle_typedb_protocol():
+    git_repository(
+        name = "vaticle_typedb_protocol",
+        remote = "https://github.com/vaticle/typedb-protocol",
+        commit = "444dc87520160fd4385f47617b98df6d7e741184",  # sync-marker: do not remove this comment, this is used for sync-dependencies by @vaticle_typedb_protocol
+    )
+
 def vaticle_typedb_behaviour():
     git_repository(
         name = "vaticle_typedb_behaviour",
         remote = "https://github.com/vaticle/typedb-behaviour",
-        commit = "90b4addcd71a398e9e52e322021c49310e66a47a" # sync-marker: do not remove this comment, this is used for sync-dependencies by @vaticle_typedb_behaviour
+        commit = "26f4a19a797203c2eeb0682b6f7d25e238b97345",  # sync-marker: do not remove this comment, this is used for sync-dependencies by @vaticle_typedb_behaviour
     )
 
+def vaticle_factory_tracing():
+    git_repository(
+        name = "vaticle_factory_tracing",
+        remote = "https://github.com/vaticle/factory-tracing",
+        tag = "2.12.0"  # sync-marker: do not remove this comment, this is used for sync-dependencies by @vaticle_factory_tracing
+    )
+
+def vaticle_typedb_driver_java():
+    git_repository(
+        name = "vaticle_typedb_driver_java",
+        remote = "https://github.com/vaticle/typedb-driver-java",
+        commit = "c162f8a58e9c70e03517d5b854acb8b055e30bc5"  # sync-marker: do not remove this comment, this is used for sync-dependencies by @vaticle_factory_tracing
+    )

requirements.txt

@@ -35,7 +35,4 @@
 
 ## Dependencies
 
-typedb-protocol==2.18.0
-grpcio>=1.43.0,<2
-protobuf>=3.15.6,<4
 parse==1.18.0

tests/behaviour/background/cluster/environment.py

@@ -31,7 +31,8 @@
 def before_all(context: Context):
     environment_base.before_all(context)
     context.credential_root_ca_path = os.environ["ROOT_CA"]
-    context.setup_context_client_fn = lambda user="admin", password="password": setup_context_client(context, user, password)
+    context.setup_context_client_fn = lambda user="admin", password="password": \
+        setup_context_client(context, user, password)
 
 
 def before_scenario(context: Context, scenario):
@@ -43,23 +44,24 @@ def before_scenario(context: Context, scenario):
 
 
 def setup_context_client(context, username, password):
-    credential = TypeDBCredential(username, password, context.credential_root_ca_path)
-    context.client = TypeDB.cluster_client(addresses=["127.0.0.1:" + context.config.userdata["port"]], credential=credential)
-    context.session_options = TypeDBOptions.cluster().set_infer(True)
-    context.transaction_options = TypeDBOptions.cluster().set_infer(True)
+    credential = TypeDBCredential(username, password, tls_root_ca_path=context.credential_root_ca_path)
+    context.client = TypeDB.cluster_client(addresses=["localhost:" + context.config.userdata["port"]],
+                                           credential=credential)
+    context.session_options = TypeDBOptions(infer=True)
+    context.transaction_options = TypeDBOptions(infer=True)
 
 
 def after_scenario(context: Context, scenario):
     environment_base.after_scenario(context, scenario)
 
     # TODO: reset the database through the TypeDB runner once it exists
     context.setup_context_client_fn()
-    for database in context.client.databases().all():
+    for database in context.client.databases.all():
         database.delete()
 
-    for user in context.client.users().all():
+    for user in context.client.users.all():
         if user.username() != "admin":
-            context.client.users().delete(user.username())
+            context.client.users.delete(user.username())
     context.client.close()
 
 

tests/behaviour/background/core/environment.py

@@ -44,18 +44,19 @@ def setup_context_client(context, username=None, password=None):
     if username is not None or password is not None:
         raise Exception("Core client does not support authentication")
     context.client = TypeDB.core_client(address="localhost:%d" % int(context.config.userdata["port"]))
-    context.session_options = TypeDBOptions.core().set_infer(True)
-    context.transaction_options = TypeDBOptions.core().set_infer(True)
+    context.session_options = TypeDBOptions(infer=True)
+    context.transaction_options = TypeDBOptions(infer=True)
 
 
 def after_scenario(context: Context, scenario):
     environment_base.after_scenario(context, scenario)
 
     # TODO: reset the database through the TypeDB runner once it exists
     context.setup_context_client_fn()
-    for database in context.client.databases().all():
+    for database in context.client.databases.all():
         database.delete()
     context.client.close()
 
+
 def after_all(context: Context):
     environment_base.after_all(context)

tests/behaviour/background/environment_base.py

@@ -18,6 +18,7 @@
 # specific language governing permissions and limitations
 # under the License.
 #
+
 from behave.model_core import Status
 from typedb.client import *
 
@@ -44,8 +45,8 @@ def before_scenario(context: Context):
     context.get_thing_type = lambda root_label, type_label: _get_thing_type_impl(context, root_label, type_label)
     context.clear_answers = lambda: _clear_answers_impl(context)
     context.option_setters = {
-        "session-idle-timeout-millis": lambda option, value: option.set_session_idle_timeout_millis(int(value)),
-        "transaction-timeout-millis": lambda option, value: option.set_transaction_timeout_millis(int(value)),
+        "session-idle-timeout-millis": lambda options, value:  setattr(options, "session_idle_timeout_millis", value),
+        "transaction-timeout-millis": lambda options, value:  setattr(options, "transaction_timeout_millis", value),
     }
 
 
@@ -55,13 +56,11 @@ def _put_impl(context: Context, variable: str, thing: Thing):
 
 def _get_thing_type_impl(context: Context, root_label: RootLabel, type_label: str):
     if root_label == RootLabel.ENTITY:
-        return context.tx().concepts().get_entity_type(type_label)
+        return context.tx().concepts.get_entity_type(type_label)
     elif root_label == RootLabel.ATTRIBUTE:
-        return context.tx().concepts().get_attribute_type(type_label)
+        return context.tx().concepts.get_attribute_type(type_label)
     elif root_label == RootLabel.RELATION:
-        return context.tx().concepts().get_relation_type(type_label)
-    elif root_label == RootLabel.THING:
-        return context.tx().concepts().get_root_thing_type()
+        return context.tx().concepts.get_relation_type(type_label)
     else:
         raise ValueError("Unrecognised value")
 

tests/behaviour/concept/serialization/json/BUILD

@@ -48,6 +48,7 @@ typedb_behaviour_py_test(
         "//tests/behaviour/util:util",
         "//tests/behaviour/config:parameters",
         "//tests/behaviour/background",
+        "//:native-client-binary",
     ],
     native_typedb_artifact = "//tests:native-typedb-artifact",
     native_typedb_cluster_artifact = "//tests:native-typedb-cluster-artifact",