Skip to content
This repository has been archived by the owner on Nov 1, 2024. It is now read-only.

Commit

Permalink
Create PyVelox package and move type dependencies over. (#503)
Browse files Browse the repository at this point in the history 
Summary:
PyVelox package is gated with CMake flags in this pr. This refactoring will allow Torcharrow and Koski to use a common set of bindings.

Pull Request resolved: #503

Differential Revision: D40082189

Pulled By: kgpai

fbshipit-source-id: 83427b5aee17eef5470f93129200fcd215286fa2
  • Loading branch information
@kgpai @facebook-github-bot
kgpai authored and facebook-github-bot committed Oct 13, 2022
1 parent 74d0df7 commit a1c7062
Show file tree
Hide file tree
Showing 14 changed files with 407 additions and 254 deletions.
3 changes: 3 additions & 0 deletions csrc/velox/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ cmake_minimum_required(VERSION 3.15)

# _torcharrow is a shared library as it's a Python extension
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
set(CREATE_PYVELOX_MODULE OFF)

# To make the right CPython is built with on GitHub Actions,
# see https://github.com/actions/setup-python/issues/121#issuecomment-1014500503
Expand Down Expand Up @@ -91,6 +92,7 @@ endif()
# set_target_properties(_torcharrow PROPERTIES CXX_VISIBILITY_PRESET default)
add_subdirectory(velox)
add_subdirectory(functions)
add_subdirectory(pyvelox)


# Link with Velox:
Expand All @@ -103,6 +105,7 @@ target_link_libraries(_torcharrow PRIVATE
velox_function_registry
velox_arrow_bridge
torcharrow_udfs
pyvelox
)

target_compile_definitions(
Expand Down
124 changes: 62 additions & 62 deletions csrc/velox/_torcharrow.pyi
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,18 +49,18 @@ __all__ = [
"SimpleColumnVARCHAR",
"TypeKind",
"VeloxType",
"VeloxArrayType",
"VeloxFixedArrayType",
"VeloxMapType",
"VeloxRowType",
"VeloxType_BIGINT",
"VeloxType_BOOLEAN",
"VeloxType_DOUBLE",
"VeloxType_INTEGER",
"VeloxType_REAL",
"VeloxType_SMALLINT",
"VeloxType_TINYINT",
"VeloxType_VARCHAR",
"ArrayType",
"FixedSizeArrayType",
"MapType",
"RowType",
"BigintType",
"BooleanType",
"DoubleType",
"IntegerType",
"RealType",
"SmallintType",
"TinyintType",
"VarcharType",
"Vocab",
"fb_SentencePiece",
"Column",
Expand Down Expand Up @@ -2153,30 +2153,30 @@ class VeloxType:
def kind_name(self) -> str: ...
__pybind11_module_local_v4_clang_libstdcpp_cxxabi1002__ = ... # type: PyCapsule # value = <capsule object NULL at 0x7f50e2a512d0>
pass
class VeloxArrayType(VeloxType):
class ArrayType(VeloxType):
def __init__(self, arg0: VeloxType) -> None: ...
def element_type(self) -> VeloxType: ...
def kind(self) -> TypeKind: ...
def kind_name(self) -> str: ...
__pybind11_module_local_v4_clang_libstdcpp_cxxabi1002__ = ... # type: PyCapsule # value = <capsule object NULL at 0x7f50e2a69660>
pass
class VeloxFixedArrayType(VeloxType):
class FixedSizeArrayType(VeloxType):
def __init__(self, arg0: int, arg1: VeloxType) -> None: ...
def element_type(self) -> VeloxType: ...
def fixed_width(self) -> int: ...
def kind(self) -> TypeKind: ...
def kind_name(self) -> str: ...
__pybind11_module_local_v4_clang_libstdcpp_cxxabi1002__ = ... # type: PyCapsule # value = <capsule object NULL at 0x7f50e2a69780>
pass
class VeloxMapType(VeloxType):
class MapType(VeloxType):
def __init__(self, arg0: VeloxType, arg1: VeloxType) -> None: ...
def key_type(self) -> VeloxType: ...
def kind(self) -> TypeKind: ...
def kind_name(self) -> str: ...
def value_type(self) -> VeloxType: ...
__pybind11_module_local_v4_clang_libstdcpp_cxxabi1002__ = ... # type: PyCapsule # value = <capsule object NULL at 0x7f50e2a69c30>
pass
class VeloxRowType(VeloxType):
class RowType(VeloxType):
def __init__(self, arg0: List[str], arg1: List[VeloxType]) -> None: ...
def child_at(self, arg0: int) -> VeloxType: ...
def contains_child(self, arg0: str) -> bool: ...
Expand All @@ -2187,49 +2187,49 @@ class VeloxRowType(VeloxType):
def size(self) -> int: ...
__pybind11_module_local_v4_clang_libstdcpp_cxxabi1002__ = ... # type: PyCapsule # value = <capsule object NULL at 0x7f50e2a70120>
pass
class VeloxType_BIGINT(VeloxType):
class BigintType(VeloxType):
def __init__(self) -> None: ...
def kind(self) -> TypeKind: ...
def kind_name(self) -> str: ...
__pybind11_module_local_v4_clang_libstdcpp_cxxabi1002__ = ... # type: PyCapsule # value = <capsule object NULL at 0x7f50e2a51540>
pass
class VeloxType_BOOLEAN(VeloxType):
class BooleanType(VeloxType):
def __init__(self) -> None: ...
def kind(self) -> TypeKind: ...
def kind_name(self) -> str: ...
__pybind11_module_local_v4_clang_libstdcpp_cxxabi1002__ = ... # type: PyCapsule # value = <capsule object NULL at 0x7f50e2a5eea0>
pass
class VeloxType_DOUBLE(VeloxType):
class DoubleType(VeloxType):
def __init__(self) -> None: ...
def kind(self) -> TypeKind: ...
def kind_name(self) -> str: ...
__pybind11_module_local_v4_clang_libstdcpp_cxxabi1002__ = ... # type: PyCapsule # value = <capsule object NULL at 0x7f50e2a65540>
pass
class VeloxType_INTEGER(VeloxType):
class IntegerType(VeloxType):
def __init__(self) -> None: ...
def kind(self) -> TypeKind: ...
def kind_name(self) -> str: ...
__pybind11_module_local_v4_clang_libstdcpp_cxxabi1002__ = ... # type: PyCapsule # value = <capsule object NULL at 0x7f50e2a553f0>
pass
class VeloxType_REAL(VeloxType):
class RealType(VeloxType):
def __init__(self) -> None: ...
def kind(self) -> TypeKind: ...
def kind_name(self) -> str: ...
__pybind11_module_local_v4_clang_libstdcpp_cxxabi1002__ = ... # type: PyCapsule # value = <capsule object NULL at 0x7f50e2a627b0>
pass
class VeloxType_SMALLINT(VeloxType):
class SmallintType(VeloxType):
def __init__(self) -> None: ...
def kind(self) -> TypeKind: ...
def kind_name(self) -> str: ...
__pybind11_module_local_v4_clang_libstdcpp_cxxabi1002__ = ... # type: PyCapsule # value = <capsule object NULL at 0x7f50e2a59240>
pass
class VeloxType_TINYINT(VeloxType):
class TinyintType(VeloxType):
def __init__(self) -> None: ...
def kind(self) -> TypeKind: ...
def kind_name(self) -> str: ...
__pybind11_module_local_v4_clang_libstdcpp_cxxabi1002__ = ... # type: PyCapsule # value = <capsule object NULL at 0x7f50e2a5e090>
pass
class VeloxType_VARCHAR(VeloxType):
class VarcharType(VeloxType):
def __init__(self) -> None: ...
def kind(self) -> TypeKind: ...
def kind_name(self) -> str: ...
Expand Down Expand Up @@ -2265,94 +2265,94 @@ class fb_SentencePiece:
def process(self, arg0: str) -> List[str]: ...
pass
@overload
def Column(arg0: VeloxType_DOUBLE) -> SimpleColumnDOUBLE:
def Column(arg0: DoubleType) -> SimpleColumnDOUBLE:
pass
@overload
def Column(arg0: VeloxType_INTEGER) -> SimpleColumnINTEGER:
def Column(arg0: IntegerType) -> SimpleColumnINTEGER:
pass
@overload
def Column(arg0: VeloxType_INTEGER, arg1: list) -> SimpleColumnINTEGER:
def Column(arg0: IntegerType, arg1: list) -> SimpleColumnINTEGER:
pass
@overload
def Column(arg0: VeloxType_SMALLINT, arg1: tuple) -> SimpleColumnSMALLINT:
def Column(arg0: SmallintType, arg1: tuple) -> SimpleColumnSMALLINT:
pass
@overload
def Column(arg0: VeloxType_SMALLINT) -> SimpleColumnSMALLINT:
def Column(arg0: SmallintType) -> SimpleColumnSMALLINT:
pass
@overload
def Column(arg0: VeloxType_REAL, arg1: tuple) -> SimpleColumnREAL:
def Column(arg0: RealType, arg1: tuple) -> SimpleColumnREAL:
pass
@overload
def Column(arg0: VeloxRowType) -> RowColumn:
def Column(arg0: RowType) -> RowColumn:
pass
@overload
def Column(arg0: VeloxType_SMALLINT, arg1: list) -> SimpleColumnSMALLINT:
def Column(arg0: SmallintType, arg1: list) -> SimpleColumnSMALLINT:
pass
@overload
def Column(arg0: VeloxType_TINYINT, arg1: tuple) -> SimpleColumnTINYINT:
def Column(arg0: TinyintType, arg1: tuple) -> SimpleColumnTINYINT:
pass
@overload
def Column(arg0: VeloxFixedArrayType, arg1: list) -> ArrayColumn:
def Column(arg0: FixedSizeArrayType, arg1: list) -> ArrayColumn:
pass
@overload
def Column(arg0: VeloxType_BIGINT, arg1: tuple) -> SimpleColumnBIGINT:
def Column(arg0: BigintType, arg1: tuple) -> SimpleColumnBIGINT:
pass
@overload
def Column(arg0: VeloxArrayType) -> ArrayColumn:
def Column(arg0: ArrayType) -> ArrayColumn:
pass
@overload
def Column(arg0: VeloxType_DOUBLE, arg1: tuple) -> SimpleColumnDOUBLE:
def Column(arg0: DoubleType, arg1: tuple) -> SimpleColumnDOUBLE:
pass
@overload
def Column(arg0: VeloxArrayType, arg1: list) -> ArrayColumn:
def Column(arg0: ArrayType, arg1: list) -> ArrayColumn:
pass
@overload
def Column(arg0: VeloxType_BOOLEAN) -> SimpleColumnBOOLEAN:
def Column(arg0: BooleanType) -> SimpleColumnBOOLEAN:
pass
@overload
def Column(arg0: VeloxType_VARCHAR, arg1: tuple) -> SimpleColumnVARCHAR:
def Column(arg0: VarcharType, arg1: tuple) -> SimpleColumnVARCHAR:
pass
@overload
def Column(arg0: VeloxMapType) -> MapColumn:
def Column(arg0: MapType) -> MapColumn:
pass
@overload
def Column(arg0: VeloxType_TINYINT) -> SimpleColumnTINYINT:
def Column(arg0: TinyintType) -> SimpleColumnTINYINT:
pass
@overload
def Column(arg0: VeloxFixedArrayType) -> ArrayColumn:
def Column(arg0: FixedSizeArrayType) -> ArrayColumn:
pass
@overload
def Column(arg0: VeloxType_INTEGER, arg1: tuple) -> SimpleColumnINTEGER:
def Column(arg0: IntegerType, arg1: tuple) -> SimpleColumnINTEGER:
pass
@overload
def Column(arg0: VeloxType_REAL) -> SimpleColumnREAL:
def Column(arg0: RealType) -> SimpleColumnREAL:
pass
@overload
def Column(arg0: VeloxType_BIGINT) -> SimpleColumnBIGINT:
def Column(arg0: BigintType) -> SimpleColumnBIGINT:
pass
@overload
def Column(arg0: VeloxType_TINYINT, arg1: list) -> SimpleColumnTINYINT:
def Column(arg0: TinyintType, arg1: list) -> SimpleColumnTINYINT:
pass
@overload
def Column(arg0: VeloxType_DOUBLE, arg1: list) -> SimpleColumnDOUBLE:
def Column(arg0: DoubleType, arg1: list) -> SimpleColumnDOUBLE:
pass
@overload
def Column(arg0: VeloxType_BIGINT, arg1: list) -> SimpleColumnBIGINT:
def Column(arg0: BigintType, arg1: list) -> SimpleColumnBIGINT:
pass
@overload
def Column(arg0: VeloxType_REAL, arg1: list) -> SimpleColumnREAL:
def Column(arg0: RealType, arg1: list) -> SimpleColumnREAL:
pass
@overload
def Column(arg0: VeloxType_BOOLEAN, arg1: tuple) -> SimpleColumnBOOLEAN:
def Column(arg0: BooleanType, arg1: tuple) -> SimpleColumnBOOLEAN:
pass
@overload
def Column(arg0: VeloxType_VARCHAR) -> SimpleColumnVARCHAR:
def Column(arg0: VarcharType) -> SimpleColumnVARCHAR:
pass
@overload
def Column(arg0: VeloxType_VARCHAR, arg1: list) -> SimpleColumnVARCHAR:
def Column(arg0: VarcharType, arg1: list) -> SimpleColumnVARCHAR:
pass
@overload
def Column(arg0: VeloxType_BOOLEAN, arg1: list) -> SimpleColumnBOOLEAN:
def Column(arg0: BooleanType, arg1: list) -> SimpleColumnBOOLEAN:
pass
@overload
def ConstantColumn(arg0: handle, arg1: int, arg2: VeloxType) -> BaseColumn:
Expand All @@ -2361,28 +2361,28 @@ def ConstantColumn(arg0: handle, arg1: int, arg2: VeloxType) -> BaseColumn:
def ConstantColumn(arg0: handle, arg1: int) -> BaseColumn:
pass
@overload
def _import_from_arrow(arg0: VeloxType_SMALLINT, arg1: int, arg2: int) -> SimpleColumnSMALLINT:
def _import_from_arrow(arg0: SmallintType, arg1: int, arg2: int) -> SimpleColumnSMALLINT:
pass
@overload
def _import_from_arrow(arg0: VeloxType_DOUBLE, arg1: int, arg2: int) -> SimpleColumnDOUBLE:
def _import_from_arrow(arg0: DoubleType, arg1: int, arg2: int) -> SimpleColumnDOUBLE:
pass
@overload
def _import_from_arrow(arg0: VeloxType_TINYINT, arg1: int, arg2: int) -> SimpleColumnTINYINT:
def _import_from_arrow(arg0: TinyintType, arg1: int, arg2: int) -> SimpleColumnTINYINT:
pass
@overload
def _import_from_arrow(arg0: VeloxType_BOOLEAN, arg1: int, arg2: int) -> SimpleColumnBOOLEAN:
def _import_from_arrow(arg0: BooleanType, arg1: int, arg2: int) -> SimpleColumnBOOLEAN:
pass
@overload
def _import_from_arrow(arg0: VeloxType_INTEGER, arg1: int, arg2: int) -> SimpleColumnINTEGER:
def _import_from_arrow(arg0: IntegerType, arg1: int, arg2: int) -> SimpleColumnINTEGER:
pass
@overload
def _import_from_arrow(arg0: VeloxRowType, arg1: int, arg2: int) -> RowColumn:
def _import_from_arrow(arg0: RowType, arg1: int, arg2: int) -> RowColumn:
pass
@overload
def _import_from_arrow(arg0: VeloxType_BIGINT, arg1: int, arg2: int) -> SimpleColumnBIGINT:
def _import_from_arrow(arg0: BigintType, arg1: int, arg2: int) -> SimpleColumnBIGINT:
pass
@overload
def _import_from_arrow(arg0: VeloxType_REAL, arg1: int, arg2: int) -> SimpleColumnREAL:
def _import_from_arrow(arg0: RealType, arg1: int, arg2: int) -> SimpleColumnREAL:
pass
def _populate_dense_features_nopresence(arg0: RowColumn, arg1: int) -> None:
pass
Expand Down
Loading

0 comments on commit a1c7062

Please sign in to comment.