diff --git a/.gitignore b/.gitignore index 75b45dc9..755089d0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ .* *~ bazel-* -/*build* +build* bin/* lib/* var/* diff --git a/CMakeLists.txt b/CMakeLists.txt index 31ccec86..294af340 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,6 +19,7 @@ message(STATUS "PROJECT_ROOT_DIR = ${PROJECT_ROOT_DIR}") include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake) +include_directories(${PROJECT_ROOT_DIR}/src/include) include_directories(${PROJECT_ROOT_DIR}/src) option(BUILD_PYTHON_BINDINGS "Build Python bindings using pybind11" OFF) @@ -33,7 +34,7 @@ cc_directories(tests) if(BUILD_TOOLS) cc_directories(tools) -endif () +endif() git_version(GIT_SRCS_VER ${PROJECT_ROOT_DIR}) set(CPACK_PACKAGE_VERSION ${GIT_SRCS_VER}) diff --git a/cmake/bazel.cmake b/cmake/bazel.cmake index 7226ce18..deaf1656 100644 --- a/cmake/bazel.cmake +++ b/cmake/bazel.cmake @@ -13,7 +13,7 @@ ## 1.3. Build a C/C++ static or shared library ## cc_library( ## NAME -## [STATIC] [SHARED] [STRICT] [ALWAYS_LINK] [EXCLUDE] [PACKED] +## [STATIC] [SHARED] [STRICT] [ALWAYS_LINK] [EXCLUDE] [PACKED] [SRCS_NO_GLOB] ## SRCS [file2 ...] ## [INCS dir1 ...] ## [PUBINCS public_dir1 ...] @@ -813,18 +813,35 @@ endfunction() ## Build a C/C++ static or shared library function(cc_library) cmake_parse_arguments( - CC_ARGS "STATIC;SHARED;EXCLUDE;PACKED" "NAME;VERSION" + CC_ARGS + "STATIC;SHARED;EXCLUDE;PACKED;SRCS_NO_GLOB" + "NAME;VERSION" "SRCS;INCS;PUBINCS;DEFS;LIBS;CFLAGS;CXXFLAGS;LDFLAGS;DEPS;PACKED_EXCLUDES" ${ARGN} - ) + ) if(NOT CC_ARGS_NAME) - message(FATAL_ERROR "No target name privated.") + message(FATAL_ERROR "No target name provided.") endif() - file(GLOB CC_ARGS_SRCS ${CC_ARGS_SRCS}) - if(NOT CC_ARGS_SRCS) - message(FATAL_ERROR "No source files found of ${CC_ARGS_NAME}.") + if(CC_ARGS_SRCS_NO_GLOB) + set(SOURCE_FILES ${CC_ARGS_SRCS}) + if(NOT SOURCE_FILES) + message(FATAL_ERROR "No source files provided for ${CC_ARGS_NAME} (SRCS_NO_GLOB mode).") + endif() + else() + set(SOURCE_FILES "") + foreach(_src IN LISTS CC_ARGS_SRCS) + if(IS_ABSOLUTE "${_src}" OR NOT "${_src}" MATCHES "[*?]") + list(APPEND SOURCE_FILES "${_src}") + else() + file(GLOB _globbed_srcs ${_src}) + list(APPEND SOURCE_FILES ${_globbed_srcs}) + endif() + endforeach() + if(NOT SOURCE_FILES) + message(FATAL_ERROR "No source files found for ${CC_ARGS_NAME} after globbing.") + endif() endif() if(CC_ARGS_VERSION) @@ -837,13 +854,13 @@ function(cc_library) endif() if(CC_ARGS_SHARED AND CC_ARGS_STATIC) - _add_library(${CC_ARGS_NAME} "${EXCLUDE_OPTION}" "${CC_ARGS_SRCS}") + _add_library(${CC_ARGS_NAME} "${EXCLUDE_OPTION}" ${SOURCE_FILES}) elseif(CC_ARGS_SHARED) - add_library(${CC_ARGS_NAME} SHARED ${EXCLUDE_OPTION} ${CC_ARGS_SRCS}) + add_library(${CC_ARGS_NAME} SHARED ${EXCLUDE_OPTION} ${SOURCE_FILES}) elseif(CC_ARGS_STATIC) - add_library(${CC_ARGS_NAME} STATIC ${EXCLUDE_OPTION} ${CC_ARGS_SRCS}) + add_library(${CC_ARGS_NAME} STATIC ${EXCLUDE_OPTION} ${SOURCE_FILES}) else() - add_library(${CC_ARGS_NAME} ${EXCLUDE_OPTION} ${CC_ARGS_SRCS}) + add_library(${CC_ARGS_NAME} ${EXCLUDE_OPTION} ${SOURCE_FILES}) endif() if(TARGET ${CC_ARGS_NAME}_objects) @@ -857,7 +874,7 @@ function(cc_library) LDFLAGS "${CC_ARGS_LDFLAGS}" DEPS "${CC_ARGS_DEPS}" "${CC_ARGS_UNPARSED_ARGUMENTS}" - ) + ) endif() if(TARGET ${CC_ARGS_NAME}_static) @@ -872,7 +889,7 @@ function(cc_library) LDFLAGS "${CC_ARGS_LDFLAGS}" DEPS "${CC_ARGS_DEPS}" "${CC_ARGS_UNPARSED_ARGUMENTS}" - ) + ) if(CC_ARGS_PACKED) install( TARGETS ${CC_ARGS_NAME}_static @@ -893,13 +910,13 @@ function(cc_library) DEPS "${CC_ARGS_DEPS}" VERSION "${CC_ARGS_VERSION}" "${CC_ARGS_UNPARSED_ARGUMENTS}" - ) + ) if(CC_ARGS_PACKED) install( TARGETS ${CC_ARGS_NAME} ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" - ) + ) if(CC_ARGS_PUBINCS) foreach(PACKED_EXCLUDE ${CC_ARGS_PACKED_EXCLUDES}) list(APPEND PATTERN_EXCLUDES "PATTERN;${PACKED_EXCLUDE};EXCLUDE") @@ -908,7 +925,7 @@ function(cc_library) DIRECTORY ${CC_ARGS_PUBINCS} DESTINATION ${CMAKE_INSTALL_INCDIR} FILES_MATCHING PATTERN "*.h" PATTERN "*.hpp" PATTERN "*.hxx" ${PATTERN_EXCLUDES} - ) + ) endif() endif() endfunction() diff --git a/examples/c++/CMakeLists.txt b/examples/c++/CMakeLists.txt new file mode 100644 index 00000000..2810f3ed --- /dev/null +++ b/examples/c++/CMakeLists.txt @@ -0,0 +1,107 @@ +cmake_minimum_required(VERSION 3.13) +cmake_policy(SET CMP0077 NEW) +project(zvec-example-c++) +set(CMAKE_CXX_STANDARD 17) + +# Enable compile_commands.json +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +# --- Paths to Zvec and dependencies --- +set(ZVEC_INCLUDE_DIR ${CMAKE_BINARY_DIR}/../../../src/include) +set(ZVEC_LIB_DIR ${CMAKE_BINARY_DIR}/../../../build/lib) +set(ZVEC_DEPENDENCY_LIB_DIR ${CMAKE_BINARY_DIR}/../../../build/external/usr/local/lib) + +# Add include and library search paths +include_directories(${ZVEC_INCLUDE_DIR}) +link_directories(${ZVEC_LIB_DIR} ${ZVEC_DEPENDENCY_LIB_DIR}) + +# --- Determine debug/release library names --- +if(CMAKE_BUILD_TYPE STREQUAL "Debug") + set(GLOG_LIB glogd) + set(GFLAGS_LIB gflags_nothreads_debug) + set(PROTOBUF_LIB protobufd) +else() + set(GLOG_LIB glog) + set(GFLAGS_LIB gflags_nothreads) + set(PROTOBUF_LIB protobuf) +endif() + +# --- Dependency groups --- +set(zvec_ailego_deps + arrow + parquet + arrow_bundled_dependencies +) + +set(zvec_core_deps + # empty +) + +set(zvec_db_deps + roaring + rocksdb + arrow + arrow_acero + arrow_bundled_dependencies + arrow_compute + arrow_dataset + parquet + antlr4-runtime + ${GLOG_LIB} + ${GFLAGS_LIB} + ${PROTOBUF_LIB} + lz4 +) + +# --- Create INTERFACE targets for Zvec components --- + +# zvec_ailego: links libzvec_ailego.a + its deps +add_library(zvec-ailego INTERFACE) +target_link_libraries(zvec-ailego INTERFACE + -lzvec_ailego + ${zvec_ailego_deps} +) + +# zvec_core: links libzvec_core.a via special flags (handled externally), but declare logical deps +add_library(zvec-core INTERFACE) +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_link_libraries(zvec-core INTERFACE + "-Wl,--whole-archive,${ZVEC_LIB_DIR}/libzvec_core.a,-Wl,--no-whole-archive" + zvec-ailego + ${zvec_core_deps} + ) +elseif(APPLE) + target_link_libraries(zvec-core INTERFACE + "-Wl,-force_load,${ZVEC_LIB_DIR}/libzvec_core.a" + zvec-ailego + ${zvec_core_deps} + ) +else() + message(FATAL_ERROR "Unsupported platform: ${CMAKE_SYSTEM_NAME}") +endif() + +# zvec_db: links libzvec_db.a + all deps +add_library(zvec-db INTERFACE) +target_link_libraries(zvec-db INTERFACE + zvec_db + zvec-core + zvec-ailego + ${zvec_db_deps} +) + + +# --- Main executable --- +add_executable(db-example db/main.cc) +target_link_libraries(db-example PRIVATE + zvec-db +) + +add_executable(core-example core/main.cc) +target_link_libraries(core-example PRIVATE + zvec-core +) + +add_executable(ailego-example ailego/main.cc) +target_link_libraries(ailego-example PRIVATE + zvec-ailego +) \ No newline at end of file diff --git a/examples/c++/ailego/main.cc b/examples/c++/ailego/main.cc new file mode 100644 index 00000000..da68df33 --- /dev/null +++ b/examples/c++/ailego/main.cc @@ -0,0 +1,11 @@ +#include +#include +#include + +using namespace zvec; + +int main() { + std::string a{"hello world"}; + + std::cout << ailego::StringHelper::StartsWith(a, "hello") << std::endl; +} \ No newline at end of file diff --git a/examples/c++/core/main.cc b/examples/c++/core/main.cc new file mode 100644 index 00000000..d269c894 --- /dev/null +++ b/examples/c++/core/main.cc @@ -0,0 +1,3 @@ +int main() { + return 0; +} \ No newline at end of file diff --git a/examples/c++/db/main.cc b/examples/c++/db/main.cc new file mode 100644 index 00000000..f820e9ed --- /dev/null +++ b/examples/c++/db/main.cc @@ -0,0 +1,263 @@ +#include +#include +#include +#include +#include +#include +#include + +using namespace zvec; + +Doc create_doc(const uint64_t doc_id, const CollectionSchema &schema, + std::string pk = "") { + Doc new_doc; + if (pk.empty()) { + pk = "pk_" + std::to_string(doc_id); + } + new_doc.set_pk(pk); + + for (auto &field : schema.fields()) { + switch (field->data_type()) { + case DataType::BINARY: { + std::string binary_str("binary_" + std::to_string(doc_id)); + new_doc.set(field->name(), binary_str); + break; + } + case DataType::BOOL: + new_doc.set(field->name(), doc_id % 10 == 0); + break; + case DataType::INT32: + new_doc.set(field->name(), (int32_t)doc_id); + break; + case DataType::INT64: + new_doc.set(field->name(), (int64_t)doc_id); + break; + case DataType::UINT32: + new_doc.set(field->name(), (uint32_t)doc_id); + break; + case DataType::UINT64: + new_doc.set(field->name(), (uint64_t)doc_id); + break; + case DataType::FLOAT: + new_doc.set(field->name(), (float)doc_id); + break; + case DataType::DOUBLE: + new_doc.set(field->name(), (double)doc_id); + break; + case DataType::STRING: + new_doc.set(field->name(), + "value_" + std::to_string(doc_id)); + break; + case DataType::ARRAY_BINARY: { + std::vector bin_vec; + for (size_t i = 0; i < (doc_id % 10); i++) { + bin_vec.push_back("bin_" + std::to_string(i)); + } + new_doc.set>(field->name(), bin_vec); + break; + } + case DataType::ARRAY_BOOL: + new_doc.set>(field->name(), + std::vector(10, doc_id % 10 == 0)); + break; + case DataType::ARRAY_INT32: + new_doc.set>( + field->name(), std::vector(10, (int32_t)doc_id)); + break; + case DataType::ARRAY_INT64: + new_doc.set>( + field->name(), std::vector(10, (int64_t)doc_id)); + break; + case DataType::ARRAY_UINT32: + new_doc.set>( + field->name(), std::vector(10, (uint32_t)doc_id)); + break; + case DataType::ARRAY_UINT64: + new_doc.set>( + field->name(), std::vector(10, (uint64_t)doc_id)); + break; + case DataType::ARRAY_FLOAT: + new_doc.set>(field->name(), + std::vector(10, (float)doc_id)); + break; + case DataType::ARRAY_DOUBLE: + new_doc.set>( + field->name(), std::vector(10, (double)doc_id)); + break; + case DataType::ARRAY_STRING: + new_doc.set>( + field->name(), + std::vector(10, "value_" + std::to_string(doc_id))); + break; + case DataType::VECTOR_BINARY32: + new_doc.set>( + field->name(), + std::vector(field->dimension(), uint32_t(doc_id + 0.1))); + break; + case DataType::VECTOR_BINARY64: + new_doc.set>( + field->name(), + std::vector(field->dimension(), uint64_t(doc_id + 0.1))); + break; + case DataType::VECTOR_FP32: + new_doc.set>( + field->name(), + std::vector(field->dimension(), float(doc_id + 0.1))); + break; + case DataType::VECTOR_FP64: + new_doc.set>( + field->name(), + std::vector(field->dimension(), double(doc_id + 0.1))); + break; + case DataType::VECTOR_FP16: + new_doc.set>( + field->name(), std::vector( + field->dimension(), + static_cast(float(doc_id + 0.1)))); + break; + case DataType::VECTOR_INT8: + new_doc.set>( + field->name(), + std::vector(field->dimension(), (int8_t)doc_id)); + break; + case DataType::VECTOR_INT16: + new_doc.set>( + field->name(), + std::vector(field->dimension(), (int16_t)doc_id)); + break; + case DataType::SPARSE_VECTOR_FP16: { + std::vector indices; + std::vector values; + for (uint32_t i = 0; i < 100; i++) { + indices.push_back(i); + values.push_back(float16_t(float(doc_id + 0.1))); + } + std::pair, std::vector> + sparse_float_vec; + sparse_float_vec.first = indices; + sparse_float_vec.second = values; + new_doc.set, std::vector>>( + field->name(), sparse_float_vec); + break; + } + case DataType::SPARSE_VECTOR_FP32: { + std::vector indices; + std::vector values; + for (uint32_t i = 0; i < 100; i++) { + indices.push_back(i); + values.push_back(float(doc_id + 0.1)); + } + std::pair, std::vector> sparse_float_vec; + sparse_float_vec.first = indices; + sparse_float_vec.second = values; + new_doc.set, std::vector>>( + field->name(), sparse_float_vec); + break; + } + default: + std::cout << "Unsupported data type: " << field->name() << std::endl; + throw std::runtime_error("Unsupported vector data type"); + } + } + + return new_doc; +} + +CollectionSchema::Ptr create_schema() { + auto schema = std::make_shared("demo"); + schema->set_max_doc_count_per_segment(1000); + + schema->add_field(std::make_shared( + "id", DataType::INT64, false, std::make_shared(true))); + schema->add_field(std::make_shared( + "name", DataType::STRING, false, + std::make_shared(false))); + schema->add_field( + std::make_shared("weight", DataType::FLOAT, true)); + + schema->add_field(std::make_shared( + "dense", DataType::VECTOR_FP32, 128, false, + std::make_shared(MetricType::IP))); + schema->add_field(std::make_shared( + "sparse", DataType::SPARSE_VECTOR_FP32, 0, false, + std::make_shared(MetricType::IP))); + + return schema; +} + +int main() { + std::string path = "./demo"; + std::string rm_cmd = "rm -rf " + path; + system(rm_cmd.c_str()); + + auto schema = create_schema(); + CollectionOptions options{false, true}; + + auto result = Collection::CreateAndOpen(path, *schema, options); + if (!result.has_value()) { + std::cout << result.error().message() << std::endl; + return -1; + } + + std::cout << "init stats: " << result.value()->Stats().value().to_string() + << std::endl; + + auto coll = std::move(result).value(); + + // insert docs + { + auto doc1 = create_doc(0, *schema); + std::vector docs{doc1}; + auto res = coll->Insert(docs); + if (!res.has_value()) { + std::cout << res.error().message() << std::endl; + return -1; + } + std::cout << "after insert stats " << coll->Stats().value().to_string() + << std::endl; + } + + // optimize + { + auto res = coll->Optimize(); + if (!res.ok()) { + std::cout << res.message() << std::endl; + return -1; + } + std::cout << "after optimize stats " << coll->Stats().value().to_string() + << std::endl; + } + + // query + { + VectorQuery query; + query.topk_ = 10; + query.field_name_ = "dense"; + query.include_vector_ = true; + std::vector query_vector = std::vector(128, 0.1); + query.query_vector_.assign((char *)query_vector.data(), + query_vector.size() * sizeof(float)); + auto res = coll->Query(query); + if (!res.has_value()) { + std::cout << res.error().message() << std::endl; + return -1; + } + std::cout << "query result: doc_count[" << res.value().size() << "]" + << std::endl; + std::cout << "first doc: " << res.value()[0]->to_detail_string() + << std::endl; + } + + // close and reopen + coll.reset(); + options.read_only_ = true; + result = Collection::Open(path, options); + if (!result.has_value()) { + std::cout << result.error().message() << std::endl; + return -1; + } + std::cout << "reopen stats: " << result.value()->Stats().value().to_string() + << std::endl; + + return 0; +} \ No newline at end of file diff --git a/src/ailego/algorithm/kmeans.h b/src/ailego/algorithm/kmeans.h index 7c003c9b..1a928d09 100644 --- a/src/ailego/algorithm/kmeans.h +++ b/src/ailego/algorithm/kmeans.h @@ -24,9 +24,9 @@ #include #include #include -#include #include #include +#include #include "lloyd_cluster.h" namespace zvec { diff --git a/src/ailego/buffer/buffer_manager.cc b/src/ailego/buffer/buffer_manager.cc index 15111b8a..acdb37b6 100644 --- a/src/ailego/buffer/buffer_manager.cc +++ b/src/ailego/buffer/buffer_manager.cc @@ -19,6 +19,25 @@ #include #include #include +#include "ailego/internal/platform.h" + +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunused-parameter" +#pragma clang diagnostic ignored "-Wshadow" +#elif defined(__GNUC__) || defined(__GNUG__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#pragma GCC diagnostic ignored "-Wshadow" +#endif + +#include + +#ifdef __clang__ +#pragma clang diagnostic pop +#elif defined(__GNUC__) || defined(__GNUG__) +#pragma GCC diagnostic pop +#endif namespace zvec { diff --git a/src/ailego/buffer/buffer_manager.h b/src/ailego/buffer/buffer_manager.h index 46ee6aa7..f9e2120a 100644 --- a/src/ailego/buffer/buffer_manager.h +++ b/src/ailego/buffer/buffer_manager.h @@ -21,28 +21,20 @@ #include #include #include +#include #include - -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wunused-parameter" -#pragma clang diagnostic ignored "-Wshadow" -#elif defined(__GNUC__) || defined(__GNUG__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-parameter" -#pragma GCC diagnostic ignored "-Wshadow" -#endif - -#include - -#ifdef __clang__ -#pragma clang diagnostic pop -#elif defined(__GNUC__) || defined(__GNUG__) -#pragma GCC diagnostic pop -#endif - -#include "ailego/pattern/singleton.h" - +#include + +namespace arrow { +class ChunkedArray; +class Array; +class DataType; +class Scalar; +template +class Result; +class Status; +class Buffer; +} // namespace arrow namespace zvec { diff --git a/src/ailego/container/vector_array.h b/src/ailego/container/vector_array.h index 7819a230..dfc82ef4 100644 --- a/src/ailego/container/vector_array.h +++ b/src/ailego/container/vector_array.h @@ -14,6 +14,7 @@ #pragma once +#include "ailego/internal/platform.h" #include "vector.h" namespace zvec { diff --git a/src/ailego/internal/cpu_features.cc b/src/ailego/internal/cpu_features.cc index 9b8ea827..4bf139ae 100644 --- a/src/ailego/internal/cpu_features.cc +++ b/src/ailego/internal/cpu_features.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "cpu_features.h" +#include #if !defined(_MSC_VER) && !defined(__ARM_ARCH) #include diff --git a/src/ailego/internal/cpu_features.h b/src/ailego/internal/cpu_features.h index b789a8d3..3db1dee2 100644 --- a/src/ailego/internal/cpu_features.h +++ b/src/ailego/internal/cpu_features.h @@ -14,8 +14,7 @@ #pragma once -#include "platform.h" - +#include namespace zvec { namespace ailego { namespace internal { diff --git a/src/ailego/io/file.cc b/src/ailego/io/file.cc index bfdefc8a..63bc162a 100644 --- a/src/ailego/io/file.cc +++ b/src/ailego/io/file.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "file.h" +#include "ailego/internal/platform.h" #if !defined(_WIN64) && !defined(_WIN32) #include #include diff --git a/src/ailego/io/mmap_file.h b/src/ailego/io/mmap_file.h index 286f50ae..41a39022 100644 --- a/src/ailego/io/mmap_file.h +++ b/src/ailego/io/mmap_file.h @@ -14,6 +14,7 @@ #pragma once +#include "ailego/internal/platform.h" #include "file.h" namespace zvec { diff --git a/src/ailego/utility/concurrency_helper.cc b/src/ailego/utility/concurrency_helper.cc index 5e15241a..08b47132 100644 --- a/src/ailego/utility/concurrency_helper.cc +++ b/src/ailego/utility/concurrency_helper.cc @@ -16,8 +16,8 @@ #include #include #include +#include #include "file_helper.h" -#include "string_helper.h" namespace zvec { namespace ailego { diff --git a/src/ailego/utility/file_helper.h b/src/ailego/utility/file_helper.h index 8bce9384..00f4f5d0 100644 --- a/src/ailego/utility/file_helper.h +++ b/src/ailego/utility/file_helper.h @@ -16,7 +16,6 @@ #include #include -#include namespace zvec { namespace ailego { diff --git a/src/ailego/utility/float_helper.cc b/src/ailego/utility/float_helper.cc index 164384a6..c2a8b96c 100644 --- a/src/ailego/utility/float_helper.cc +++ b/src/ailego/utility/float_helper.cc @@ -12,8 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "float_helper.h" #include +#include +#include // #if defined(__F16C__) && defined(__AVX__) // #define float16(x) _cvtss_sh((x), _MM_FROUND_NO_EXC) diff --git a/src/ailego/utility/math_helper.h b/src/ailego/utility/math_helper.h index 211346e5..01f7e18b 100644 --- a/src/ailego/utility/math_helper.h +++ b/src/ailego/utility/math_helper.h @@ -18,7 +18,7 @@ #include #include #include -#include "float_helper.h" +#include namespace zvec { namespace ailego { diff --git a/src/ailego/utility/memory_helper.cc b/src/ailego/utility/memory_helper.cc index b2f00948..bb4ef8d3 100644 --- a/src/ailego/utility/memory_helper.cc +++ b/src/ailego/utility/memory_helper.cc @@ -18,8 +18,8 @@ #include #include #include +#include #include "file_helper.h" -#include "string_helper.h" #if defined(_WIN64) || defined(_WIN32) #include diff --git a/src/ailego/utility/string_helper.cc b/src/ailego/utility/string_helper.cc index cdd81db5..6c201578 100644 --- a/src/ailego/utility/string_helper.cc +++ b/src/ailego/utility/string_helper.cc @@ -12,8 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "string_helper.h" +#include #include +#include namespace zvec { namespace ailego { diff --git a/src/ailego/utility/type_helper.h b/src/ailego/utility/type_helper.h index 576a9e3c..778ff471 100644 --- a/src/ailego/utility/type_helper.h +++ b/src/ailego/utility/type_helper.h @@ -16,7 +16,7 @@ #include #include -#include "float_helper.h" +#include namespace zvec { namespace ailego { diff --git a/src/ailego/version.i b/src/ailego/version.i index e46c0496..198dbb83 100644 --- a/src/ailego/version.i +++ b/src/ailego/version.i @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "internal/platform.h" +#include "ailego/internal/platform.h" #ifndef AILEGO_VERSION_TO_STRING_ #define AILEGO_VERSION_TO_STRING_(x) #x diff --git a/src/binding/python/CMakeLists.txt b/src/binding/python/CMakeLists.txt index 07074c72..83bfd4ce 100644 --- a/src/binding/python/CMakeLists.txt +++ b/src/binding/python/CMakeLists.txt @@ -32,7 +32,7 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux") $ $ -Wl,--no-whole-archive - zvec + zvec_db ) target_link_options(_zvec PRIVATE "LINKER:--version-script=${CMAKE_CURRENT_SOURCE_DIR}/exports.map" @@ -49,7 +49,7 @@ elseif (APPLE) -Wl,-force_load,$ -Wl,-force_load,$ -Wl,-force_load,$ - zvec + zvec_db ) target_link_libraries(_zvec PRIVATE -Wl,-exported_symbols_list,${CMAKE_CURRENT_SOURCE_DIR}/exports.mac diff --git a/src/binding/python/include/python_collection.h b/src/binding/python/include/python_collection.h index 4a365bc6..7c417756 100644 --- a/src/binding/python/include/python_collection.h +++ b/src/binding/python/include/python_collection.h @@ -13,7 +13,7 @@ // limitations under the License.#pragma once #include -#include "db/collection.h" +#include namespace py = pybind11; diff --git a/src/binding/python/include/python_config.h b/src/binding/python/include/python_config.h index 65f5986f..5cb927c4 100644 --- a/src/binding/python/include/python_config.h +++ b/src/binding/python/include/python_config.h @@ -13,7 +13,7 @@ // limitations under the License.#pragma once #include -#include "db/common/config.h" +#include namespace py = pybind11; diff --git a/src/binding/python/include/python_doc.h b/src/binding/python/include/python_doc.h index 9998db09..c3867105 100644 --- a/src/binding/python/include/python_doc.h +++ b/src/binding/python/include/python_doc.h @@ -13,7 +13,7 @@ // limitations under the License.#pragma once #include -#include "db/index/common/doc.h" +#include namespace py = pybind11; diff --git a/src/binding/python/include/python_param.h b/src/binding/python/include/python_param.h index ac5ab73e..65ac563a 100644 --- a/src/binding/python/include/python_param.h +++ b/src/binding/python/include/python_param.h @@ -13,8 +13,8 @@ // limitations under the License.#pragma once #include -#include "db/index/common/options.h" -#include "db/index/common/type.h" +#include +#include namespace py = pybind11; diff --git a/src/binding/python/include/python_schema.h b/src/binding/python/include/python_schema.h index eba8031a..d151ef2c 100644 --- a/src/binding/python/include/python_schema.h +++ b/src/binding/python/include/python_schema.h @@ -13,7 +13,7 @@ // limitations under the License.#pragma once #include -#include "db/index/common/type.h" +#include namespace py = pybind11; diff --git a/src/binding/python/include/python_type.h b/src/binding/python/include/python_type.h index 221af5f1..90a5b67d 100644 --- a/src/binding/python/include/python_type.h +++ b/src/binding/python/include/python_type.h @@ -13,8 +13,8 @@ // limitations under the License.#pragma once #include -#include "db/common/status.h" -#include "db/index/common/type.h" +#include +#include namespace py = pybind11; diff --git a/src/binding/python/model/param/python_param.cc b/src/binding/python/model/param/python_param.cc index 0a528d15..98c2adf4 100644 --- a/src/binding/python/model/param/python_param.cc +++ b/src/binding/python/model/param/python_param.cc @@ -15,8 +15,8 @@ #include "python_param.h" #include #include -#include "core/interface/constants.h" -#include "db/index/common/index_params.h" +#include +#include #include "python_doc.h" namespace zvec { diff --git a/src/binding/python/model/python_collection.cc b/src/binding/python/model/python_collection.cc index 211c6711..838cfb0a 100644 --- a/src/binding/python/model/python_collection.cc +++ b/src/binding/python/model/python_collection.cc @@ -14,7 +14,7 @@ #include "python_collection.h" #include -#include "db/collection.h" +#include namespace zvec { diff --git a/src/binding/python/model/schema/python_schema.cc b/src/binding/python/model/schema/python_schema.cc index c833a774..fcf3b603 100644 --- a/src/binding/python/model/schema/python_schema.cc +++ b/src/binding/python/model/schema/python_schema.cc @@ -14,8 +14,8 @@ #include "python_schema.h" #include -#include "db/index/common/schema.h" -#include "db/index/common/stats.h" +#include +#include namespace zvec { diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index c0011397..7742db59 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -7,4 +7,15 @@ cc_directory(metric) cc_directory(quantizer) cc_directory(utility) cc_directory(interface) -cc_directory(mixed_reducer) \ No newline at end of file +cc_directory(mixed_reducer) + +git_version(GIT_SRCS_VER ${CMAKE_CURRENT_SOURCE_DIR}) +file(GLOB_RECURSE ALL_CORE_SRCS *.cc *.c *.h) + +cc_library( + NAME zvec_core STATIC STRICT PACKED + SRCS ${ALL_CORE_SRCS} + LIBS zvec_ailego sparsehash magic_enum + INCS . ${PROJECT_ROOT_DIR}/src/core + VERSION "${GIT_SRCS_VER}" +) \ No newline at end of file diff --git a/src/core/algorithm/cluster/kmeans_cluster.cc b/src/core/algorithm/cluster/kmeans_cluster.cc index d1cdfedd..21ccf409 100644 --- a/src/core/algorithm/cluster/kmeans_cluster.cc +++ b/src/core/algorithm/cluster/kmeans_cluster.cc @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. #include -#include #include +#include #include "framework/index_cluster.h" #include "framework/index_error.h" #include "framework/index_factory.h" diff --git a/src/core/algorithm/cluster/stratified_cluster_trainer.cc b/src/core/algorithm/cluster/stratified_cluster_trainer.cc index 27a2f91d..e2f96a68 100644 --- a/src/core/algorithm/cluster/stratified_cluster_trainer.cc +++ b/src/core/algorithm/cluster/stratified_cluster_trainer.cc @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. #include "stratified_cluster_trainer.h" -#include #include +#include #include "framework/index_error.h" #include "framework/index_factory.h" #include "framework/index_helper.h" diff --git a/src/core/algorithm/cluster/vector_mean.h b/src/core/algorithm/cluster/vector_mean.h index f4d021cf..529f19c3 100644 --- a/src/core/algorithm/cluster/vector_mean.h +++ b/src/core/algorithm/cluster/vector_mean.h @@ -18,8 +18,8 @@ #include #include #include -#include #include +#include namespace zvec { namespace core { diff --git a/src/core/algorithm/flat/flat_streamer_entity.cc b/src/core/algorithm/flat/flat_streamer_entity.cc index 00ac983e..80e26c4b 100644 --- a/src/core/algorithm/flat/flat_streamer_entity.cc +++ b/src/core/algorithm/flat/flat_streamer_entity.cc @@ -14,8 +14,8 @@ #include "flat_streamer_entity.h" #include +#include "framework/index_error.h" #include "flat_utility.h" -#include "index_error.h" namespace zvec { namespace core { diff --git a/src/core/algorithm/flat/flat_streamer_entity.h b/src/core/algorithm/flat/flat_streamer_entity.h index 8a2afd88..0c4b5350 100644 --- a/src/core/algorithm/flat/flat_streamer_entity.h +++ b/src/core/algorithm/flat/flat_streamer_entity.h @@ -17,10 +17,10 @@ #include #include #include -#include #include #include #include +#include #include "flat_index_format.h" #include "flat_utility.h" diff --git a/src/core/algorithm/flat_sparse/flat_sparse_streamer_entity.h b/src/core/algorithm/flat_sparse/flat_sparse_streamer_entity.h index a864555c..d1adff4d 100644 --- a/src/core/algorithm/flat_sparse/flat_sparse_streamer_entity.h +++ b/src/core/algorithm/flat_sparse/flat_sparse_streamer_entity.h @@ -20,11 +20,11 @@ #include #include #include -#include #include #include #include #include +#include #include "flat_sparse_entity.h" #include "flat_sparse_index_format.h" #include "flat_sparse_utility.h" diff --git a/src/core/algorithm/hnsw/hnsw_chunk.h b/src/core/algorithm/hnsw/hnsw_chunk.h index f042672e..5633b0b5 100644 --- a/src/core/algorithm/hnsw/hnsw_chunk.h +++ b/src/core/algorithm/hnsw/hnsw_chunk.h @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include "framework/index_error.h" #include "framework/index_logger.h" #include "framework/index_storage.h" diff --git a/src/core/algorithm/hnsw/hnsw_dist_calculator.h b/src/core/algorithm/hnsw/hnsw_dist_calculator.h index cba6918f..fa86ea15 100644 --- a/src/core/algorithm/hnsw/hnsw_dist_calculator.h +++ b/src/core/algorithm/hnsw/hnsw_dist_calculator.h @@ -209,8 +209,8 @@ class HnswDistCalculator { const void *query_; uint32_t dim_; - uint32_t compare_cnt_; // record distance compute times - uint32_t compare_cnt_batch_; // record batch distance compute time + uint32_t compare_cnt_; // record distance compute times + // uint32_t compare_cnt_batch_; // record batch distance compute time bool error_{false}; }; diff --git a/src/core/algorithm/hnsw_sparse/hnsw_sparse_chunk.h b/src/core/algorithm/hnsw_sparse/hnsw_sparse_chunk.h index b5a0086d..a850350a 100644 --- a/src/core/algorithm/hnsw_sparse/hnsw_sparse_chunk.h +++ b/src/core/algorithm/hnsw_sparse/hnsw_sparse_chunk.h @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include "framework/index_error.h" #include "framework/index_logger.h" #include "framework/index_storage.h" diff --git a/src/core/algorithm/ivf/ivf_builder.cc b/src/core/algorithm/ivf/ivf_builder.cc index 42bce135..6166b888 100644 --- a/src/core/algorithm/ivf/ivf_builder.cc +++ b/src/core/algorithm/ivf/ivf_builder.cc @@ -13,8 +13,8 @@ // limitations under the License. #include "ivf_builder.h" #include -#include -#include "cluster/cluster_params.h" +#include +#include "algorithm/cluster/cluster_params.h" #include "ivf_dumper.h" namespace zvec { diff --git a/src/core/framework/CMakeLists.txt b/src/core/framework/CMakeLists.txt index edfa1ca2..50915927 100644 --- a/src/core/framework/CMakeLists.txt +++ b/src/core/framework/CMakeLists.txt @@ -3,7 +3,7 @@ include(${PROJECT_ROOT_DIR}/cmake/option.cmake) cc_library( NAME core_framework - STATIC SHARED STRICT ALWAYS_LINK + STATIC STRICT ALWAYS_LINK SRCS *.cc LIBS zvec_ailego INCS . ${PROJECT_ROOT_DIR}/src/core diff --git a/src/core/framework/index_error.h b/src/core/framework/index_error.h index e95acc1e..b0f246f2 100644 --- a/src/core/framework/index_error.h +++ b/src/core/framework/index_error.h @@ -14,7 +14,7 @@ #pragma once #include -#include "ailego/pattern/expected.hpp" +#include namespace zvec { namespace core { diff --git a/src/core/framework/index_mapping.cc b/src/core/framework/index_mapping.cc index 470e0261..5287779c 100644 --- a/src/core/framework/index_mapping.cc +++ b/src/core/framework/index_mapping.cc @@ -496,6 +496,7 @@ bool IndexMapping::Ishugetlbfs(const std::string &path) const { } return static_cast(buf.f_type) == HUGETLBFS_MAGIC; #else + static_cast(path); return false; #endif } diff --git a/src/core/interface/CMakeLists.txt b/src/core/interface/CMakeLists.txt index 819b89dd..82b4fa78 100644 --- a/src/core/interface/CMakeLists.txt +++ b/src/core/interface/CMakeLists.txt @@ -5,6 +5,6 @@ cc_library( NAME core_interface STATIC STRICT ALWAYS_LINK SRCS *.cc indexes/*.cc INCS . ${PROJECT_ROOT_DIR}/src/ ${PROJECT_ROOT_DIR}/src/core - LIBS zvec_ailego core_framework_static sparsehash magic_enum + LIBS zvec_ailego core_framework sparsehash magic_enum VERSION "${PROXIMA_ZVEC_VERSION}" ) diff --git a/src/core/interface/index.cc b/src/core/interface/index.cc index 3b50d92c..1b6fd503 100644 --- a/src/core/interface/index.cc +++ b/src/core/interface/index.cc @@ -62,6 +62,10 @@ int Index::CreateAndInitMetric(const BaseIndexParam & /*param*/) { auto &metric_name = proxima_index_meta_.metric_name(); metric_ = core::IndexFactory::CreateMetric(metric_name); + if (!metric_) { + LOG_ERROR("Failed to create metric, name %s", metric_name.c_str()); + return core::IndexError_Runtime; + } if (const auto ret = metric_->init(proxima_index_meta_, proxima_index_meta_.metric_params()); ret != 0) { diff --git a/src/core/interface/index.h b/src/core/interface/index.h index 4b5c88c7..78d11437 100644 --- a/src/core/interface/index.h +++ b/src/core/interface/index.h @@ -309,8 +309,8 @@ class HNSWIndex : public Index { virtual int _prepare_for_search( const VectorData &query, const BaseIndexQueryParam::Pointer &search_param, core::IndexContext::Pointer &context) override; - virtual int _get_coarse_search_topk( - const BaseIndexQueryParam::Pointer &search_param); + int _get_coarse_search_topk( + const BaseIndexQueryParam::Pointer &search_param) override; private: diff --git a/src/core/interface/index_param.h b/src/core/interface/index_param.h index 626e36b9..ff7cc980 100644 --- a/src/core/interface/index_param.h +++ b/src/core/interface/index_param.h @@ -29,10 +29,10 @@ #include #include #include +#include #include "core/framework/index_filter.h" #include "core/framework/index_meta.h" #include "utils/utils.h" -#include "constants.h" namespace zvec::core_interface { #define MAX_DIMENSION 65536 diff --git a/src/core/mixed_reducer/mixed_streamer_reducer.cc b/src/core/mixed_reducer/mixed_streamer_reducer.cc index 5e8701ef..f4ea0638 100644 --- a/src/core/mixed_reducer/mixed_streamer_reducer.cc +++ b/src/core/mixed_reducer/mixed_streamer_reducer.cc @@ -14,13 +14,13 @@ #include "mixed_streamer_reducer.h" #include #include -#include #include #include #include #include #include #include +#include namespace zvec { namespace core { diff --git a/src/core/quantizer/half_float_converter.cc b/src/core/quantizer/half_float_converter.cc index 88253f9b..2db004b5 100644 --- a/src/core/quantizer/half_float_converter.cc +++ b/src/core/quantizer/half_float_converter.cc @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -#include +#include #include "framework/index_framework.h" namespace zvec { diff --git a/src/core/quantizer/half_float_reformer.cc b/src/core/quantizer/half_float_reformer.cc index f79297f4..cffe3229 100644 --- a/src/core/quantizer/half_float_reformer.cc +++ b/src/core/quantizer/half_float_reformer.cc @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -#include +#include #include "framework/index_factory.h" #include "record_quantizer.h" diff --git a/src/db/CMakeLists.txt b/src/db/CMakeLists.txt index 8fd24d94..fd95c1dc 100644 --- a/src/db/CMakeLists.txt +++ b/src/db/CMakeLists.txt @@ -2,21 +2,33 @@ include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake) include(${PROJECT_ROOT_DIR}/cmake/option.cmake) cc_proto_library( - NAME zvec_proto STATIC - SRCS proto/*.proto - PROTOROOT ./ - ) + NAME zvec_proto STATIC + SRCS proto/*.proto + PROTOROOT ./ +) cc_directory(common) cc_directory(index) cc_directory(sqlengine) +file(GLOB_RECURSE ALL_DB_SRCS *.cc *.c *.h) + cc_library( - NAME zvec STATIC STRICT - SRCS *.cc - LIBS zvec_common - zvec_index - zvec_sqlengine - INCS . - VERSION "${PROXIMA_ZVEC_VERSION}" + NAME zvec_db STATIC STRICT SRCS_NO_GLOB + SRCS ${ALL_DB_SRCS} ${CMAKE_CURRENT_BINARY_DIR}/proto/zvec.pb.cc + INCS . ${CMAKE_CURRENT_BINARY_DIR} + LIBS + zvec_ailego + zvec_core + glog + roaring + rocksdb + antlr4 + libprotobuf + Arrow::arrow_static + Arrow::arrow_compute + Arrow::arrow_dataset + Arrow::arrow_acero + DEPS zvec_proto + VERSION "${PROXIMA_ZVEC_VERSION}" ) \ No newline at end of file diff --git a/src/db/collection.cc b/src/db/collection.cc index 2149bbd5..ff260bc9 100644 --- a/src/db/collection.cc +++ b/src/db/collection.cc @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "collection.h" #include #include #include @@ -23,17 +22,20 @@ #include #include #include -#include #include -#include +#include +#include +#include +#include +#include +#include +#include #include "ailego/logger/logger.h" #include "db/common/constants.h" #include "db/common/file_helper.h" #include "db/common/profiler.h" -#include "db/common/status.h" #include "db/common/typedef.h" #include "db/index/common/delete_store.h" -#include "db/index/common/doc.h" #include "db/index/common/id_map.h" #include "db/index/common/index_filter.h" #include "db/index/common/version_manager.h" @@ -41,8 +43,6 @@ #include "db/index/segment/segment_helper.h" #include "db/index/segment/segment_manager.h" #include "db/sqlengine/sqlengine.h" -#include "index/common/options.h" -#include "index/common/schema.h" namespace zvec { diff --git a/src/db/common/concurrent_roaring_bitmap.h b/src/db/common/concurrent_roaring_bitmap.h index 5f140756..54f13628 100644 --- a/src/db/common/concurrent_roaring_bitmap.h +++ b/src/db/common/concurrent_roaring_bitmap.h @@ -21,7 +21,8 @@ #include #include #include -#include "status.h" +#include +#include "ailego/internal/platform.h" namespace zvec { diff --git a/src/db/common/config.cc b/src/db/common/config.cc index 073d5b2d..50d7a59f 100644 --- a/src/db/common/config.cc +++ b/src/db/common/config.cc @@ -12,14 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "db/common/config.h" #include #include +#include +#include +#include "db/common/constants.h" #include "db/common/global_resource.h" -#include "constants.h" +#include "cgroup_util.h" +#include "global_resource.h" #include "glogger.h" #include "logger.h" -#include "status.h" #include "typedef.h" namespace zvec { @@ -28,6 +30,15 @@ static void ExitLogHandler() { LogUtil::Shutdown(); } +GlobalConfig::ConfigData::ConfigData() + : memory_limit_bytes(CgroupUtil::getMemoryLimit() * + DEFAULT_MEMORY_LIMIT_RATIO), + log_config(std::make_shared()), + query_thread_count(CgroupUtil::getCpuLimit()), + invert_to_forward_scan_ratio(0.9), + brute_force_by_keys_ratio(0.1), + optimize_thread_count(CgroupUtil::getCpuLimit()) {} + Status GlobalConfig::Validate(const ConfigData &config) const { if (config.memory_limit_bytes < MIN_MEMORY_LIMIT_BYTES) { return Status::InvalidArgument("memory_limit_bytes must be greater than ", diff --git a/src/db/common/constants.h b/src/db/common/constants.h index 7a808bb9..39b13f44 100644 --- a/src/db/common/constants.h +++ b/src/db/common/constants.h @@ -24,20 +24,6 @@ const float DEFAULT_MEMORY_LIMIT_RATIO = 0.8f; const uint32_t MIN_MEMORY_LIMIT_BYTES = 100 * 1024 * 1024; -const uint32_t MIN_LOG_FILE_SIZE = 128; - -const uint32_t DEFAULT_LOG_FILE_SIZE = 2048; - -const uint32_t DEFAULT_LOG_OVERDUE_DAYS = 7; - -const std::string CONSOLE_LOG_TYPE_NAME = "ConsoleLogger"; - -const std::string FILE_LOG_TYPE_NAME = "AppendLogger"; - -const std::string DEFAULT_LOG_DIR = "./logs"; - -const std::string DEFAULT_LOG_BASENAME = "zvec.log"; - const uint64_t INVALID_DOC_ID = -1UL; const std::string LOCAL_ROW_ID = "_zvec_row_id_"; @@ -52,10 +38,6 @@ const int64_t kMaxRecordBatchNumRows = 4096; constexpr uint32_t MAX_ARRAY_FIELD_LEN = 32; -const uint64_t MAX_DOC_COUNT_PER_SEGMENT = 10000000; - -const uint64_t MAX_DOC_COUNT_PER_SEGMENT_MIN_THRESHOLD = 1000; - const float COMPACT_DELETE_RATIO_THRESHOLD = 0.3f; const std::regex COLLECTION_NAME_REGEX("^[a-zA-Z0-9_-]{3,64}$"); @@ -98,8 +80,5 @@ const std::string INVERT_KEY_SEALED{"$ZVEC$SEALED"}; const uint32_t INVERT_ID_LIST_SIZE_THRESHOLD = 3; -// Segment Options -const uint32_t DEFAULT_MAX_BUFFER_SIZE = 64 * 1024 * 1024; // 128M - } // namespace zvec diff --git a/src/db/common/error_code.h b/src/db/common/error_code.h index c372bdb2..625cd66a 100644 --- a/src/db/common/error_code.h +++ b/src/db/common/error_code.h @@ -14,7 +14,7 @@ #pragma once #include -#include +#include namespace zvec { /*! Error diff --git a/src/db/common/file_helper.cc b/src/db/common/file_helper.cc index 25110be9..aa766f8c 100644 --- a/src/db/common/file_helper.cc +++ b/src/db/common/file_helper.cc @@ -19,6 +19,7 @@ #include #include #include +#include #include #include diff --git a/src/db/common/file_helper.h b/src/db/common/file_helper.h index 56d058e4..4fb95621 100644 --- a/src/db/common/file_helper.h +++ b/src/db/common/file_helper.h @@ -18,7 +18,7 @@ #include #include #include -#include +#include namespace zvec { diff --git a/src/db/common/global_resource.cc b/src/db/common/global_resource.cc index 73c39e76..d17b38e8 100644 --- a/src/db/common/global_resource.cc +++ b/src/db/common/global_resource.cc @@ -14,7 +14,7 @@ #include "db/common/global_resource.h" #include #include -#include "db/common/config.h" +#include namespace zvec { diff --git a/src/db/common/global_resource.h b/src/db/common/global_resource.h index 5069c05a..ea3e8128 100644 --- a/src/db/common/global_resource.h +++ b/src/db/common/global_resource.h @@ -14,7 +14,7 @@ #pragma once #include -#include +#include #include "ailego/parallel/thread_pool.h" namespace zvec { diff --git a/src/db/common/logger.h b/src/db/common/logger.h index 5f0929a3..bbc60d3a 100644 --- a/src/db/common/logger.h +++ b/src/db/common/logger.h @@ -17,10 +17,10 @@ #include #include #include +#include #include "ailego/pattern/factory.h" -#include "constants.h" +#include "db/common/constants.h" #include "error_code.h" -#include "status.h" namespace zvec { diff --git a/src/db/common/rocksdb_context.h b/src/db/common/rocksdb_context.h index ea45b612..a833f9d3 100644 --- a/src/db/common/rocksdb_context.h +++ b/src/db/common/rocksdb_context.h @@ -18,7 +18,7 @@ #include #include -#include "status.h" +#include namespace zvec { diff --git a/src/db/common/status.cc b/src/db/common/status.cc index 46df7c88..afad85d2 100644 --- a/src/db/common/status.cc +++ b/src/db/common/status.cc @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "status.h" #include -#include +#include +#include namespace zvec { diff --git a/src/db/index/column/inverted_column/inverted_codec.h b/src/db/index/column/inverted_column/inverted_codec.h index 906425e9..62fb76ac 100644 --- a/src/db/index/column/inverted_column/inverted_codec.h +++ b/src/db/index/column/inverted_column/inverted_codec.h @@ -19,9 +19,9 @@ #include #include #include +#include +#include #include "db/common/constants.h" -#include "db/common/status.h" -#include "db/index/common/type.h" namespace zvec { diff --git a/src/db/index/column/inverted_column/inverted_column_indexer.h b/src/db/index/column/inverted_column/inverted_column_indexer.h index 3eecfd91..94e3c05b 100644 --- a/src/db/index/column/inverted_column/inverted_column_indexer.h +++ b/src/db/index/column/inverted_column/inverted_column_indexer.h @@ -16,10 +16,10 @@ #pragma once -#include +#include +#include #include "db/common/concurrent_roaring_bitmap.h" #include "db/common/rocksdb_context.h" -#include "db/index/common/schema.h" #include "inverted_codec.h" #include "inverted_doc_range.h" #include "inverted_search_result.h" diff --git a/src/db/index/column/inverted_column/inverted_column_indexer_search.cc b/src/db/index/column/inverted_column/inverted_column_indexer_search.cc index 2df4c83d..ac7681ca 100644 --- a/src/db/index/column/inverted_column/inverted_column_indexer_search.cc +++ b/src/db/index/column/inverted_column/inverted_column_indexer_search.cc @@ -15,7 +15,7 @@ #include #include -#include +#include #include "inverted_codec.h" #include "inverted_column_indexer.h" diff --git a/src/db/index/column/inverted_column/inverted_doc_range.h b/src/db/index/column/inverted_column/inverted_doc_range.h index 0d1db9a6..bdb10329 100644 --- a/src/db/index/column/inverted_column/inverted_doc_range.h +++ b/src/db/index/column/inverted_column/inverted_doc_range.h @@ -21,7 +21,7 @@ #include #include #include -#include "db/index/common/type.h" +#include namespace zvec { diff --git a/src/db/index/column/vector_column/engine_helper.hpp b/src/db/index/column/vector_column/engine_helper.hpp index e6e8dddd..c2865c5c 100644 --- a/src/db/index/column/vector_column/engine_helper.hpp +++ b/src/db/index/column/vector_column/engine_helper.hpp @@ -14,11 +14,11 @@ #pragma once #include -#include +#include +#include +#include +#include #include "core/interface/index.h" -#include "db/common/status.h" -#include "db/index/common/doc.h" -#include "db/index/common/query_params.h" #include "vector_column_indexer.h" #include "vector_column_params.h" diff --git a/src/db/index/column/vector_column/vector_column_indexer.cc b/src/db/index/column/vector_column/vector_column_indexer.cc index 1b9fdf78..d56a2cb3 100644 --- a/src/db/index/column/vector_column/vector_column_indexer.cc +++ b/src/db/index/column/vector_column/vector_column_indexer.cc @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. #include "vector_column_indexer.h" -#include "ailego/pattern/expected.hpp" -#include "db/common/status.h" +#include +#include #include "engine_helper.hpp" diff --git a/src/db/index/column/vector_column/vector_column_indexer.h b/src/db/index/column/vector_column/vector_column_indexer.h index ad03eccd..756e93f4 100644 --- a/src/db/index/column/vector_column/vector_column_indexer.h +++ b/src/db/index/column/vector_column/vector_column_indexer.h @@ -16,17 +16,17 @@ #include #include #include -#include -#include #include +#include +#include +#include +#include #include "core/interface/index.h" #include "core/interface/index_param.h" #include "db/common/constants.h" -#include "db/common/status.h" #include "db/common/typedef.h" #include "db/index/column/common/index_results.h" #include "db/index/common/meta.h" -#include "db/index/common/schema.h" #include "vector_column_params.h" #include "vector_index_results.h" diff --git a/src/db/index/column/vector_column/vector_column_params.h b/src/db/index/column/vector_column/vector_column_params.h index b265c78e..1db49438 100644 --- a/src/db/index/column/vector_column/vector_column_params.h +++ b/src/db/index/column/vector_column/vector_column_params.h @@ -21,10 +21,10 @@ #include // #include // #include "common/constants.h" +#include +#include #include "core/interface/index_param.h" #include "db/index/common/index_filter.h" -#include "db/index/common/query_params.h" -#include "db/index/common/type.h" namespace zvec { class VectorColumnIndexer; diff --git a/src/db/index/common/doc.cc b/src/db/index/common/doc.cc index 66480cfa..dad9bbdd 100644 --- a/src/db/index/common/doc.cc +++ b/src/db/index/common/doc.cc @@ -11,12 +11,14 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -#include "doc.h" #include #include #include #include #include +#include +#include "db/common/constants.h" +#include "db/index/common/type_helper.h" #if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ #define IS_BIG_ENDIAN 1 diff --git a/src/db/index/common/id_map.h b/src/db/index/common/id_map.h index fa5313ab..02ac9aa4 100644 --- a/src/db/index/common/id_map.h +++ b/src/db/index/common/id_map.h @@ -18,8 +18,8 @@ #include #include #include +#include #include "db/common/rocksdb_context.h" -#include "db/common/status.h" namespace zvec { diff --git a/src/db/index/common/index_params.cc b/src/db/index/common/index_params.cc new file mode 100644 index 00000000..cb06f077 --- /dev/null +++ b/src/db/index/common/index_params.cc @@ -0,0 +1,41 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include "type_helper.h" + +namespace zvec { + +std::string InvertIndexParams::to_string() const { + std::ostringstream oss; + oss << "InvertIndexParams{" + << "enable_range_optimization:" + << (enable_range_optimization_ ? "true" : "false") + << ", enable_extended_wildcard:" + << (enable_extended_wildcard_ ? "true" : "false") << "}"; + return oss.str(); +} + +std::string VectorIndexParams::vector_index_params_to_string( + const std::string &class_name, MetricType metric_type, + QuantizeType quantize_type) const { + std::ostringstream oss; + oss << class_name << "{" + << "metric:" << MetricTypeCodeBook::AsString(metric_type) + << ",quantize:" << QuantizeTypeCodeBook::AsString(quantize_type); + return oss.str(); +} + +} // namespace zvec \ No newline at end of file diff --git a/src/db/index/common/meta.h b/src/db/index/common/meta.h index c38d498f..ae0ba3df 100644 --- a/src/db/index/common/meta.h +++ b/src/db/index/common/meta.h @@ -124,8 +124,8 @@ class BlockMeta { std::string to_string() const { std::ostringstream oss; - oss << "BlockMeta{" << "id:" << id_ - << ",type:" << BlockTypeCodeBook::AsString(type_) + oss << "BlockMeta{" + << "id:" << id_ << ",type:" << BlockTypeCodeBook::AsString(type_) << ",min_doc_id:" << min_doc_id_ << ",max_doc_id:" << max_doc_id_ << ",doc_count:" << doc_count_ << ",columns:["; @@ -358,7 +358,8 @@ class SegmentMeta { std::string to_string() const { std::ostringstream oss; - oss << "SegmentMeta{" << "id:" << id_ << ",persisted_blocks:["; + oss << "SegmentMeta{" + << "id:" << id_ << ",persisted_blocks:["; for (size_t i = 0; i < persisted_blocks_.size(); ++i) { if (i > 0) oss << ","; diff --git a/src/db/index/common/proto_converter.h b/src/db/index/common/proto_converter.h index 9b9c8513..48e17016 100644 --- a/src/db/index/common/proto_converter.h +++ b/src/db/index/common/proto_converter.h @@ -13,9 +13,9 @@ // limitations under the License. #pragma once -#include "db/index/common/index_params.h" +#include +#include #include "db/index/common/meta.h" -#include "db/index/common/schema.h" namespace zvec { diff --git a/src/db/index/common/schema.cc b/src/db/index/common/schema.cc index f8abbc29..ef50bf99 100644 --- a/src/db/index/common/schema.cc +++ b/src/db/index/common/schema.cc @@ -11,15 +11,19 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -#include "schema.h" + #include +#include #include #include -#include "common/index_params.h" -#include "common/type.h" -#include "common/type_helper.h" -#include "db/common/status.h" +#include +#include +#include +#include +#include "db/common/constants.h" #include "db/common/typedef.h" +#include "db/common/utils.h" +#include "db/index/common/type_helper.h" namespace zvec { @@ -175,6 +179,48 @@ Status FieldSchema::validate() const { return Status::OK(); } +std::string FieldSchema::to_string() const { + std::ostringstream oss; + oss << "FieldSchema{" + << "name:'" << name_ << "'" + << ",data_type:" << DataTypeCodeBook::AsString(data_type_) + << ",nullable:" << (nullable_ ? "true" : "false") + << ",dimension:" << dimension_; + + if (index_params_) { + oss << ",index_params:" << index_params_->to_string(); + } else { + oss << ",index_params:null"; + } + + oss << "}"; + return oss.str(); +} + +std::string FieldSchema::to_string_formatted(int indent_level) const { + std::ostringstream oss; + oss << indent(indent_level) << "FieldSchema{\n" + << indent(indent_level + 1) << "name: '" << name_ << "',\n" + << indent(indent_level + 1) + << "data_type: " << DataTypeCodeBook::AsString(data_type_) << ",\n" + << indent(indent_level + 1) + << "nullable: " << (nullable_ ? "true" : "false") << ",\n"; + + if (dimension_ != 0) { + oss << indent(indent_level + 1) << "dimension: " << dimension_ << ",\n"; + } + + if (index_params_) { + oss << indent(indent_level + 1) + << "index_params: " << index_params_->to_string() << "\n"; + } else { + oss << indent(indent_level + 1) << "index_params: null\n"; + } + + oss << indent(indent_level) << "}"; + return oss.str(); +} + Status CollectionSchema::validate() const { if (name_.empty()) { return Status::InvalidArgument("schema validate failed: name is empty"); @@ -211,6 +257,43 @@ Status CollectionSchema::validate() const { return Status::OK(); } +std::string CollectionSchema::to_string() const { + std::ostringstream oss; + oss << "CollectionSchema{" + << "name:'" << name_ << "'" + << ",max_doc_count_per_segment:" << max_doc_count_per_segment_ + << ",fields:["; + + for (size_t i = 0; i < fields_.size(); ++i) { + if (i > 0) oss << ","; + oss << fields_[i]->to_string(); + } + + oss << "]}"; + return oss.str(); +} + + +std::string CollectionSchema::to_string_formatted(int indent_level) const { + std::ostringstream oss; + oss << indent(indent_level) << "CollectionSchema{\n" + << indent(indent_level + 1) << "name: '" << name_ << "',\n" + << indent(indent_level + 1) + << "max_doc_count_per_segment: " << max_doc_count_per_segment_ << ",\n" + << indent(indent_level + 1) << "fields: [\n"; + + for (size_t i = 0; i < fields_.size(); ++i) { + oss << fields_[i]->to_string_formatted(indent_level + 2); + if (i < fields_.size() - 1) { + oss << ","; + } + oss << "\n"; + } + + oss << indent(indent_level + 1) << "]\n" << indent(indent_level) << "}"; + return oss.str(); +} + Status CollectionSchema::add_field(FieldSchema::Ptr column_schema) { // Check if field already exists if (has_field(column_schema->name())) { diff --git a/src/db/index/common/stats.cc b/src/db/index/common/stats.cc new file mode 100644 index 00000000..fdb26f25 --- /dev/null +++ b/src/db/index/common/stats.cc @@ -0,0 +1,57 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include "db/common/utils.h" + +namespace zvec { +std::string CollectionStats::to_string() const { + std::ostringstream oss; + oss << "CollectionStats{" + << "doc_count:" << doc_count << ",index_completeness:{"; + + size_t i = 0; + for (const auto &pair : index_completeness) { + if (i > 0) oss << ","; + oss << pair.first << ":" << pair.second; + ++i; + } + + oss << "}}"; + return oss.str(); +} + +std::string CollectionStats::to_string_formatted(int indent_level) const { + std::ostringstream oss; + oss << indent(indent_level) << "CollectionStats{\n" + << indent(indent_level + 1) << "doc_count: " << doc_count << ",\n" + << indent(indent_level + 1) << "index_completeness: {\n"; + + size_t i = 0; + for (const auto &pair : index_completeness) { + if (i > 0) oss << ",\n"; + oss << indent(indent_level + 2) << pair.first << ": " << pair.second; + ++i; + } + + if (!index_completeness.empty()) { + oss << "\n"; + } + oss << indent(indent_level + 1) << "}\n" << indent(indent_level) << "}"; + + return oss.str(); +} + +} // namespace zvec \ No newline at end of file diff --git a/src/db/index/common/stats.h b/src/db/index/common/stats.h deleted file mode 100644 index 2563d173..00000000 --- a/src/db/index/common/stats.h +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2025-present the zvec project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#pragma once - -#include -#include -#include -#include -#include "db/common/utils.h" - -namespace zvec { - -/* - * Collection stats - */ -struct CollectionStats { - uint64_t doc_count{0}; - // column -> completeness - std::unordered_map index_completeness; - - std::string to_string() const { - std::ostringstream oss; - oss << "CollectionStats{" << "doc_count:" << doc_count - << ",index_completeness:{"; - - size_t i = 0; - for (const auto &pair : index_completeness) { - if (i > 0) oss << ","; - oss << pair.first << ":" << pair.second; - ++i; - } - - oss << "}}"; - return oss.str(); - } - - std::string to_string_formatted(int indent_level = 0) const { - std::ostringstream oss; - oss << indent(indent_level) << "CollectionStats{\n" - << indent(indent_level + 1) << "doc_count: " << doc_count << ",\n" - << indent(indent_level + 1) << "index_completeness: {\n"; - - size_t i = 0; - for (const auto &pair : index_completeness) { - if (i > 0) oss << ",\n"; - oss << indent(indent_level + 2) << pair.first << ": " << pair.second; - ++i; - } - - if (!index_completeness.empty()) { - oss << "\n"; - } - oss << indent(indent_level + 1) << "}\n" << indent(indent_level) << "}"; - - return oss.str(); - } -}; - -} // namespace zvec \ No newline at end of file diff --git a/src/db/index/common/type_helper.h b/src/db/index/common/type_helper.h index 0f01c333..967a8b19 100644 --- a/src/db/index/common/type_helper.h +++ b/src/db/index/common/type_helper.h @@ -14,7 +14,7 @@ #pragma once #include -#include "db/index/common/type.h" +#include #include "proto/zvec.pb.h" namespace zvec { diff --git a/src/db/index/common/version_manager.cc b/src/db/index/common/version_manager.cc index 278af257..97e3c3c7 100644 --- a/src/db/index/common/version_manager.cc +++ b/src/db/index/common/version_manager.cc @@ -22,11 +22,11 @@ #include #include #include -#include #include -#include "ailego/pattern/expected.hpp" +#include +#include +#include #include "db/common/file_helper.h" -#include "db/common/status.h" #include "db/common/typedef.h" #include "db/index/common/proto_converter.h" #include "db/index/common/type_helper.h" diff --git a/src/db/index/common/version_manager.h b/src/db/index/common/version_manager.h index c9d189f1..bed0a724 100644 --- a/src/db/index/common/version_manager.h +++ b/src/db/index/common/version_manager.h @@ -19,9 +19,9 @@ #include #include #include -#include "db/common/status.h" +#include +#include #include "db/index/common/meta.h" -#include "db/index/common/schema.h" namespace zvec { diff --git a/src/db/index/segment/segment.cc b/src/db/index/segment/segment.cc index 7124eff8..6fc2ac30 100644 --- a/src/db/index/segment/segment.cc +++ b/src/db/index/segment/segment.cc @@ -31,28 +31,28 @@ #include #include #include -#include "column/inverted_column/inverted_indexer.h" -#include "column/vector_column/vector_column_indexer.h" -#include "column/vector_column/vector_column_params.h" -#include "common/doc.h" -#include "common/index_filter.h" -#include "common/index_params.h" -#include "common/meta.h" -#include "common/schema.h" -#include "common/type.h" -#include "db/common/config.h" +#include +#include +#include +#include +#include +#include #include "db/common/constants.h" #include "db/common/file_helper.h" #include "db/common/global_resource.h" -#include "db/common/status.h" #include "db/common/typedef.h" +#include "db/index/column/inverted_column/inverted_indexer.h" +#include "db/index/column/vector_column/vector_column_indexer.h" +#include "db/index/column/vector_column/vector_column_params.h" +#include "db/index/common/index_filter.h" +#include "db/index/common/meta.h" +#include "db/index/segment/segment_helper.h" +#include "db/index/storage/base_forward_store.h" +#include "db/index/storage/bufferpool_forward_store.h" +#include "db/index/storage/memory_forward_store.h" +#include "db/index/storage/mmap_forward_store.h" #include "db/index/storage/store_helper.h" -#include "segment/segment_helper.h" -#include "storage/base_forward_store.h" -#include "storage/bufferpool_forward_store.h" -#include "storage/memory_forward_store.h" -#include "storage/mmap_forward_store.h" -#include "storage/wal/wal_file.h" +#include "db/index/storage/wal/wal_file.h" #include "column_merging_reader.h" #include "sql_expr_parser.h" diff --git a/src/db/index/segment/segment.h b/src/db/index/segment/segment.h index 778ca4a9..2e6e9bbc 100644 --- a/src/db/index/segment/segment.h +++ b/src/db/index/segment/segment.h @@ -18,20 +18,20 @@ #include #include #include -#include #include -#include "db/common/status.h" +#include +#include +#include +#include +#include +#include #include "db/index/column/inverted_column/inverted_column_indexer.h" #include "db/index/column/inverted_column/inverted_indexer.h" #include "db/index/column/vector_column/combined_vector_column_indexer.h" #include "db/index/column/vector_column/vector_column_indexer.h" #include "db/index/common/delete_store.h" -#include "db/index/common/doc.h" #include "db/index/common/id_map.h" -#include "db/index/common/index_params.h" #include "db/index/common/meta.h" -#include "db/index/common/options.h" -#include "db/index/common/schema.h" #include "db/index/common/version_manager.h" #include "db/index/storage/base_forward_store.h" diff --git a/src/db/index/segment/segment_helper.cc b/src/db/index/segment/segment_helper.cc index 1b47d48c..08a7a81d 100644 --- a/src/db/index/segment/segment_helper.cc +++ b/src/db/index/segment/segment_helper.cc @@ -18,17 +18,17 @@ #include #include #include +#include +#include #include "ailego/logger/logger.h" -#include "column/inverted_column/inverted_indexer.h" -#include "column/vector_column/vector_column_indexer.h" -#include "common/meta.h" #include "db/common/constants.h" #include "db/common/file_helper.h" #include "db/common/global_resource.h" -#include "db/common/status.h" #include "db/common/typedef.h" +#include "db/index/column/inverted_column/inverted_indexer.h" +#include "db/index/column/vector_column/vector_column_indexer.h" #include "db/index/common/index_filter.h" -#include "db/index/common/type.h" +#include "db/index/common/meta.h" #include "db/index/storage/forward_writer.h" #include "roaring.hh" diff --git a/src/db/index/segment/segment_helper.h b/src/db/index/segment/segment_helper.h index 9565c670..31b09631 100644 --- a/src/db/index/segment/segment_helper.h +++ b/src/db/index/segment/segment_helper.h @@ -20,9 +20,9 @@ #include #include #include +#include #include "db/index/column/inverted_column/inverted_indexer.h" #include "db/index/common/index_filter.h" -#include "db/index/common/index_params.h" #include "db/index/common/meta.h" #include "segment.h" diff --git a/src/db/index/segment/segment_manager.cc b/src/db/index/segment/segment_manager.cc index a5b2c783..1f9e546a 100644 --- a/src/db/index/segment/segment_manager.cc +++ b/src/db/index/segment/segment_manager.cc @@ -18,7 +18,7 @@ #include #include #include -#include "db/common/status.h" +#include #include "db/common/typedef.h" namespace zvec { diff --git a/src/db/index/storage/base_forward_store.h b/src/db/index/storage/base_forward_store.h index f5a50bbc..1ec42b1c 100644 --- a/src/db/index/storage/base_forward_store.h +++ b/src/db/index/storage/base_forward_store.h @@ -20,7 +20,7 @@ #include #include #include -#include "db/common/status.h" +#include namespace cp = arrow::compute; diff --git a/src/db/index/storage/bufferpool_forward_store.cc b/src/db/index/storage/bufferpool_forward_store.cc index 11415fe5..60b0ad3c 100644 --- a/src/db/index/storage/bufferpool_forward_store.cc +++ b/src/db/index/storage/bufferpool_forward_store.cc @@ -23,7 +23,7 @@ #include #include #include -#include "storage/store_helper.h" +#include "db/index/storage/store_helper.h" #include "lazy_record_batch_reader.h" diff --git a/src/db/index/storage/bufferpool_forward_store.h b/src/db/index/storage/bufferpool_forward_store.h index 1a506962..c0c357af 100644 --- a/src/db/index/storage/bufferpool_forward_store.h +++ b/src/db/index/storage/bufferpool_forward_store.h @@ -24,7 +24,7 @@ #include #include #include -#include "db/common/status.h" +#include #include "base_forward_store.h" namespace zvec { diff --git a/src/db/index/storage/chunked_file_writer.h b/src/db/index/storage/chunked_file_writer.h index 3406d8eb..56c62cfc 100644 --- a/src/db/index/storage/chunked_file_writer.h +++ b/src/db/index/storage/chunked_file_writer.h @@ -19,7 +19,7 @@ #include #include #include -#include "db/index/common/type.h" +#include namespace zvec { diff --git a/src/db/index/storage/memory_forward_store.h b/src/db/index/storage/memory_forward_store.h index 1a7ec12e..7c341263 100644 --- a/src/db/index/storage/memory_forward_store.h +++ b/src/db/index/storage/memory_forward_store.h @@ -21,8 +21,8 @@ #include #include #include -#include "db/common/status.h" -#include "db/index/common/doc.h" +#include +#include #include "base_forward_store.h" #include "chunked_file_writer.h" #include "store_helper.h" diff --git a/src/db/index/storage/mmap_forward_store.cc b/src/db/index/storage/mmap_forward_store.cc index 53649742..f96127c0 100644 --- a/src/db/index/storage/mmap_forward_store.cc +++ b/src/db/index/storage/mmap_forward_store.cc @@ -19,7 +19,7 @@ #include #include #include -#include "storage/base_forward_store.h" +#include "db/index/storage/base_forward_store.h" #include "lazy_record_batch_reader.h" diff --git a/src/db/index/storage/mmap_forward_store.h b/src/db/index/storage/mmap_forward_store.h index c47a97eb..8c018a6e 100644 --- a/src/db/index/storage/mmap_forward_store.h +++ b/src/db/index/storage/mmap_forward_store.h @@ -33,7 +33,7 @@ #include #include #include -#include "db/common/status.h" +#include #include "base_forward_store.h" #include "store_helper.h" diff --git a/src/db/index/storage/store_helper.h b/src/db/index/storage/store_helper.h index d6b97ba4..f9b3b515 100644 --- a/src/db/index/storage/store_helper.h +++ b/src/db/index/storage/store_helper.h @@ -29,10 +29,11 @@ #include #include #include +#include +#include +#include "db/common/constants.h" #include "db/common/file_helper.h" -#include "db/index/common/doc.h" #include "db/index/common/meta.h" -#include "db/index/common/schema.h" #include "chunked_file_writer.h" diff --git a/src/db/sqlengine/CMakeLists.txt b/src/db/sqlengine/CMakeLists.txt index 61b8bf7a..f32c7b0d 100644 --- a/src/db/sqlengine/CMakeLists.txt +++ b/src/db/sqlengine/CMakeLists.txt @@ -1,23 +1,11 @@ include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake) include(${PROJECT_ROOT_DIR}/cmake/option.cmake) - -cc_library( - NAME zvec_parser STATIC STRICT - SRCS common/*.cc antlr/gen/*.cc parser/*.cc - LIBS zvec_common - antlr4 - zvec_index - INCS ${PROJECT_ROOT_DIR}/src - VERSION "${PROXIMA_ZVEC_VERSION}" -) - cc_library( NAME zvec_sqlengine STATIC STRICT - SRCS *.cc analyzer/*.cc planner/*.cc planner/ops/*.cc planner/physical_rules/*.cc + SRCS *.cc common/*.cc antlr/gen/*.cc parser/*.cc analyzer/*.cc planner/*.cc planner/ops/*.cc planner/physical_rules/*.cc LIBS zvec_index zvec_common - zvec_parser antlr4 Arrow::arrow_acero INCS . ${PROJECT_ROOT_DIR}/src diff --git a/src/db/sqlengine/analyzer/query_analyzer.cc b/src/db/sqlengine/analyzer/query_analyzer.cc index 39f60fc3..bd01174e 100644 --- a/src/db/sqlengine/analyzer/query_analyzer.cc +++ b/src/db/sqlengine/analyzer/query_analyzer.cc @@ -15,16 +15,16 @@ #include "query_analyzer.h" #include #include -#include -#include "ailego/pattern/expected.hpp" -#include "analyzer/query_node.h" +#include +#include +#include +#include +#include #include "core/framework/index_meta.h" -#include "db/common/config.h" #include "db/common/constants.h" #include "db/common/error_code.h" -#include "db/common/status.h" -#include "db/index/common/type.h" #include "db/index/common/type_helper.h" +#include "db/sqlengine/analyzer/query_node.h" #include "db/sqlengine/common/util.h" #include "db/sqlengine/parser/select_info.h" #include "query_info_helper.h" diff --git a/src/db/sqlengine/analyzer/query_analyzer.h b/src/db/sqlengine/analyzer/query_analyzer.h index cb615925..386b47cf 100644 --- a/src/db/sqlengine/analyzer/query_analyzer.h +++ b/src/db/sqlengine/analyzer/query_analyzer.h @@ -17,7 +17,7 @@ #include #include #include -#include "db/common/status.h" +#include #include "db/sqlengine/parser/sql_info.h" #include "query_info.h" #include "query_node_walker.h" diff --git a/src/db/sqlengine/analyzer/query_field_info.h b/src/db/sqlengine/analyzer/query_field_info.h index 12872d18..ac68fe1b 100644 --- a/src/db/sqlengine/analyzer/query_field_info.h +++ b/src/db/sqlengine/analyzer/query_field_info.h @@ -16,7 +16,7 @@ #include #include -#include "db/index/common/schema.h" +#include namespace zvec::sqlengine { diff --git a/src/db/sqlengine/analyzer/query_info.cc b/src/db/sqlengine/analyzer/query_info.cc index 92f0abbf..f6f06631 100644 --- a/src/db/sqlengine/analyzer/query_info.cc +++ b/src/db/sqlengine/analyzer/query_info.cc @@ -13,8 +13,8 @@ // limitations under the License. #include "query_info.h" -#include -#include +#include +#include #include "db/common/constants.h" namespace zvec::sqlengine { diff --git a/src/db/sqlengine/analyzer/query_info.h b/src/db/sqlengine/analyzer/query_info.h index 4db0b04c..5dfd7381 100644 --- a/src/db/sqlengine/analyzer/query_info.h +++ b/src/db/sqlengine/analyzer/query_info.h @@ -20,8 +20,8 @@ #include #include #include +#include #include "db/common/constants.h" -#include "db/index/common/schema.h" #include "db/sqlengine/common/group_by.h" #include "query_field_info.h" #include "query_node.h" diff --git a/src/db/sqlengine/analyzer/query_info_helper.cc b/src/db/sqlengine/analyzer/query_info_helper.cc index cbf342ed..058f7ba9 100644 --- a/src/db/sqlengine/analyzer/query_info_helper.cc +++ b/src/db/sqlengine/analyzer/query_info_helper.cc @@ -15,7 +15,7 @@ #include "query_info_helper.h" #include #include -#include +#include namespace zvec::sqlengine { diff --git a/src/db/sqlengine/analyzer/query_node.h b/src/db/sqlengine/analyzer/query_node.h index 75e52d6c..f54d1dcb 100644 --- a/src/db/sqlengine/analyzer/query_node.h +++ b/src/db/sqlengine/analyzer/query_node.h @@ -17,7 +17,7 @@ #include #include #include -#include "db/index/common/query_params.h" +#include #include "db/sqlengine/common/generic_node.h" #include "db/sqlengine/parser/node.h" diff --git a/src/db/sqlengine/analyzer/query_node_walker.cc b/src/db/sqlengine/analyzer/query_node_walker.cc index d94122d9..2993f046 100644 --- a/src/db/sqlengine/analyzer/query_node_walker.cc +++ b/src/db/sqlengine/analyzer/query_node_walker.cc @@ -15,13 +15,14 @@ #include "query_node_walker.h" #include #include -#include -#include -#include "ailego/pattern/expected.hpp" -#include "analyzer/query_node.h" +#include +#include +#include +#include +#include #include "db/common/constants.h" -#include "db/index/common/index_params.h" -#include "db/index/common/type.h" +#include "db/index/common/type_helper.h" +#include "db/sqlengine/analyzer/query_node.h" #include "db/sqlengine/common/util.h" #include "query_info_helper.h" diff --git a/src/db/sqlengine/analyzer/query_node_walker.h b/src/db/sqlengine/analyzer/query_node_walker.h index d20d5b9d..b8d6ad49 100644 --- a/src/db/sqlengine/analyzer/query_node_walker.h +++ b/src/db/sqlengine/analyzer/query_node_walker.h @@ -17,9 +17,9 @@ #include #include #include -#include "ailego/pattern/expected.hpp" -#include "analyzer/query_node.h" -#include "db/index/common/type.h" +#include +#include +#include "db/sqlengine/analyzer/query_node.h" #include "query_info.h" namespace zvec::sqlengine { diff --git a/src/db/sqlengine/analyzer/query_orderby_info.h b/src/db/sqlengine/analyzer/query_orderby_info.h index 206fa3e4..54438646 100644 --- a/src/db/sqlengine/analyzer/query_orderby_info.h +++ b/src/db/sqlengine/analyzer/query_orderby_info.h @@ -16,7 +16,7 @@ #include #include -#include "db/index/common/schema.h" +#include namespace zvec::sqlengine { diff --git a/src/db/sqlengine/analyzer/simple_rewriter.cc b/src/db/sqlengine/analyzer/simple_rewriter.cc index b2906b26..a2114c0f 100644 --- a/src/db/sqlengine/analyzer/simple_rewriter.cc +++ b/src/db/sqlengine/analyzer/simple_rewriter.cc @@ -16,7 +16,7 @@ #include #include #include -#include "analyzer/query_node.h" +#include "db/sqlengine/analyzer/query_node.h" namespace zvec::sqlengine { diff --git a/src/db/sqlengine/common/group_by.h b/src/db/sqlengine/common/group_by.h index 34dc4381..c442bab9 100644 --- a/src/db/sqlengine/common/group_by.h +++ b/src/db/sqlengine/common/group_by.h @@ -16,8 +16,8 @@ #include #include -#include -#include "db/index/common/doc.h" +#include +#include namespace zvec::sqlengine { diff --git a/src/db/sqlengine/common/util.cc b/src/db/sqlengine/common/util.cc index d2a55208..5c5d231b 100644 --- a/src/db/sqlengine/common/util.cc +++ b/src/db/sqlengine/common/util.cc @@ -19,9 +19,9 @@ #include #include #include -#include #include #include +#include namespace zvec::sqlengine { diff --git a/src/db/sqlengine/parser/node.h b/src/db/sqlengine/parser/node.h index fcbe3cf0..a08dd4fe 100644 --- a/src/db/sqlengine/parser/node.h +++ b/src/db/sqlengine/parser/node.h @@ -17,7 +17,7 @@ #include #include #include -#include "db/index/common/query_params.h" +#include #include "db/sqlengine/common/generic_node.h" namespace zvec::sqlengine { diff --git a/src/db/sqlengine/parser/sql_info_helper.cc b/src/db/sqlengine/parser/sql_info_helper.cc index 5f7c02b0..ae7a2ccf 100644 --- a/src/db/sqlengine/parser/sql_info_helper.cc +++ b/src/db/sqlengine/parser/sql_info_helper.cc @@ -15,8 +15,8 @@ #include "sql_info_helper.h" #include #include -#include -#include "db/index/common/doc.h" +#include +#include #include "db/sqlengine/common/group_by.h" #include "db/sqlengine/common/util.h" #include "db/sqlengine/parser/node.h" diff --git a/src/db/sqlengine/parser/sql_info_helper.h b/src/db/sqlengine/parser/sql_info_helper.h index ca334cfa..465ccdce 100644 --- a/src/db/sqlengine/parser/sql_info_helper.h +++ b/src/db/sqlengine/parser/sql_info_helper.h @@ -14,7 +14,7 @@ #pragma once -#include "db/index/common/doc.h" +#include #include "db/sqlengine/common/group_by.h" #include "db/sqlengine/parser/node.h" #include "db/sqlengine/parser/sql_info.h" diff --git a/src/db/sqlengine/parser/zvec_cached_sql_parser.cc b/src/db/sqlengine/parser/zvec_cached_sql_parser.cc index b02c0292..52147693 100644 --- a/src/db/sqlengine/parser/zvec_cached_sql_parser.cc +++ b/src/db/sqlengine/parser/zvec_cached_sql_parser.cc @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include "atn/ParserATNSimulator.h" #include "db/sqlengine/antlr/gen/SQLLexer.h" #include "db/sqlengine/antlr/gen/SQLParser.h" diff --git a/src/db/sqlengine/parser/zvec_parser.cc b/src/db/sqlengine/parser/zvec_parser.cc index 5e1cfcf5..91ce93dc 100644 --- a/src/db/sqlengine/parser/zvec_parser.cc +++ b/src/db/sqlengine/parser/zvec_parser.cc @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include "db/sqlengine/common/util.h" #include "tree/ParseTree.h" #include "zvec_cached_sql_parser.h" diff --git a/src/db/sqlengine/planner/doc_filter.cc b/src/db/sqlengine/planner/doc_filter.cc index 0c6618de..43ce1d5a 100644 --- a/src/db/sqlengine/planner/doc_filter.cc +++ b/src/db/sqlengine/planner/doc_filter.cc @@ -16,8 +16,8 @@ #include #include #include +#include #include "ailego/logger/logger.h" -#include "db/common/config.h" #include "db/sqlengine/planner/invert_search.h" namespace zvec::sqlengine { diff --git a/src/db/sqlengine/planner/doc_filter.h b/src/db/sqlengine/planner/doc_filter.h index 1d0669d0..b662a742 100644 --- a/src/db/sqlengine/planner/doc_filter.h +++ b/src/db/sqlengine/planner/doc_filter.h @@ -17,7 +17,7 @@ #include #include #include -#include "db/common/status.h" +#include #include "db/index/column/inverted_column/inverted_search_result.h" #include "db/index/common/index_filter.h" #include "db/index/segment/segment.h" diff --git a/src/db/sqlengine/planner/invert_search.cc b/src/db/sqlengine/planner/invert_search.cc index df36223a..8cd76ab2 100644 --- a/src/db/sqlengine/planner/invert_search.cc +++ b/src/db/sqlengine/planner/invert_search.cc @@ -14,8 +14,8 @@ #include "invert_search.h" #include -#include "analyzer/query_node.h" -#include "db/index/common/type.h" +#include +#include "db/sqlengine/analyzer/query_node.h" #include "db/sqlengine/common/util.h" namespace zvec::sqlengine { diff --git a/src/db/sqlengine/planner/invert_search.h b/src/db/sqlengine/planner/invert_search.h index bfae2590..fc291c2d 100644 --- a/src/db/sqlengine/planner/invert_search.h +++ b/src/db/sqlengine/planner/invert_search.h @@ -14,9 +14,9 @@ #pragma once -#include -#include "analyzer/query_node.h" +#include #include "db/index/segment/segment.h" +#include "db/sqlengine/analyzer/query_node.h" namespace zvec::sqlengine { diff --git a/src/db/sqlengine/planner/ops/contain_op.cc b/src/db/sqlengine/planner/ops/contain_op.cc index e59e000f..9eddb26d 100644 --- a/src/db/sqlengine/planner/ops/contain_op.cc +++ b/src/db/sqlengine/planner/ops/contain_op.cc @@ -15,7 +15,7 @@ #include "db/sqlengine/planner/ops/contain_op.h" #include #include -#include "db/index/common/type.h" +#include #include "db/sqlengine/common/util.h" namespace zvec::sqlengine { diff --git a/src/db/sqlengine/planner/ops/contain_op.h b/src/db/sqlengine/planner/ops/contain_op.h index ad3bdc1d..004fc34e 100644 --- a/src/db/sqlengine/planner/ops/contain_op.h +++ b/src/db/sqlengine/planner/ops/contain_op.h @@ -17,7 +17,7 @@ #include #include #include -#include "db/index/common/type.h" +#include namespace zvec::sqlengine { diff --git a/src/db/sqlengine/planner/ops/fetch_vector_op.h b/src/db/sqlengine/planner/ops/fetch_vector_op.h index 847fb64d..1fa76738 100644 --- a/src/db/sqlengine/planner/ops/fetch_vector_op.h +++ b/src/db/sqlengine/planner/ops/fetch_vector_op.h @@ -15,7 +15,7 @@ #pragma once #include -#include "db/common/status.h" +#include #include "db/index/column/vector_column/combined_vector_column_indexer.h" #include "db/index/segment/segment.h" diff --git a/src/db/sqlengine/planner/optimizer.cc b/src/db/sqlengine/planner/optimizer.cc index 6c8908b8..f63950e1 100644 --- a/src/db/sqlengine/planner/optimizer.cc +++ b/src/db/sqlengine/planner/optimizer.cc @@ -14,8 +14,8 @@ #include "optimizer.h" #include -#include "db/common/config.h" -#include "db/index/common/type.h" +#include +#include #include "db/sqlengine/analyzer/query_info_helper.h" #include "db/sqlengine/common/util.h" #include "db/sqlengine/planner/invert_search.h" diff --git a/src/db/sqlengine/planner/plan_info.cc b/src/db/sqlengine/planner/plan_info.cc index 9a072721..6f7681c5 100644 --- a/src/db/sqlengine/planner/plan_info.cc +++ b/src/db/sqlengine/planner/plan_info.cc @@ -15,7 +15,7 @@ #include "plan_info.h" #include #include -#include "ailego/pattern/expected.hpp" +#include #include "db/common/error_code.h" namespace zvec::sqlengine { diff --git a/src/db/sqlengine/planner/plan_info.h b/src/db/sqlengine/planner/plan_info.h index a5b7daee..1c45d0bf 100644 --- a/src/db/sqlengine/planner/plan_info.h +++ b/src/db/sqlengine/planner/plan_info.h @@ -19,7 +19,7 @@ #include #include #include -#include "db/common/status.h" +#include namespace zvec::sqlengine { diff --git a/src/db/sqlengine/planner/query_planner.cc b/src/db/sqlengine/planner/query_planner.cc index c56a364c..e4d87786 100644 --- a/src/db/sqlengine/planner/query_planner.cc +++ b/src/db/sqlengine/planner/query_planner.cc @@ -20,21 +20,21 @@ #include #include #include -#include "analyzer/query_info.h" -#include "analyzer/query_node.h" +#include +#include +#include #include "db/common/constants.h" #include "db/common/global_resource.h" -#include "db/common/status.h" -#include "db/index/common/schema.h" -#include "db/index/common/type.h" +#include "db/sqlengine/analyzer/query_info.h" +#include "db/sqlengine/analyzer/query_node.h" #include "db/sqlengine/common/util.h" #include "db/sqlengine/planner/invert_recall_node.h" #include "db/sqlengine/planner/ops/check_not_filtered_op.h" #include "db/sqlengine/planner/ops/contain_op.h" #include "db/sqlengine/planner/ops/fetch_vector_op.h" +#include "db/sqlengine/planner/plan_info.h" #include "db/sqlengine/planner/segment_node.h" #include "db/sqlengine/planner/vector_recall_node.h" -#include "planner/plan_info.h" #include "optimizer.h" namespace zvec::sqlengine { diff --git a/src/db/sqlengine/planner/query_planner.h b/src/db/sqlengine/planner/query_planner.h index 51104169..b93fa34e 100644 --- a/src/db/sqlengine/planner/query_planner.h +++ b/src/db/sqlengine/planner/query_planner.h @@ -16,10 +16,10 @@ #include #include -#include #include #include -#include "db/common/status.h" +#include +#include #include "db/index/segment/segment.h" #include "db/sqlengine/analyzer/query_info.h" #include "plan_info.h" diff --git a/src/db/sqlengine/planner/segment_node.h b/src/db/sqlengine/planner/segment_node.h index 74a86a48..88524059 100644 --- a/src/db/sqlengine/planner/segment_node.h +++ b/src/db/sqlengine/planner/segment_node.h @@ -21,7 +21,7 @@ #include #include #include -#include "db/common/status.h" +#include #include "db/sqlengine/planner/plan_info.h" namespace zvec::sqlengine { diff --git a/src/db/sqlengine/planner/vector_recall_node.cc b/src/db/sqlengine/planner/vector_recall_node.cc index f715d176..aa5dd1e4 100644 --- a/src/db/sqlengine/planner/vector_recall_node.cc +++ b/src/db/sqlengine/planner/vector_recall_node.cc @@ -20,11 +20,11 @@ #include #include #include -#include "ailego/pattern/expected.hpp" +#include +#include +#include +#include #include "db/index/column/vector_column/vector_column_params.h" -#include "db/index/common/index_params.h" -#include "db/index/common/schema.h" -#include "db/index/common/type.h" #include "db/sqlengine/common/util.h" #include "db/sqlengine/planner/ops/fetch_vector_op.h" diff --git a/src/db/sqlengine/planner/vector_recall_node.h b/src/db/sqlengine/planner/vector_recall_node.h index 7a78d887..19b9bd29 100644 --- a/src/db/sqlengine/planner/vector_recall_node.h +++ b/src/db/sqlengine/planner/vector_recall_node.h @@ -17,7 +17,7 @@ #include #include #include -#include "db/common/status.h" +#include #include "db/index/column/common/index_results.h" #include "db/index/segment/segment.h" #include "db/sqlengine/analyzer/query_info.h" diff --git a/src/db/sqlengine/sqlengine.h b/src/db/sqlengine/sqlengine.h index b3755ac4..d86fd69b 100644 --- a/src/db/sqlengine/sqlengine.h +++ b/src/db/sqlengine/sqlengine.h @@ -14,9 +14,9 @@ #pragma once +#include +#include #include "db/common/profiler.h" -#include "db/common/status.h" -#include "db/index/common/doc.h" #include "db/index/segment/segment.h" namespace zvec::sqlengine { diff --git a/src/db/sqlengine/sqlengine_impl.cc b/src/db/sqlengine/sqlengine_impl.cc index 1bf677b6..f889adb1 100644 --- a/src/db/sqlengine/sqlengine_impl.cc +++ b/src/db/sqlengine/sqlengine_impl.cc @@ -15,9 +15,9 @@ #include "db/sqlengine/sqlengine_impl.h" #include #include +#include +#include #include "db/common/constants.h" -#include "db/index/common/doc.h" -#include "db/index/common/type.h" #include "db/sqlengine/analyzer/query_analyzer.h" #include "db/sqlengine/parser/sql_info_helper.h" #include "db/sqlengine/parser/zvec_parser.h" diff --git a/src/db/sqlengine/sqlengine_impl.h b/src/db/sqlengine/sqlengine_impl.h index b5ac8cdc..88c27928 100644 --- a/src/db/sqlengine/sqlengine_impl.h +++ b/src/db/sqlengine/sqlengine_impl.h @@ -18,10 +18,10 @@ #include #include #include +#include +#include #include "analyzer/query_info.h" #include "common/group_by.h" -#include "db/index/common/doc.h" -#include "db/index/common/schema.h" #include "db/sqlengine/common/util.h" #include "db/sqlengine/parser/sql_info.h" #include "db/sqlengine/sqlengine.h" diff --git a/src/ailego/pattern/expected.hpp b/src/include/zvec/ailego/pattern/expected.hpp similarity index 100% rename from src/ailego/pattern/expected.hpp rename to src/include/zvec/ailego/pattern/expected.hpp diff --git a/src/ailego/pattern/singleton.h b/src/include/zvec/ailego/pattern/singleton.h similarity index 100% rename from src/ailego/pattern/singleton.h rename to src/include/zvec/ailego/pattern/singleton.h diff --git a/src/ailego/string/string_concat_helper.h b/src/include/zvec/ailego/string/string_concat_helper.h similarity index 98% rename from src/ailego/string/string_concat_helper.h rename to src/include/zvec/ailego/string/string_concat_helper.h index 9348e0a7..78a830ba 100644 --- a/src/ailego/string/string_concat_helper.h +++ b/src/include/zvec/ailego/string/string_concat_helper.h @@ -16,7 +16,7 @@ #include #include -#include "string_view.h" +#include namespace zvec { namespace ailego { diff --git a/src/ailego/string/string_view.h b/src/include/zvec/ailego/string/string_view.h similarity index 98% rename from src/ailego/string/string_view.h rename to src/include/zvec/ailego/string/string_view.h index 42d74575..3fb1b80a 100644 --- a/src/ailego/string/string_view.h +++ b/src/include/zvec/ailego/string/string_view.h @@ -14,6 +14,7 @@ #pragma once +#include #include namespace zvec { diff --git a/src/ailego/utility/float_helper.h b/src/include/zvec/ailego/utility/float_helper.h similarity index 99% rename from src/ailego/utility/float_helper.h rename to src/include/zvec/ailego/utility/float_helper.h index 7a6d140c..5dc2fe69 100644 --- a/src/ailego/utility/float_helper.h +++ b/src/include/zvec/ailego/utility/float_helper.h @@ -14,7 +14,8 @@ #pragma once -#include +#include +#include namespace zvec { namespace ailego { diff --git a/src/ailego/utility/string_helper.h b/src/include/zvec/ailego/utility/string_helper.h similarity index 98% rename from src/ailego/utility/string_helper.h rename to src/include/zvec/ailego/utility/string_helper.h index 1f540557..7a924ec0 100644 --- a/src/ailego/utility/string_helper.h +++ b/src/include/zvec/ailego/utility/string_helper.h @@ -14,12 +14,10 @@ #pragma once -#include #include #include -#include -#include -#include "string_helper_impl.h" +#include +#include namespace zvec { namespace ailego { diff --git a/src/ailego/utility/string_helper_impl.h b/src/include/zvec/ailego/utility/string_helper_impl.h similarity index 100% rename from src/ailego/utility/string_helper_impl.h rename to src/include/zvec/ailego/utility/string_helper_impl.h diff --git a/src/core/interface/constants.h b/src/include/zvec/core/interface/constants.h similarity index 100% rename from src/core/interface/constants.h rename to src/include/zvec/core/interface/constants.h diff --git a/src/db/collection.h b/src/include/zvec/db/collection.h similarity index 94% rename from src/db/collection.h rename to src/include/zvec/db/collection.h index da194419..3e868bbd 100644 --- a/src/db/collection.h +++ b/src/include/zvec/db/collection.h @@ -16,12 +16,10 @@ #include #include #include -#include -#include -#include "db/common/status.h" -#include "index/common/doc.h" -#include "index/common/options.h" -#include "index/common/stats.h" +#include +#include +#include +#include namespace zvec { diff --git a/src/db/common/config.h b/src/include/zvec/db/config.h similarity index 87% rename from src/db/common/config.h rename to src/include/zvec/db/config.h index fd3c09e4..29fe1967 100644 --- a/src/db/common/config.h +++ b/src/include/zvec/db/config.h @@ -16,14 +16,19 @@ #include #include #include -#include -#include -#include "cgroup_util.h" -#include "constants.h" -#include "status.h" +#include +#include namespace zvec { +const uint32_t MIN_LOG_FILE_SIZE = 128; +const uint32_t DEFAULT_LOG_FILE_SIZE = 2048; +const uint32_t DEFAULT_LOG_OVERDUE_DAYS = 7; +const std::string CONSOLE_LOG_TYPE_NAME = "ConsoleLogger"; +const std::string FILE_LOG_TYPE_NAME = "AppendLogger"; +const std::string DEFAULT_LOG_DIR = "./logs"; +const std::string DEFAULT_LOG_BASENAME = "zvec.log"; + class GlobalConfig : public ailego::Singleton { friend class ailego::Singleton; @@ -78,20 +83,20 @@ class GlobalConfig : public ailego::Singleton { // Configuration data structure struct ConfigData { - uint64_t memory_limit_bytes = - CgroupUtil::getMemoryLimit() * DEFAULT_MEMORY_LIMIT_RATIO; + uint64_t memory_limit_bytes; // log - std::shared_ptr log_config = - std::make_shared(); + std::shared_ptr log_config; // query - uint32_t query_thread_count = CgroupUtil::getCpuLimit(); - float invert_to_forward_scan_ratio = 0.9; - float brute_force_by_keys_ratio = 0.1; + uint32_t query_thread_count; + float invert_to_forward_scan_ratio; + float brute_force_by_keys_ratio; // optimize - uint32_t optimize_thread_count = CgroupUtil::getCpuLimit(); + uint32_t optimize_thread_count; + + ConfigData(); }; // Initialize the configuration (can only be called once) diff --git a/src/db/index/common/doc.h b/src/include/zvec/db/doc.h similarity index 98% rename from src/db/index/common/doc.h rename to src/include/zvec/db/doc.h index 8b2faba9..5f927fa1 100644 --- a/src/db/index/common/doc.h +++ b/src/include/zvec/db/doc.h @@ -19,11 +19,11 @@ #include #include #include -#include -#include "db/common/status.h" -#include "query_params.h" -#include "schema.h" -#include "type.h" +#include +#include +#include +#include +#include namespace zvec { diff --git a/src/db/index/common/index_params.h b/src/include/zvec/db/index_params.h similarity index 93% rename from src/db/index/common/index_params.h rename to src/include/zvec/db/index_params.h index 1c04baea..4bfa3bf9 100644 --- a/src/db/index/common/index_params.h +++ b/src/include/zvec/db/index_params.h @@ -16,9 +16,8 @@ #include #include #include -#include "core/interface/constants.h" -#include "db/index/common/type_helper.h" -#include "type.h" +#include +#include namespace zvec { @@ -74,14 +73,7 @@ class InvertIndexParams : public IndexParams { enable_extended_wildcard_); } - std::string to_string() const override { - std::ostringstream oss; - oss << "InvertIndexParams{" << "enable_range_optimization:" - << (enable_range_optimization_ ? "true" : "false") - << ", enable_extended_wildcard:" - << (enable_extended_wildcard_ ? "true" : "false") << "}"; - return oss.str(); - } + std::string to_string() const override; bool operator==(const IndexParams &other) const override { if (type() != other.type()) { @@ -131,13 +123,7 @@ class VectorIndexParams : public IndexParams { std::string vector_index_params_to_string(const std::string &class_name, MetricType metric_type, - QuantizeType quantize_type) const { - std::ostringstream oss; - oss << class_name << "{" - << "metric:" << MetricTypeCodeBook::AsString(metric_type) - << ",quantize:" << QuantizeTypeCodeBook::AsString(quantize_type); - return oss.str(); - } + QuantizeType quantize_type) const; MetricType metric_type() const { return metric_type_; diff --git a/src/db/index/common/options.h b/src/include/zvec/db/options.h similarity index 77% rename from src/db/index/common/options.h rename to src/include/zvec/db/options.h index d98cf557..1f2a9cbf 100644 --- a/src/db/index/common/options.h +++ b/src/include/zvec/db/options.h @@ -14,14 +14,14 @@ #pragma once #include -#include "db/common/constants.h" namespace zvec { +const uint32_t DEFAULT_MAX_BUFFER_SIZE = 64 * 1024 * 1024; // 128M struct CollectionOptions { - bool read_only_; - bool enable_mmap_; // ignnored when load collection + bool read_only_{false}; + bool enable_mmap_{true}; // ignnored when load collection uint32_t max_buffer_size_{ DEFAULT_MAX_BUFFER_SIZE}; // ignored when read_only=true @@ -34,6 +34,14 @@ struct CollectionOptions { bool operator!=(const CollectionOptions &other) const { return !(*this == other); } + + CollectionOptions() = default; + + CollectionOptions(bool read_only, bool enable_mmap, + uint32_t max_buffer_size = DEFAULT_MAX_BUFFER_SIZE) + : read_only_(read_only), + enable_mmap_(enable_mmap), + max_buffer_size_(max_buffer_size) {} }; struct SegmentOptions { diff --git a/src/db/index/common/query_params.h b/src/include/zvec/db/query_params.h similarity index 97% rename from src/db/index/common/query_params.h rename to src/include/zvec/db/query_params.h index cc5e1ffa..d187d762 100644 --- a/src/db/index/common/query_params.h +++ b/src/include/zvec/db/query_params.h @@ -14,8 +14,8 @@ #pragma once #include -#include "core/interface/constants.h" -#include "type.h" +#include +#include namespace zvec { diff --git a/src/db/index/common/schema.h b/src/include/zvec/db/schema.h similarity index 80% rename from src/db/index/common/schema.h rename to src/include/zvec/db/schema.h index 1ead0326..09885c16 100644 --- a/src/db/index/common/schema.h +++ b/src/include/zvec/db/schema.h @@ -15,14 +15,15 @@ #include #include -#include "db/common/constants.h" -#include "db/common/status.h" -#include "db/common/utils.h" -#include "index_params.h" -#include "type.h" +#include +#include +#include namespace zvec { +const uint64_t MAX_DOC_COUNT_PER_SEGMENT = 10000000; +const uint64_t MAX_DOC_COUNT_PER_SEGMENT_MIN_THRESHOLD = 1000; + /* * Field schema */ @@ -95,44 +96,9 @@ class FieldSchema { return !(*this == other); } - std::string to_string() const { - std::ostringstream oss; - oss << "FieldSchema{" - << "name:'" << name_ << "'" - << ",data_type:" << DataTypeCodeBook::AsString(data_type_) - << ",nullable:" << (nullable_ ? "true" : "false") - << ",dimension:" << dimension_; - - if (index_params_) { - oss << ",index_params:" << index_params_->to_string(); - } else { - oss << ",index_params:null"; - } - - oss << "}"; - return oss.str(); - } - - std::string to_string_formatted(int indent_level = 0) const { - std::ostringstream oss; - oss << indent(indent_level) << "FieldSchema{\n" - << indent(indent_level + 1) << "name: '" << name_ << "',\n" - << indent(indent_level + 1) - << "data_type: " << DataTypeCodeBook::AsString(data_type_) << ",\n" - << indent(indent_level + 1) - << "nullable: " << (nullable_ ? "true" : "false") << ",\n" - << indent(indent_level + 1) << "dimension: " << dimension_ << ",\n"; - - if (index_params_) { - oss << indent(indent_level + 1) - << "index_params: " << index_params_->to_string() << "\n"; - } else { - oss << indent(indent_level + 1) << "index_params: null\n"; - } + std::string to_string() const; - oss << indent(indent_level) << "}"; - return oss.str(); - } + std::string to_string_formatted(int indent_level = 0) const; public: void set_name(const std::string &name) { @@ -335,42 +301,10 @@ class CollectionSchema { } public: - std::string to_string() const { - std::ostringstream oss; - oss << "CollectionSchema{" - << "name:'" << name_ << "'" - << ",max_doc_count_per_segment:" << max_doc_count_per_segment_ - << ",fields:["; - - for (size_t i = 0; i < fields_.size(); ++i) { - if (i > 0) oss << ","; - oss << fields_[i]->to_string(); - } - - oss << "]}"; - return oss.str(); - } + std::string to_string() const; - std::string to_string_formatted(int indent_level = 0) const { - std::ostringstream oss; - oss << indent(indent_level) << "CollectionSchema{\n" - << indent(indent_level + 1) << "name: '" << name_ << "',\n" - << indent(indent_level + 1) - << "max_doc_count_per_segment: " << max_doc_count_per_segment_ << ",\n" - << indent(indent_level + 1) << "fields: [\n"; - - for (size_t i = 0; i < fields_.size(); ++i) { - oss << fields_[i]->to_string_formatted(indent_level + 2); - if (i < fields_.size() - 1) { - oss << ","; - } - oss << "\n"; - } - - oss << indent(indent_level + 1) << "]\n" << indent(indent_level) << "}"; - return oss.str(); - } + std::string to_string_formatted(int indent_level = 0) const; std::string name() const { return name_; diff --git a/src/include/zvec/db/stats.h b/src/include/zvec/db/stats.h new file mode 100644 index 00000000..ea9bf9ad --- /dev/null +++ b/src/include/zvec/db/stats.h @@ -0,0 +1,35 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include +#include +#include + +namespace zvec { + +/* + * Collection stats + */ +struct CollectionStats { + uint64_t doc_count{0}; + // column -> completeness + std::unordered_map index_completeness; + + std::string to_string() const; + + std::string to_string_formatted(int indent_level = 0) const; +}; + +} // namespace zvec \ No newline at end of file diff --git a/src/db/common/status.h b/src/include/zvec/db/status.h similarity index 98% rename from src/db/common/status.h rename to src/include/zvec/db/status.h index 95683683..dac3f8f2 100644 --- a/src/db/common/status.h +++ b/src/include/zvec/db/status.h @@ -14,8 +14,8 @@ #pragma once #include #include -#include -#include +#include +#include namespace zvec { diff --git a/src/db/index/common/type.h b/src/include/zvec/db/type.h similarity index 100% rename from src/db/index/common/type.h rename to src/include/zvec/db/type.h diff --git a/tests/ailego/pattern/singleton_test.cc b/tests/ailego/pattern/singleton_test.cc index e17023dd..951495db 100644 --- a/tests/ailego/pattern/singleton_test.cc +++ b/tests/ailego/pattern/singleton_test.cc @@ -13,8 +13,8 @@ // limitations under the License. #include -#include #include +#include using namespace zvec::ailego; diff --git a/tests/ailego/utility/float_helper_test.cc b/tests/ailego/utility/float_helper_test.cc index 4b980e0d..db780608 100644 --- a/tests/ailego/utility/float_helper_test.cc +++ b/tests/ailego/utility/float_helper_test.cc @@ -15,8 +15,8 @@ #include #include #include -#include #include +#include using namespace zvec; diff --git a/tests/ailego/utility/string_helper_test.cc b/tests/ailego/utility/string_helper_test.cc index 2c226d69..137600de 100644 --- a/tests/ailego/utility/string_helper_test.cc +++ b/tests/ailego/utility/string_helper_test.cc @@ -14,8 +14,8 @@ #include #include -#include #include +#include using namespace zvec; diff --git a/tests/ailego/utility/type_helper_test.cc b/tests/ailego/utility/type_helper_test.cc index 6b057dec..57e5a20b 100644 --- a/tests/ailego/utility/type_helper_test.cc +++ b/tests/ailego/utility/type_helper_test.cc @@ -13,9 +13,9 @@ // limitations under the License. #include -#include #include #include +#include using namespace zvec; diff --git a/tests/core/metric/cosine_metric_test.cc b/tests/core/metric/cosine_metric_test.cc index d476eeb1..7265f023 100644 --- a/tests/core/metric/cosine_metric_test.cc +++ b/tests/core/metric/cosine_metric_test.cc @@ -13,8 +13,8 @@ // limitations under the License. #include #include -#include #include +#include #include "framework/index_factory.h" diff --git a/tests/core/metric/quantized_integer_metric_test.cc b/tests/core/metric/quantized_integer_metric_test.cc index bc7ac2fb..0d7309be 100644 --- a/tests/core/metric/quantized_integer_metric_test.cc +++ b/tests/core/metric/quantized_integer_metric_test.cc @@ -18,10 +18,10 @@ #include #include #include -#include #include #include #include +#include #include "core/quantizer/quantizer_params.h" #include "framework/index_factory.h" diff --git a/tests/db/CMakeLists.txt b/tests/db/CMakeLists.txt index afef267e..29f7fcc3 100644 --- a/tests/db/CMakeLists.txt +++ b/tests/db/CMakeLists.txt @@ -25,7 +25,8 @@ foreach(CC_SRCS ${ALL_TEST_SRCS}) get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE) cc_gmock( NAME ${CC_TARGET} STRICT - LIBS zvec + LIBS zvec_db + zvec_proto core_knn_flat core_knn_flat_sparse core_knn_hnsw diff --git a/tests/db/collection_test.cc b/tests/db/collection_test.cc index a203651c..47e533b5 100644 --- a/tests/db/collection_test.cc +++ b/tests/db/collection_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "db/collection.h" +#include "zvec/db/collection.h" #include #include #include @@ -26,17 +26,17 @@ #include #include #include "ailego/logger/logger.h" -#include "ailego/utility/float_helper.h" -#include "db/common/config.h" #include "db/common/file_helper.h" -#include "db/common/status.h" -#include "db/index/common/doc.h" -#include "db/index/common/index_params.h" -#include "db/index/common/options.h" -#include "db/index/common/schema.h" -#include "db/index/common/type.h" #include "db/index/common/type_helper.h" #include "index/utils/utils.h" +#include "zvec/ailego/utility/float_helper.h" +#include "zvec/db/config.h" +#include "zvec/db/doc.h" +#include "zvec/db/index_params.h" +#include "zvec/db/options.h" +#include "zvec/db/schema.h" +#include "zvec/db/status.h" +#include "zvec/db/type.h" using namespace zvec; using namespace zvec::test; @@ -4117,7 +4117,7 @@ TEST_F(CollectionTest, CornerCase_CreateAndOpen) { // create collection with non-exist path with read-only mode auto schema = TestHelper::CreateNormalSchema(); auto result = Collection::CreateAndOpen("non-exist-path", *schema, - CollectionOptions{true}); + CollectionOptions{true, false}); ASSERT_FALSE(result.has_value()); } diff --git a/tests/db/common/config_test.cc b/tests/db/common/config_test.cc index 63c61bf9..4cc0d4df 100644 --- a/tests/db/common/config_test.cc +++ b/tests/db/common/config_test.cc @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "db/common/config.h" +#include "zvec/db/config.h" #include #include #include "db/common/constants.h" -#include "db/common/status.h" +#include "zvec/db/status.h" using namespace zvec; diff --git a/tests/db/common/status_test.cc b/tests/db/common/status_test.cc index 8a7ae1ed..28cf5ac0 100644 --- a/tests/db/common/status_test.cc +++ b/tests/db/common/status_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "db/common/status.h" +#include "zvec/db/status.h" #include using namespace zvec; diff --git a/tests/db/index/CMakeLists.txt b/tests/db/index/CMakeLists.txt index c6c78162..e06e736e 100644 --- a/tests/db/index/CMakeLists.txt +++ b/tests/db/index/CMakeLists.txt @@ -22,8 +22,8 @@ foreach(CC_SRCS ${ALL_TEST_SRCS}) get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE) cc_gmock( NAME ${CC_TARGET} STRICT - LIBS zvec - zvec_sqlengine + LIBS zvec_db + zvec_proto core_metric_static core_utility_static core_quantizer_static diff --git a/tests/db/index/column/vector_column_indexer_test.cc b/tests/db/index/column/vector_column_indexer_test.cc index e40b796f..483efcde 100644 --- a/tests/db/index/column/vector_column_indexer_test.cc +++ b/tests/db/index/column/vector_column_indexer_test.cc @@ -17,10 +17,10 @@ #include #include #include -#include "ailego/utility/float_helper.h" #include "db/index/column/vector_column/vector_column_params.h" -#include "db/index/common/doc.h" -#include "db/index/common/index_params.h" +#include "zvec/ailego/utility/float_helper.h" +#include "zvec/db/doc.h" +#include "zvec/db/index_params.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push diff --git a/tests/db/index/common/doc_test.cc b/tests/db/index/common/doc_test.cc index 36b27f8c..5047ecb5 100644 --- a/tests/db/index/common/doc_test.cc +++ b/tests/db/index/common/doc_test.cc @@ -12,15 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "db/index/common/doc.h" +#include "zvec/db/doc.h" #include #include #include -#include #include -#include "db/common/status.h" -#include "db/index/common/type.h" +#include #include "utils/utils.h" +#include "zvec/db/status.h" +#include "zvec/db/type.h" using namespace zvec; diff --git a/tests/db/index/common/index_params_test.cc b/tests/db/index/common/index_params_test.cc index 914be3df..af67e739 100644 --- a/tests/db/index/common/index_params_test.cc +++ b/tests/db/index/common/index_params_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "db/index/common/index_params.h" +#include "zvec/db/index_params.h" #include using namespace zvec; diff --git a/tests/db/index/common/query_params_test.cc b/tests/db/index/common/query_params_test.cc index 2519f094..012d1775 100644 --- a/tests/db/index/common/query_params_test.cc +++ b/tests/db/index/common/query_params_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "db/index/common/query_params.h" +#include "zvec/db/query_params.h" #include using namespace zvec; diff --git a/tests/db/index/common/schema_test.cc b/tests/db/index/common/schema_test.cc index 28cd051d..fe026a6f 100644 --- a/tests/db/index/common/schema_test.cc +++ b/tests/db/index/common/schema_test.cc @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "db/index/common/schema.h" +#include "zvec/db/schema.h" #include -#include "db/common/status.h" -#include "db/index/common/index_params.h" +#include "zvec/db/index_params.h" +#include "zvec/db/status.h" using namespace zvec; diff --git a/tests/db/index/common/version_manager_test.cc b/tests/db/index/common/version_manager_test.cc index 29d4ae0f..7a16a0ca 100644 --- a/tests/db/index/common/version_manager_test.cc +++ b/tests/db/index/common/version_manager_test.cc @@ -18,8 +18,8 @@ #include #include "db/common/file_helper.h" #include "db/index/common/meta.h" -#include "db/index/common/schema.h" #include "proto/zvec.pb.h" +#include "zvec/db/schema.h" namespace zvec { diff --git a/tests/db/index/segment/segment_helper_test.cc b/tests/db/index/segment/segment_helper_test.cc index fdf7cefc..03f1c377 100644 --- a/tests/db/index/segment/segment_helper_test.cc +++ b/tests/db/index/segment/segment_helper_test.cc @@ -31,11 +31,11 @@ #include "db/index/common/delete_store.h" #include "db/index/common/id_map.h" #include "db/index/common/meta.h" -#include "db/index/common/options.h" -#include "db/index/common/schema.h" #include "db/index/common/version_manager.h" #include "db/index/segment/segment.h" #include "utils/utils.h" +#include "zvec/db/options.h" +#include "zvec/db/schema.h" using namespace zvec; diff --git a/tests/db/index/segment/segment_test.cc b/tests/db/index/segment/segment_test.cc index 91d6f6b5..57c1f510 100644 --- a/tests/db/index/segment/segment_test.cc +++ b/tests/db/index/segment/segment_test.cc @@ -33,11 +33,11 @@ #include "db/common/file_helper.h" #include "db/index/common/delete_store.h" #include "db/index/common/id_map.h" -#include "db/index/common/options.h" #include "db/index/common/version_manager.h" #include "db/index/storage/store_helper.h" #include "db/index/storage/wal/wal_file.h" #include "utils/utils.h" +#include "zvec/db/options.h" using namespace zvec; diff --git a/tests/db/index/storage/mem_store_test.cc b/tests/db/index/storage/mem_store_test.cc index a8a79cf0..27c01914 100644 --- a/tests/db/index/storage/mem_store_test.cc +++ b/tests/db/index/storage/mem_store_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "db/index/common/doc.h" +#include "zvec/db/doc.h" #define private public #define protected public #include "db/index/storage/memory_forward_store.h" diff --git a/tests/db/index/storage/wal_file_test.cc b/tests/db/index/storage/wal_file_test.cc index facd77be..79d4ba71 100644 --- a/tests/db/index/storage/wal_file_test.cc +++ b/tests/db/index/storage/wal_file_test.cc @@ -23,9 +23,9 @@ #include #include #include -#include #include #include +#include #include "db/common/file_helper.h" #if defined(__GNUC__) || defined(__GNUG__) diff --git a/tests/db/index/utils/utils.cc b/tests/db/index/utils/utils.cc index 3f5f09b2..3a38dd18 100644 --- a/tests/db/index/utils/utils.cc +++ b/tests/db/index/utils/utils.cc @@ -17,12 +17,12 @@ #include #include #include "ailego/logger/logger.h" -#include "db/collection.h" -#include "db/common/status.h" -#include "db/index/common/doc.h" -#include "db/index/common/index_params.h" -#include "db/index/common/schema.h" -#include "db/index/common/type.h" +#include "zvec/db/collection.h" +#include "zvec/db/doc.h" +#include "zvec/db/index_params.h" +#include "zvec/db/schema.h" +#include "zvec/db/status.h" +#include "zvec/db/type.h" using namespace zvec; using namespace zvec::test; diff --git a/tests/db/index/utils/utils.h b/tests/db/index/utils/utils.h index f858d56a..3eb1c7f7 100644 --- a/tests/db/index/utils/utils.h +++ b/tests/db/index/utils/utils.h @@ -26,15 +26,15 @@ #include #include #include -#include "db/collection.h" #include "db/common/constants.h" #include "db/common/typedef.h" -#include "db/index/common/doc.h" #include "db/index/common/meta.h" -#include "db/index/common/schema.h" -#include "db/index/common/type.h" #include "db/index/segment/segment.h" #include "db/index/storage/store_helper.h" +#include "zvec/db/collection.h" +#include "zvec/db/doc.h" +#include "zvec/db/schema.h" +#include "zvec/db/type.h" namespace zvec::test { diff --git a/tests/db/sqlengine/contain_test.cc b/tests/db/sqlengine/contain_test.cc index 2687d4d3..635fb663 100644 --- a/tests/db/sqlengine/contain_test.cc +++ b/tests/db/sqlengine/contain_test.cc @@ -22,9 +22,9 @@ #include "db/common/file_helper.h" #include "db/index/segment/segment.h" #include "db/sqlengine/sqlengine.h" -#include "index/common/index_params.h" -#include "index/common/schema.h" -#include "index/common/type.h" +#include "zvec/db/index_params.h" +#include "zvec/db/schema.h" +#include "zvec/db/type.h" #include "test_helper.h" namespace zvec::sqlengine { diff --git a/tests/db/sqlengine/forward_recall_test.cc b/tests/db/sqlengine/forward_recall_test.cc index 42fa5c41..6eea1464 100644 --- a/tests/db/sqlengine/forward_recall_test.cc +++ b/tests/db/sqlengine/forward_recall_test.cc @@ -16,7 +16,7 @@ #include #include #include "db/sqlengine/sqlengine.h" -#include "index/common/schema.h" +#include "zvec/db/schema.h" #include "recall_base.h" namespace zvec::sqlengine { diff --git a/tests/db/sqlengine/invert_recall_test.cc b/tests/db/sqlengine/invert_recall_test.cc index 458614a8..3095b2ee 100644 --- a/tests/db/sqlengine/invert_recall_test.cc +++ b/tests/db/sqlengine/invert_recall_test.cc @@ -16,7 +16,7 @@ #include #include #include "db/sqlengine/sqlengine.h" -#include "index/common/schema.h" +#include "zvec/db/schema.h" #include "recall_base.h" namespace zvec::sqlengine { diff --git a/tests/db/sqlengine/like_test.cc b/tests/db/sqlengine/like_test.cc index a861e4b2..d1f6bbb0 100644 --- a/tests/db/sqlengine/like_test.cc +++ b/tests/db/sqlengine/like_test.cc @@ -24,9 +24,9 @@ #include "db/index/common/version_manager.h" #include "db/index/segment/segment.h" #include "db/sqlengine/sqlengine.h" -#include "index/common/index_params.h" -#include "index/common/schema.h" -#include "index/common/type.h" +#include "zvec/db/index_params.h" +#include "zvec/db/schema.h" +#include "zvec/db/type.h" #include "test_helper.h" namespace zvec::sqlengine { diff --git a/tests/db/sqlengine/optimizer_test.cc b/tests/db/sqlengine/optimizer_test.cc index ddfd292b..4ac0ab43 100644 --- a/tests/db/sqlengine/optimizer_test.cc +++ b/tests/db/sqlengine/optimizer_test.cc @@ -15,7 +15,7 @@ #include #include "db/sqlengine/analyzer/query_info_helper.h" #include "db/sqlengine/sqlengine_impl.h" -#include "index/common/index_params.h" +#include "zvec/db/index_params.h" // #define private public #include #include "db/sqlengine/planner/optimizer.h" diff --git a/tests/db/sqlengine/query_info_test.cc b/tests/db/sqlengine/query_info_test.cc index 3dda23fe..56166f58 100644 --- a/tests/db/sqlengine/query_info_test.cc +++ b/tests/db/sqlengine/query_info_test.cc @@ -16,8 +16,8 @@ #include #include #include "db/sqlengine/sqlengine_impl.h" -#include "index/common/doc.h" -#include "index/common/schema.h" +#include "zvec/db/doc.h" +#include "zvec/db/schema.h" #include "profiler.h" diff --git a/tests/db/sqlengine/recall_base.h b/tests/db/sqlengine/recall_base.h index 81d15217..8c2a88ab 100644 --- a/tests/db/sqlengine/recall_base.h +++ b/tests/db/sqlengine/recall_base.h @@ -25,9 +25,9 @@ #include "db/common/file_helper.h" #include "db/index/common/version_manager.h" #include "db/index/segment/segment.h" -#include "index/common/index_params.h" -#include "index/common/schema.h" -#include "index/common/type.h" +#include "zvec/db/index_params.h" +#include "zvec/db/schema.h" +#include "zvec/db/type.h" namespace zvec { diff --git a/tests/db/sqlengine/simple_rewriter_test.cc b/tests/db/sqlengine/simple_rewriter_test.cc index 170775f2..c2a1d625 100644 --- a/tests/db/sqlengine/simple_rewriter_test.cc +++ b/tests/db/sqlengine/simple_rewriter_test.cc @@ -14,10 +14,10 @@ #include "sqlengine/analyzer/simple_rewriter.h" #include -#include "db/index/common/doc.h" -#include "db/index/common/schema.h" #include "db/sqlengine/analyzer/query_info.h" #include "db/sqlengine/sqlengine_impl.h" +#include "zvec/db/doc.h" +#include "zvec/db/schema.h" namespace zvec::sqlengine { diff --git a/tests/db/sqlengine/sqlengine_test.cc b/tests/db/sqlengine/sqlengine_test.cc index c6e0e595..8c4c900d 100644 --- a/tests/db/sqlengine/sqlengine_test.cc +++ b/tests/db/sqlengine/sqlengine_test.cc @@ -16,9 +16,9 @@ #include #include #include -#include "index/common/query_params.h" -#include "index/common/schema.h" -#include "index/common/type.h" +#include "zvec/db//schema.h" +#include "zvec/db/query_params.h" +#include "zvec/db/type.h" #include "mock_segment.h" namespace zvec::sqlengine { diff --git a/tests/db/sqlengine/test_helper.h b/tests/db/sqlengine/test_helper.h index 118af86b..6bdb613d 100644 --- a/tests/db/sqlengine/test_helper.h +++ b/tests/db/sqlengine/test_helper.h @@ -26,9 +26,9 @@ #include "db/index/common/version_manager.h" #include "db/index/segment/segment.h" #include "db/sqlengine/sqlengine.h" -#include "index/common/index_params.h" -#include "index/common/schema.h" -#include "index/common/type.h" +#include "zvec/db/index_params.h" +#include "zvec/db/schema.h" +#include "zvec/db/type.h" namespace zvec::sqlengine { diff --git a/tests/db/sqlengine/vector_recall_test.cc b/tests/db/sqlengine/vector_recall_test.cc index c9fe2ac1..597b63a6 100644 --- a/tests/db/sqlengine/vector_recall_test.cc +++ b/tests/db/sqlengine/vector_recall_test.cc @@ -16,7 +16,6 @@ #include #include #include "db/sqlengine/sqlengine.h" -#include "index/common/schema.h" #include "recall_base.h" namespace zvec::sqlengine { diff --git a/thirdparty/CRoaring/CMakeLists.txt b/thirdparty/CRoaring/CMakeLists.txt index 20ae0674..af6dfb31 100644 --- a/thirdparty/CRoaring/CMakeLists.txt +++ b/thirdparty/CRoaring/CMakeLists.txt @@ -1,6 +1,10 @@ set(ENABLE_ROARING_TESTS OFF CACHE BOOL "Disable testing in CRoaring" FORCE) +set(_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${EXTERNAL_LIB_DIR}) add_subdirectory(CRoaring-2.0.4) +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) +unset(_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY) if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") target_compile_options(roaring PRIVATE diff --git a/thirdparty/antlr/CMakeLists.txt b/thirdparty/antlr/CMakeLists.txt index c784b07b..fe620ef0 100644 --- a/thirdparty/antlr/CMakeLists.txt +++ b/thirdparty/antlr/CMakeLists.txt @@ -8,7 +8,7 @@ add_library(antlr4 UNKNOWN IMPORTED GLOBAL) set_target_properties( antlr4 PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/antlr4/runtime/Cpp/runtime/src/" - IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/antlr4/runtime/Cpp/runtime/output/lib/libantlr4-runtime.a" + IMPORTED_LOCATION "${EXTERNAL_LIB_DIR}/libantlr4-runtime.a" ) add_dependencies(antlr4 antlr4_static) diff --git a/thirdparty/antlr/antlr4.patch b/thirdparty/antlr/antlr4.patch index ad02a28d..1156a4f7 100644 --- a/thirdparty/antlr/antlr4.patch +++ b/thirdparty/antlr/antlr4.patch @@ -26,7 +26,7 @@ index 2c5e7376f..bcc0134dc 100644 add_library(antlr4_static STATIC ${libantlrcpp_SRC}) -set(LIB_OUTPUT_DIR "${CMAKE_HOME_DIRECTORY}/dist") # put generated libraries here. -+set(LIB_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/output/lib") ++set(LIB_OUTPUT_DIR "${EXTERNAL_LIB_DIR}") message(STATUS "Output libraries to ${LIB_OUTPUT_DIR}") # make sure 'make' works fine even if ${LIB_OUTPUT_DIR} is deleted. diff --git a/thirdparty/gflags/CMakeLists.txt b/thirdparty/gflags/CMakeLists.txt index ab4f64df..917fc495 100644 --- a/thirdparty/gflags/CMakeLists.txt +++ b/thirdparty/gflags/CMakeLists.txt @@ -9,7 +9,11 @@ else() endif() set(GFLAGS_IS_SUBPROJECT ON) +set(_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${EXTERNAL_LIB_DIR}) add_subdirectory(gflags-2.2.2) +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) +unset(_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY) get_target_property(gflags_OUTPUT_NAME gflags OUTPUT_NAME) get_target_property(gflags_BINARY_DIR gflags BINARY_DIR) diff --git a/thirdparty/glog/CMakeLists.txt b/thirdparty/glog/CMakeLists.txt index 8357d83e..04c1d085 100644 --- a/thirdparty/glog/CMakeLists.txt +++ b/thirdparty/glog/CMakeLists.txt @@ -9,7 +9,12 @@ set(GLOG_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/glog-0.5.0) set(GLOG_PATCH ${CMAKE_CURRENT_SOURCE_DIR}/glog.patch) apply_patch_once("glog_fix" "${GLOG_SRC_DIR}" "${GLOG_PATCH}") +set(_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${EXTERNAL_LIB_DIR}) add_subdirectory(glog-0.5.0) +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) +unset(_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY) + add_dependencies(glog gflags) if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") diff --git a/thirdparty/googletest/CMakeLists.txt b/thirdparty/googletest/CMakeLists.txt index 765d23d1..e72a0c13 100644 --- a/thirdparty/googletest/CMakeLists.txt +++ b/thirdparty/googletest/CMakeLists.txt @@ -2,7 +2,13 @@ add_compile_options(-Wno-deprecated-copy) if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") add_compile_options(-Wno-maybe-uninitialized) endif() + +set(_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${EXTERNAL_LIB_DIR}) add_subdirectory(googletest-1.10.0) +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) +unset(_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY) + get_target_property(GTEST_INCLUDE_DIRS gtest INTERFACE_INCLUDE_DIRECTORIES) set(GTEST_FOUND TRUE PARENT_SCOPE) diff --git a/thirdparty/lz4/CMakeLists.txt b/thirdparty/lz4/CMakeLists.txt index 34ecd2b3..ad5f96b1 100644 --- a/thirdparty/lz4/CMakeLists.txt +++ b/thirdparty/lz4/CMakeLists.txt @@ -9,8 +9,8 @@ ExternalProject_Add( PREFIX lz4 URL "${CMAKE_CURRENT_SOURCE_DIR}/lz4-1.9.4" CONFIGURE_COMMAND "" - BUILD_COMMAND env CFLAGS=-fPIC make -j - INSTALL_COMMAND make DESTDIR=${EXTERNAL_BINARY_DIR} install + BUILD_COMMAND env CFLAGS=-fPIC BUILD_SHARED=no make -j + INSTALL_COMMAND make DESTDIR=${EXTERNAL_BINARY_DIR} BUILD_SHARED=no install BUILD_IN_SOURCE ON LOG_DOWNLOAD ON LOG_CONFIGURE ON diff --git a/thirdparty/protobuf/CMakeLists.txt b/thirdparty/protobuf/CMakeLists.txt index ff720349..7e4c1b27 100644 --- a/thirdparty/protobuf/CMakeLists.txt +++ b/thirdparty/protobuf/CMakeLists.txt @@ -1,6 +1,11 @@ set(protobuf_BUILD_TESTS OFF CACHE BOOL "Disable testing in protobuf" FORCE) set(protobuf_WITH_ZLIB ON CACHE BOOL "Disable zlib support in protobuf" FORCE) + +set(_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${EXTERNAL_LIB_DIR}) add_subdirectory(protobuf-3.21.12/cmake protobuf-3.21.12) +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) +unset(_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY) if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") target_compile_options(libprotobuf PRIVATE diff --git a/thirdparty/rocksdb/CMakeLists.txt b/thirdparty/rocksdb/CMakeLists.txt index 6aaa6f92..8458d338 100644 --- a/thirdparty/rocksdb/CMakeLists.txt +++ b/thirdparty/rocksdb/CMakeLists.txt @@ -9,7 +9,11 @@ set(WITH_LZ4 ON CACHE BOOL "build with lz4" FORCE) set(USE_RTTI ON CACHE BOOL "build with RTTI" FORCE) set(FAIL_ON_WARNINGS OFF CACHE BOOL "build with no Werror" FORCE) +set(_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${EXTERNAL_LIB_DIR}) add_subdirectory(rocksdb-8.1.1) +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) +unset(_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY) get_target_property(rocksdb_SOURCE_DIR rocksdb SOURCE_DIR) set(ROCKSDB_INCLUDE_DIR ${rocksdb_SOURCE_DIR}/include) diff --git a/thirdparty/yaml-cpp/CMakeLists.txt b/thirdparty/yaml-cpp/CMakeLists.txt index 307e4f7e..d237d6fd 100644 --- a/thirdparty/yaml-cpp/CMakeLists.txt +++ b/thirdparty/yaml-cpp/CMakeLists.txt @@ -2,7 +2,11 @@ set(YAML_CPP_BUILD_TESTS OFF CACHE BOOL "Disable testing in yaml-cpp" FORCE) set(YAML_CPP_BUILD_TOOLS OFF CACHE BOOL "Disable parse tools in yaml-cpp" FORCE) set(YAML_CPP_BUILD_CONTRIB OFF CACHE BOOL "Disable contrib stuff in yaml-cpp" FORCE) +set(_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${EXTERNAL_LIB_DIR}) add_subdirectory(yaml-cpp-0.6.3) +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) +unset(_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY) if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") target_compile_options(yaml-cpp PRIVATE -Wno-shadow) diff --git a/tools/core/bench_original.cc b/tools/core/bench_original.cc index 24113270..d074d1c1 100644 --- a/tools/core/bench_original.cc +++ b/tools/core/bench_original.cc @@ -16,10 +16,10 @@ #include #include #include -#include "ailego/utility/string_helper.h" #include "framework/index_plugin.h" #include "interface/index_factory.h" #include "interface/index_param.h" +#include "zvec/ailego/utility/string_helper.h" #include "bench_result.h" #include "filter_result_cache.h" #include "flow.h" diff --git a/tools/core/helper.h b/tools/core/helper.h index 60b9dbe6..7c59f98a 100644 --- a/tools/core/helper.h +++ b/tools/core/helper.h @@ -23,8 +23,8 @@ #include #include #include -#include #include +#include #include "framework/index_error.h" #include "framework/index_factory.h" #include "framework/index_plugin.h" diff --git a/tools/core/recall_original.cc b/tools/core/recall_original.cc index b80d0274..9d334a40 100644 --- a/tools/core/recall_original.cc +++ b/tools/core/recall_original.cc @@ -21,8 +21,8 @@ #include #include #include -#include #include +#include #include "framework/index_plugin.h" #include "interface/index_factory.h" #include "interface/index_param.h" diff --git a/tools/core/txt_input_reader.h b/tools/core/txt_input_reader.h index eb0c44ea..82b147dc 100644 --- a/tools/core/txt_input_reader.h +++ b/tools/core/txt_input_reader.h @@ -19,7 +19,7 @@ #include #include #include -#include +#include namespace zvec { namespace core {