diff --git a/CHANGELOG.md b/CHANGELOG.md index 66e2fe4a793..19627784a74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,27 @@ +# NEXT MAJOR RELEASE + +### Enhancements +* (PR [#????](https://github.com/realm/realm-core/pull/????)) +* Storage of integers changed so that they take up less space in the file. This can cause commits and some queries to take a bit longer (PR [#7668](https://github.com/realm/realm-core/pull/7668)) +* We now allow synchronizing, getting and setting properties that are not defined in the object schema (PR [#7886](https://github.com/realm/realm-core/pull/7886)) + +### Fixed +* ([#????](https://github.com/realm/realm-core/issues/????), since v?.?.?) +* None. + +### Breaking changes +* None. + +### Compatibility +* Fileformat: Generates files with format v25. Reads and automatically upgrade from fileformat v10. If you want to upgrade from an earlier file format version you will have to use RealmCore v13.x.y or earlier. + +----------- + +### Internals +* None. + +---------------------------------------------- + # 14.12.0 Release notes ### Enhancements @@ -33,7 +57,6 @@ ### Fixed * Sync client may report duplicate compensating write errors ([#7708](https://github.com/realm/realm-core/issues/7708), since v14.8.0). -* String serialization of timestamps with a sufficiently large timestamp value could overflow an int causing undefined behavior, causing potentially bad values for the month/day/year values in stringified dates. ([PR #7934](https://github.com/realm/realm-core/pull/7934)). ### Breaking changes * None. @@ -174,7 +197,7 @@ ### Enhancements * It is no longer an error to set a base url for an App with a trailing slash - for example, `https://services.cloud.mongodb.com/` instead of `https://services.cloud.mongodb.com` - before this change that would result in a 404 error from the server ([PR #7791](https://github.com/realm/realm-core/pull/7791)). -* Performance has been improved for range queries on integers and timestamps. Requires that you use the "BETWEEN" operation in RQL or the Query::between() method when you build the query. (PR [#7785](https://github.com/realm/realm-core/pull/7785)) +* Performance has been improved for range queries on integers and timestamps. Requires that you use the "BETWEEN" operation in MQL or the Query::between() method when you build the query. (PR [#7785](https://github.com/realm/realm-core/pull/7785)) * Expose `Obj::add_int()` in the bindgen spec. ([PR #7797](https://github.com/realm/realm-core/pull/7797)). ### Fixed diff --git a/CMakeLists.txt b/CMakeLists.txt index a693f1c8542..181cb1f5d50 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -266,12 +266,13 @@ option(REALM_ENABLE_ALLOC_SET_ZERO "Zero all allocations." OFF) if(NOT EMSCRIPTEN) option(REALM_ENABLE_ENCRYPTION "Enable encryption." ON) endif() -option(REALM_ENABLE_MEMDEBUG "Add additional memory checks" OFF) -option(REALM_VALGRIND "Tell the test suite we are running with valgrind" OFF) -option(REALM_SYNC_MULTIPLEXING "Enables/disables sync session multiplexing by default" ON) +option(REALM_ENABLE_MEMDEBUG "Add additional memory checks." OFF) +option(REALM_VALGRIND "Tell the test suite we are running with valgrind." OFF) +option(REALM_SYNC_MULTIPLEXING "Enables/disables sync session multiplexing by default." ON) set(REALM_MAX_BPNODE_SIZE "1000" CACHE STRING "Max B+ tree node size.") option(REALM_ENABLE_GEOSPATIAL "Enable geospatial types and queries." ON) option(REALM_APP_SERVICES "Enable the default app services implementation." ON) +option(REALM_COMPRESS "Compress all the arrays by default in flex format." OFF) # Find dependencies set(THREADS_PREFER_PTHREAD_FLAG ON) diff --git a/Package.swift b/Package.swift index efcd53cf67f..7c1d1483691 100644 --- a/Package.swift +++ b/Package.swift @@ -52,6 +52,7 @@ let notSyncServerSources: [String] = [ "realm/array_blobs_small.cpp", "realm/array_decimal128.cpp", "realm/array_fixed_bytes.cpp", + "realm/array_aggregate_optimizations.cpp", "realm/array_integer.cpp", "realm/array_key.cpp", "realm/array_mixed.cpp", @@ -78,6 +79,9 @@ let notSyncServerSources: [String] = [ "realm/group.cpp", "realm/group_writer.cpp", "realm/history.cpp", + "realm/integer_compressor.cpp", + "realm/integer_flex_compressor.cpp", + "realm/integer_packed_compressor.cpp", "realm/impl", "realm/index_string.cpp", "realm/link_translator.cpp", diff --git a/bindgen/spec.yml b/bindgen/spec.yml index 91fac6394fa..c4db39ca7a2 100644 --- a/bindgen/spec.yml +++ b/bindgen/spec.yml @@ -440,6 +440,9 @@ records: schema_mode: type: SchemaMode default: SchemaMode::Automatic + flexible_schema: + type: bool + default: false disable_format_upgrade: type: bool default: false @@ -785,6 +788,7 @@ classes: remove_object: '(key: ObjKey)' get_link_target: '(column: ColKey) -> TableRef' clear: () + get_column_key: '(column: StringData) -> ColKey' get_primary_key_column: '() -> ColKey' Obj: @@ -807,17 +811,28 @@ classes: - '(column: ColKey, value: Mixed)' - sig: '(column: ColKey, value: Mixed, is_default: bool)' suffix: with_default - set_collection: '(column: ColKey, type: CollectionType) -> Obj' + - sig: '(column: StringData, value: Mixed)' + suffix: by_name + set_collection: + - '(column: ColKey, type: CollectionType) -> Obj' + - sig: '(prop_name: StringData, type: CollectionType)' + suffix: by_name add_int: '(column: ColKey, value: int64_t) -> Obj' get_linked_object: '(column: ColKey) const -> Nullable' to_string: () const -> std::string get_backlink_count: '() const -> count_t' + erase_additional_prop: '(prop_name: StringData) const -> Obj' + get_additional_properties: '() -> std::vector' get_backlink_view: '(src_table: TableRef, src_col_key: ColKey) -> TableView' create_and_set_linked_object: '(column: ColKey) -> Obj' - + has_schema_property: '(column: StringData) -> bool' + get_collection_ptr: '(prop_name: StringData) -> CollectionPointer' Transaction: sharedPtrWrapped: TransactionRef + CollectionPointer: + cppName: CollectionBasePtr + ObjectStore: staticMethods: get_schema_version: '(group: Group) -> SchemaVersion' @@ -1035,6 +1050,7 @@ classes: Collection: cppName: object_store::Collection + sharedPtrWrapped: SharedCollection abstract: true properties: get_type: PropertyType @@ -1063,6 +1079,7 @@ classes: base: Collection constructors: make: '(r: SharedRealm, parent: const Obj&, col: ColKey)' + make_other: '(r: SharedRealm, parent: const Obj&, prop_name: StringData)' methods: get: - sig: '(ndx: count_t) -> Obj' diff --git a/evergreen/config.yml b/evergreen/config.yml index e14afde6c9b..2f0bfd89afe 100644 --- a/evergreen/config.yml +++ b/evergreen/config.yml @@ -137,8 +137,16 @@ functions: set_cmake_var realm_vars REALM_LLVM_COVERAGE BOOL On fi + if [[ -n "${compress|}" ]]; then + set_cmake_var realm_vars REALM_COMPRESS PATH "${cmake_toolchain_file}" + fi + set_cmake_var realm_vars REALM_BUILD_COMMANDLINE_TOOLS BOOL "${build_command_line_tools|On}" set_cmake_var realm_vars REALM_ENABLE_ENCRYPTION BOOL "${enable_realm_encryption|On}" + if [[ -n "${compress|}" ]]; then + set_cmake_var realm_vars REALM_COMPRESS PATH "${cmake_toolchain_file}" + fi + if [[ -n "${fetch_missing_dependencies|}" ]]; then set_cmake_var realm_vars REALM_FETCH_MISSING_DEPENDENCIES BOOL On @@ -1980,6 +1988,22 @@ buildvariants: - name: compile_test_coverage - name: finalize_coverage_data +- name: macos-array-compression + display_name: "MacOS 14 arm64 (Compress Arrays)" + run_on: macos-14-arm64 + expansions: + cmake_bindir: "/opt/homebrew/bin" + cmake_toolchain_file: "./tools/cmake/xcode.toolchain.cmake" + cmake_build_tool_options: "-sdk macosx" + cmake_generator: Xcode + max_jobs: $(sysctl -n hw.logicalcpu) + xcode_developer_dir: /Applications/Xcode15.2.app/Contents/Developer + extra_flags: -DCMAKE_SYSTEM_NAME=Darwin -DCMAKE_OSX_ARCHITECTURES=arm64 + compress: On + cmake_build_type: Debug + tasks: + - name: compile_test + - name: ubuntu-trace-logging display_name: "Ubuntu (Trace Logging Enabled)" run_on: ubuntu2204-arm64-large diff --git a/src/realm.h b/src/realm.h index bfc8b304694..f885f05e7e1 100644 --- a/src/realm.h +++ b/src/realm.h @@ -925,6 +925,11 @@ RLM_API bool realm_config_get_cached(realm_config_t*) RLM_API_NOEXCEPT; */ RLM_API void realm_config_set_automatic_backlink_handling(realm_config_t*, bool) RLM_API_NOEXCEPT; +/** + * Allow realm objects in the realm to have additional properties that are not defined in the schema. + */ +RLM_API void realm_config_set_flexible_schema(realm_config_t*, bool) RLM_API_NOEXCEPT; + /** * Create a custom scheduler object from callback functions. * @@ -1645,6 +1650,13 @@ RLM_API realm_object_t* realm_object_from_thread_safe_reference(const realm_t*, */ RLM_API bool realm_get_value(const realm_object_t*, realm_property_key_t, realm_value_t* out_value); +/** + * Get the value for a property. + * + * @return True if no exception occurred. + */ +RLM_API bool realm_get_value_by_name(const realm_object_t*, const char* property_name, realm_value_t* out_value); + /** * Get the values for several properties. * @@ -1680,6 +1692,41 @@ RLM_API bool realm_get_values(const realm_object_t*, size_t num_values, const re */ RLM_API bool realm_set_value(realm_object_t*, realm_property_key_t, realm_value_t new_value, bool is_default); +/** + * Set the value for a property. Property need not be defined in schema if flexible + * schema is enabled in configuration + * + * @param property_name The name of the property. + * @param new_value The new value for the property. + * @return True if no exception occurred. + */ +RLM_API bool realm_set_value_by_name(realm_object_t*, const char* property_name, realm_value_t new_value); + +/** + * Examines if the object has a property with the given name. + * @param out_has_property will be true if the property exists. + * @return True if no exception occurred. + */ +RLM_API bool realm_has_property(realm_object_t*, const char* property_name, bool* out_has_property); + +/** + * Get a list of properties set on the object that are not defined in the schema. + * + * @param out_prop_names A pointer to an array of const char* of size @a max. If the pointer is NULL, + * no names will be copied, but @a out_n will be set to the required size. + * @param max size of @a out_prop_names + * @param out_n number of names actually returned. + */ +RLM_API void realm_get_additional_properties(realm_object_t*, const char** out_prop_names, size_t max, size_t* out_n); + +/** + * Erases a property from an object. You can't erase a property that is defined in the current schema. + * + * @param property_name The name of the property. + * @return True if the property was removed. + */ +RLM_API bool realm_erase_additional_property(realm_object_t*, const char* property_name); + /** * Assign a JSON formatted string to a Mixed property. Underlying structures will be created as needed * @@ -1701,6 +1748,8 @@ RLM_API realm_object_t* realm_set_embedded(realm_object_t*, realm_property_key_t */ RLM_API realm_list_t* realm_set_list(realm_object_t*, realm_property_key_t); RLM_API realm_dictionary_t* realm_set_dictionary(realm_object_t*, realm_property_key_t); +RLM_API realm_list_t* realm_set_list_by_name(realm_object_t*, const char* property_name); +RLM_API realm_dictionary_t* realm_set_dictionary_by_name(realm_object_t*, const char* property_name); /** Return the object linked by the given property * @@ -1753,6 +1802,15 @@ RLM_API bool realm_set_values(realm_object_t*, size_t num_values, const realm_pr */ RLM_API realm_list_t* realm_get_list(realm_object_t*, realm_property_key_t); +/** + * Get a list instance for the property of an object by name. + * + * Note: It is up to the caller to call `realm_release()` on the returned list. + * + * @return A non-null pointer if no exception occurred. + */ +RLM_API realm_list_t* realm_get_list_by_name(realm_object_t*, const char*); + /** * Create a `realm_list_t` from a pointer to a `realm::List`, copy-constructing * the internal representation. @@ -2258,6 +2316,15 @@ RLM_API realm_set_t* realm_set_from_thread_safe_reference(const realm_t*, realm_ */ RLM_API realm_dictionary_t* realm_get_dictionary(realm_object_t*, realm_property_key_t); +/** + * Get a dictionary instance for the property of an object by name. + * + * Note: It is up to the caller to call `realm_release()` on the returned dictionary. + * + * @return A non-null pointer if no exception occurred. + */ +RLM_API realm_dictionary_t* realm_get_dictionary_by_name(realm_object_t*, const char*); + /** * Create a `realm_dictionary_t` from a pointer to a `realm::object_store::Dictionary`, * copy-constructing the internal representation. diff --git a/src/realm/CMakeLists.txt b/src/realm/CMakeLists.txt index b5aebd5d3bf..18583f3549a 100644 --- a/src/realm/CMakeLists.txt +++ b/src/realm/CMakeLists.txt @@ -13,6 +13,7 @@ set(REALM_SOURCES array_blobs_big.cpp array_decimal128.cpp array_fixed_bytes.cpp + array_aggregate_optimizations.cpp array_integer.cpp array_key.cpp array_mixed.cpp @@ -36,6 +37,9 @@ set(REALM_SOURCES db.cpp group_writer.cpp history.cpp + integer_compressor.cpp + integer_flex_compressor.cpp + integer_packed_compressor.cpp impl/copy_replication.cpp impl/output_stream.cpp impl/simulated_failure.cpp @@ -163,6 +167,9 @@ set(REALM_INSTALL_HEADERS handover_defs.hpp history.hpp index_string.hpp + integer_compressor.hpp + integer_flex_compressor.hpp + integer_packed_compressor.hpp keys.hpp list.hpp mixed.hpp diff --git a/src/realm/alloc.cpp b/src/realm/alloc.cpp index 7f9817e2fbe..0c129fa82fd 100644 --- a/src/realm/alloc.cpp +++ b/src/realm/alloc.cpp @@ -113,14 +113,17 @@ Allocator& Allocator::get_default() noexcept // * adding a cross-over mapping. (if the array crosses a mapping boundary) // * using an already established cross-over mapping. (ditto) // this can proceed concurrently with other calls to translate() -char* Allocator::translate_less_critical(RefTranslation* ref_translation_ptr, ref_type ref) const noexcept +char* Allocator::translate_less_critical(RefTranslation* ref_translation_ptr, ref_type ref, + bool known_in_slab) const noexcept { size_t idx = get_section_index(ref); RefTranslation& txl = ref_translation_ptr[idx]; size_t offset = ref - get_section_base(idx); char* addr = txl.mapping_addr + offset; util::encryption_read_barrier(addr, NodeHeader::header_size, txl.encrypted_mapping); - auto size = NodeHeader::get_byte_size_from_header(addr); + // if we know the translation is inside the slab area, we don't need to check + // for anything beyond the header, and we don't need to check if decryption is needed + auto size = known_in_slab ? 8 : NodeHeader::get_byte_size_from_header(addr); bool crosses_mapping = offset + size > (1 << section_shift); // Move the limit on use of the existing primary mapping. // Take into account that another thread may attempt to change / have changed it concurrently, diff --git a/src/realm/alloc.hpp b/src/realm/alloc.hpp index 2a18e93ab1d..be6b22f6d60 100644 --- a/src/realm/alloc.hpp +++ b/src/realm/alloc.hpp @@ -113,6 +113,9 @@ class Allocator { /// Calls do_translate(). char* translate(ref_type ref) const noexcept; + /// Simpler version if we know the ref points inside the slab area + char* translate_in_slab(ref_type ref) const noexcept; + /// Returns true if, and only if the object at the specified 'ref' /// is in the immutable part of the memory managed by this /// allocator. The method by which some objects become part of the @@ -247,8 +250,8 @@ class Allocator { /// then entirely the responsibility of the caller that the memory /// is not modified by way of the returned memory pointer. virtual char* do_translate(ref_type ref) const noexcept = 0; - char* translate_critical(RefTranslation*, ref_type ref) const noexcept; - char* translate_less_critical(RefTranslation*, ref_type ref) const noexcept; + char* translate_critical(RefTranslation*, ref_type ref, bool known_in_slab = false) const noexcept; + char* translate_less_critical(RefTranslation*, ref_type ref, bool known_in_slab = false) const noexcept; virtual void get_or_add_xover_mapping(RefTranslation&, size_t, size_t, size_t) = 0; Allocator() noexcept = default; size_t get_section_index(size_t pos) const noexcept; @@ -544,7 +547,8 @@ inline bool Allocator::is_read_only(ref_type ref) const noexcept } // performance critical part of the translation process. Less critical code is in translate_less_critical. -inline char* Allocator::translate_critical(RefTranslation* ref_translation_ptr, ref_type ref) const noexcept +inline char* Allocator::translate_critical(RefTranslation* ref_translation_ptr, ref_type ref, + bool known_in_slab) const noexcept { size_t idx = get_section_index(ref); RefTranslation& txl = ref_translation_ptr[idx]; @@ -560,7 +564,7 @@ inline char* Allocator::translate_critical(RefTranslation* ref_translation_ptr, return addr; } // the lowest possible xover offset may grow concurrently, but that will be handled inside the call - return translate_less_critical(ref_translation_ptr, ref); + return translate_less_critical(ref_translation_ptr, ref, known_in_slab); } realm::util::terminate("Invalid ref translation entry", __FILE__, __LINE__, txl.cookie, 0x1234567890, ref, idx); } @@ -573,6 +577,17 @@ inline char* Allocator::translate(ref_type ref) const noexcept return do_translate(ref); } +inline char* Allocator::translate_in_slab(ref_type ref) const noexcept +{ + auto ref_translation_ptr = m_ref_translation_ptr.load(std::memory_order_acquire); + if (REALM_LIKELY(ref_translation_ptr)) { + return translate_critical(ref_translation_ptr, ref, true); + } + else { + return do_translate(ref); + } +} + } // namespace realm diff --git a/src/realm/alloc_slab.cpp b/src/realm/alloc_slab.cpp index c421e1ce3c8..40a14c6cdc6 100644 --- a/src/realm/alloc_slab.cpp +++ b/src/realm/alloc_slab.cpp @@ -229,7 +229,7 @@ MemRef SlabAlloc::do_alloc(size_t size) #endif char* addr = reinterpret_cast(entry); - REALM_ASSERT_EX(addr == translate(ref), addr, ref, get_file_path_for_assertions()); + REALM_ASSERT_EX(addr == translate_in_slab(ref), addr, ref, get_file_path_for_assertions()); #if REALM_ENABLE_ALLOC_SET_ZERO std::fill(addr, addr + size, 0); @@ -294,6 +294,7 @@ SlabAlloc::FreeBlock* SlabAlloc::pop_freelist_entry(FreeList list) void SlabAlloc::FreeBlock::unlink() { + REALM_ASSERT_DEBUG(next != nullptr && prev != nullptr); auto _next = next; auto _prev = prev; _next->prev = prev; @@ -381,6 +382,10 @@ SlabAlloc::FreeBlock* SlabAlloc::allocate_block(int size) if (remaining) push_freelist_entry(remaining); REALM_ASSERT_EX(size_from_block(block) >= size, size_from_block(block), size, get_file_path_for_assertions()); + const auto block_before = bb_before(block); + REALM_ASSERT_DEBUG(block_before && block_before->block_after_size >= size); + const auto after_block_size = size_from_block(block); + REALM_ASSERT_DEBUG(after_block_size >= size); return block; } diff --git a/src/realm/array.cpp b/src/realm/array.cpp index 25b0d9f5538..a2f61e4491c 100644 --- a/src/realm/array.cpp +++ b/src/realm/array.cpp @@ -42,7 +42,6 @@ #pragma warning(disable : 4127) // Condition is constant warning #endif - // Header format (8 bytes): // ------------------------ // @@ -190,39 +189,79 @@ using namespace realm::util; void QueryStateBase::dyncast() {} -size_t Array::bit_width(int64_t v) +uint8_t Array::bit_width(int64_t v) { // FIXME: Assuming there is a 64-bit CPU reverse bitscan // instruction and it is fast, then this function could be // implemented as a table lookup on the result of the scan - if ((uint64_t(v) >> 4) == 0) { static const int8_t bits[] = {0, 1, 2, 2, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4}; return bits[int8_t(v)]; } - - // First flip all bits if bit 63 is set (will now always be zero) if (v < 0) v = ~v; - // Then check if bits 15-31 used (32b), 7-31 used (16b), else (8b) return uint64_t(v) >> 31 ? 64 : uint64_t(v) >> 15 ? 32 : uint64_t(v) >> 7 ? 16 : 8; } +template +struct Array::VTableForWidth { + struct PopulatedVTable : VTable { + PopulatedVTable() + { + getter = &Array::get; + setter = &Array::set; + chunk_getter = &Array::get_chunk; + finder[cond_Equal] = &Array::find_vtable; + finder[cond_NotEqual] = &Array::find_vtable; + finder[cond_Greater] = &Array::find_vtable; + finder[cond_Less] = &Array::find_vtable; + } + }; + static const PopulatedVTable vtable; +}; + +template +const typename Array::VTableForWidth::PopulatedVTable Array::VTableForWidth::vtable; void Array::init_from_mem(MemRef mem) noexcept { - char* header = Node::init_from_mem(mem); - // Parse header + // Header is the type of header that has been allocated, in case we are decompressing, + // the header is of kind A, which is kind of deceiving the purpose of these checks. + // Since we will try to fetch some data from the just initialised header, and never reset + // important fields used for type A arrays, like width, lower, upper_bound which are used + // for expanding the array, but also query the data. + const auto header = mem.get_addr(); + const auto is_extended = m_integer_compressor.init(header); + m_is_inner_bptree_node = get_is_inner_bptree_node_from_header(header); m_has_refs = get_hasrefs_from_header(header); m_context_flag = get_context_flag_from_header(header); - update_width_cache_from_header(); + + if (is_extended) { + m_ref = mem.get_ref(); + m_data = get_data_from_header(header); + m_size = m_integer_compressor.size(); + m_width = m_integer_compressor.v_width(); + m_lbound = -m_integer_compressor.v_mask(); + m_ubound = m_integer_compressor.v_mask() - 1; + m_integer_compressor.set_vtable(*this); + m_getter = m_vtable->getter; + } + else { + // Old init phase. + Node::init_from_mem(mem); + update_width_cache_from_header(); + } +} + +MemRef Array::get_mem() const noexcept +{ + return MemRef(get_header_from_data(m_data), m_ref, m_alloc); } void Array::update_from_parent() noexcept { - REALM_ASSERT_DEBUG(is_attached()); ArrayParent* parent = get_parent(); REALM_ASSERT_DEBUG(parent); ref_type new_ref = get_ref_from_parent(); @@ -231,7 +270,7 @@ void Array::update_from_parent() noexcept void Array::set_type(Type type) { - REALM_ASSERT(is_attached()); + REALM_ASSERT_DEBUG(is_attached()); copy_on_write(); // Throws @@ -255,7 +294,6 @@ void Array::set_type(Type type) set_hasrefs_in_header(init_has_refs, header); } - void Array::destroy_children(size_t offset) noexcept { for (size_t i = offset; i != m_size; ++i) { @@ -276,20 +314,33 @@ void Array::destroy_children(size_t offset) noexcept } } +// size_t Array::get_byte_size() const noexcept +//{ +// const auto header = get_header(); +// auto num_bytes = get_byte_size_from_header(header); +// auto read_only = m_alloc.is_read_only(m_ref) == true; +// auto capacity = get_capacity_from_header(header); +// auto bytes_ok = num_bytes <= capacity; +// REALM_ASSERT(read_only || bytes_ok); +// REALM_ASSERT_7(m_alloc.is_read_only(m_ref), ==, true, ||, num_bytes, <=, get_capacity_from_header(header)); +// return num_bytes; +// } ref_type Array::do_write_shallow(_impl::ArrayWriterBase& out) const { - // Write flat array + // here we might want to compress the array and write down. const char* header = get_header_from_data(m_data); size_t byte_size = get_byte_size(); - uint32_t dummy_checksum = 0x41414141UL; // "AAAA" in ASCII - ref_type new_ref = out.write_array(header, byte_size, dummy_checksum); // Throws - REALM_ASSERT_3(new_ref % 8, ==, 0); // 8-byte alignment + const auto compressed = is_compressed(); + uint32_t dummy_checksum = compressed ? 0x42424242UL : 0x41414141UL; // + uint32_t dummy_checksum_bytes = compressed ? 2 : 4; // AAAA / BB (only 2 bytes for extended arrays) + ref_type new_ref = out.write_array(header, byte_size, dummy_checksum, dummy_checksum_bytes); // Throws + REALM_ASSERT_3(new_ref % 8, ==, 0); // 8-byte alignment return new_ref; } -ref_type Array::do_write_deep(_impl::ArrayWriterBase& out, bool only_if_modified) const +ref_type Array::do_write_deep(_impl::ArrayWriterBase& out, bool only_if_modified, bool compress) const { // Temp array for updated refs Array new_array(Allocator::get_default()); @@ -304,12 +355,11 @@ ref_type Array::do_write_deep(_impl::ArrayWriterBase& out, bool only_if_modified bool is_ref = (value != 0 && (value & 1) == 0); if (is_ref) { ref_type subref = to_ref(value); - ref_type new_subref = write(subref, m_alloc, out, only_if_modified); // Throws + ref_type new_subref = write(subref, m_alloc, out, only_if_modified, compress); // Throws value = from_ref(new_subref); } new_array.add(value); // Throws } - return new_array.do_write_shallow(out); // Throws } @@ -334,8 +384,8 @@ void Array::move(size_t begin, size_t end, size_t dest_begin) if (bits_per_elem < 8) { // FIXME: Should be optimized for (size_t i = begin; i != end; ++i) { - int_fast64_t v = (this->*m_getter)(i); - (this->*(m_vtable->setter))(dest_begin++, v); + int_fast64_t v = m_getter(*this, i); + m_vtable->setter(*this, dest_begin++, v); } return; } @@ -361,8 +411,8 @@ void Array::move(Array& dst, size_t ndx) size_t sz = m_size; for (size_t i = ndx; i < sz; i++) { - auto v = (this->*getter)(i); - (dst.*setter)(dest_begin++, v); + auto v = getter(*this, i); + setter(dst, dest_begin++, v); } truncate(ndx); @@ -371,17 +421,15 @@ void Array::move(Array& dst, size_t ndx) void Array::set(size_t ndx, int64_t value) { REALM_ASSERT_3(ndx, <, m_size); - if ((this->*(m_vtable->getter))(ndx) == value) + if (m_vtable->getter(*this, ndx) == value) return; // Check if we need to copy before modifying copy_on_write(); // Throws - // Grow the array if needed to store this value ensure_minimum_width(value); // Throws - // Set the value - (this->*(m_vtable->setter))(ndx, value); + m_vtable->setter(*this, ndx, value); } void Array::set_as_ref(size_t ndx, ref_type ref) @@ -429,6 +477,7 @@ void Array::insert(size_t ndx, int_fast64_t value) { REALM_ASSERT_DEBUG(ndx <= m_size); + decompress_array(*this); const auto old_width = m_width; const auto old_size = m_size; const Getter old_getter = m_getter; // Save old getter before potential width expansion @@ -448,8 +497,8 @@ void Array::insert(size_t ndx, int_fast64_t value) size_t i = old_size; while (i > ndx) { --i; - int64_t v = (this->*old_getter)(i); - (this->*(m_vtable->setter))(i + 1, v); + int64_t v = old_getter(*this, i); + m_vtable->setter(*this, i + 1, v); } } else if (ndx != old_size) { @@ -463,19 +512,30 @@ void Array::insert(size_t ndx, int_fast64_t value) } // Insert the new value - (this->*(m_vtable->setter))(ndx, value); + m_vtable->setter(*this, ndx, value); // Expand values above insertion if (do_expand) { size_t i = ndx; while (i != 0) { --i; - int64_t v = (this->*old_getter)(i); - (this->*(m_vtable->setter))(i, v); + int64_t v = old_getter(*this, i); + m_vtable->setter(*this, i, v); } } } +void Array::copy_on_write() +{ + if (is_read_only() && !decompress_array(*this)) + Node::copy_on_write(); +} + +void Array::copy_on_write(size_t min_size) +{ + if (is_read_only() && !decompress_array(*this)) + Node::copy_on_write(min_size); +} void Array::truncate(size_t new_size) { @@ -500,7 +560,6 @@ void Array::truncate(size_t new_size) } } - void Array::truncate_and_destroy_children(size_t new_size) { REALM_ASSERT(is_attached()); @@ -529,10 +588,8 @@ void Array::truncate_and_destroy_children(size_t new_size) } } - void Array::do_ensure_minimum_width(int_fast64_t value) { - // Make room for the new value const size_t width = bit_width(value); @@ -545,353 +602,32 @@ void Array::do_ensure_minimum_width(int_fast64_t value) size_t i = m_size; while (i != 0) { --i; - int64_t v = (this->*old_getter)(i); - (this->*(m_vtable->setter))(i, v); + int64_t v = old_getter(*this, i); + m_vtable->setter(*this, i, v); } } -int64_t Array::sum(size_t start, size_t end) const +bool Array::compress_array(Array& arr) const { - REALM_TEMPEX(return sum, m_width, (start, end)); + if (m_integer_compressor.get_encoding() == NodeHeader::Encoding::WTypBits) { + return m_integer_compressor.compress(*this, arr); + } + return false; } -template -int64_t Array::sum(size_t start, size_t end) const +bool Array::decompress_array(Array& arr) const { - if (end == size_t(-1)) - end = m_size; - REALM_ASSERT_EX(end <= m_size && start <= end, start, end, m_size); - - if (w == 0 || start == end) - return 0; - - int64_t s = 0; - - // Sum manually until 128 bit aligned - for (; (start < end) && (((size_t(m_data) & 0xf) * 8 + start * w) % 128 != 0); start++) { - s += get(start); - } - - if (w == 1 || w == 2 || w == 4) { - // Sum of bitwidths less than a byte (which are always positive) - // uses a divide and conquer algorithm that is a variation of popolation count: - // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel - - // static values needed for fast sums - const uint64_t m2 = 0x3333333333333333ULL; - const uint64_t m4 = 0x0f0f0f0f0f0f0f0fULL; - const uint64_t h01 = 0x0101010101010101ULL; - - int64_t* data = reinterpret_cast(m_data + start * w / 8); - size_t chunks = (end - start) * w / 8 / sizeof(int64_t); - - for (size_t t = 0; t < chunks; t++) { - if (w == 1) { -#if 0 -#if defined(USE_SSE42) && defined(_MSC_VER) && defined(REALM_PTR_64) - s += __popcnt64(data[t]); -#elif !defined(_MSC_VER) && defined(USE_SSE42) && defined(REALM_PTR_64) - s += __builtin_popcountll(data[t]); -#else - uint64_t a = data[t]; - const uint64_t m1 = 0x5555555555555555ULL; - a -= (a >> 1) & m1; - a = (a & m2) + ((a >> 2) & m2); - a = (a + (a >> 4)) & m4; - a = (a * h01) >> 56; - s += a; -#endif -#endif - s += fast_popcount64(data[t]); - } - else if (w == 2) { - uint64_t a = data[t]; - a = (a & m2) + ((a >> 2) & m2); - a = (a + (a >> 4)) & m4; - a = (a * h01) >> 56; - - s += a; - } - else if (w == 4) { - uint64_t a = data[t]; - a = (a & m4) + ((a >> 4) & m4); - a = (a * h01) >> 56; - s += a; - } - } - start += sizeof(int64_t) * 8 / no0(w) * chunks; - } - -#ifdef REALM_COMPILER_SSE - if (sseavx<42>()) { - // 2000 items summed 500000 times, 8/16/32 bits, miliseconds: - // Naive, templated get<>: 391 371 374 - // SSE: 97 148 282 - - if ((w == 8 || w == 16 || w == 32) && end - start > sizeof(__m128i) * 8 / no0(w)) { - __m128i* data = reinterpret_cast<__m128i*>(m_data + start * w / 8); - __m128i sum_result = {0}; - __m128i sum2; - - size_t chunks = (end - start) * w / 8 / sizeof(__m128i); - - for (size_t t = 0; t < chunks; t++) { - if (w == 8) { - /* - // 469 ms AND disadvantage of handling max 64k elements before overflow - __m128i vl = _mm_cvtepi8_epi16(data[t]); - __m128i vh = data[t]; - vh.m128i_i64[0] = vh.m128i_i64[1]; - vh = _mm_cvtepi8_epi16(vh); - sum_result = _mm_add_epi16(sum_result, vl); - sum_result = _mm_add_epi16(sum_result, vh); - */ - - /* - // 424 ms - __m128i vl = _mm_unpacklo_epi8(data[t], _mm_set1_epi8(0)); - __m128i vh = _mm_unpackhi_epi8(data[t], _mm_set1_epi8(0)); - sum_result = _mm_add_epi32(sum_result, _mm_madd_epi16(vl, _mm_set1_epi16(1))); - sum_result = _mm_add_epi32(sum_result, _mm_madd_epi16(vh, _mm_set1_epi16(1))); - */ - - __m128i vl = _mm_cvtepi8_epi16(data[t]); // sign extend lower words 8->16 - __m128i vh = data[t]; - vh = _mm_srli_si128(vh, 8); // v >>= 64 - vh = _mm_cvtepi8_epi16(vh); // sign extend lower words 8->16 - __m128i sum1 = _mm_add_epi16(vl, vh); - __m128i sumH = _mm_cvtepi16_epi32(sum1); - __m128i sumL = _mm_srli_si128(sum1, 8); // v >>= 64 - sumL = _mm_cvtepi16_epi32(sumL); - sum_result = _mm_add_epi32(sum_result, sumL); - sum_result = _mm_add_epi32(sum_result, sumH); - } - else if (w == 16) { - // todo, can overflow for array size > 2^32 - __m128i vl = _mm_cvtepi16_epi32(data[t]); // sign extend lower words 16->32 - __m128i vh = data[t]; - vh = _mm_srli_si128(vh, 8); // v >>= 64 - vh = _mm_cvtepi16_epi32(vh); // sign extend lower words 16->32 - sum_result = _mm_add_epi32(sum_result, vl); - sum_result = _mm_add_epi32(sum_result, vh); - } - else if (w == 32) { - __m128i v = data[t]; - __m128i v0 = _mm_cvtepi32_epi64(v); // sign extend lower dwords 32->64 - v = _mm_srli_si128(v, 8); // v >>= 64 - __m128i v1 = _mm_cvtepi32_epi64(v); // sign extend lower dwords 32->64 - sum_result = _mm_add_epi64(sum_result, v0); - sum_result = _mm_add_epi64(sum_result, v1); - - /* - __m128i m = _mm_set1_epi32(0xc000); // test if overflow could happen (still need - underflow test). - __m128i mm = _mm_and_si128(data[t], m); - zz = _mm_or_si128(mm, zz); - sum_result = _mm_add_epi32(sum_result, data[t]); - */ - } - } - start += sizeof(__m128i) * 8 / no0(w) * chunks; - - // prevent taking address of 'state' to make the compiler keep it in SSE register in above loop - // (vc2010/gcc4.6) - sum2 = sum_result; - - // Avoid aliasing bug where sum2 might not yet be initialized when accessed by get_universal - char sum3[sizeof sum2]; - memcpy(&sum3, &sum2, sizeof sum2); - - // Sum elements of sum - for (size_t t = 0; t < sizeof(__m128i) * 8 / ((w == 8 || w == 16) ? 32 : 64); ++t) { - int64_t v = get_universal < (w == 8 || w == 16) ? 32 : 64 > (reinterpret_cast(&sum3), t); - s += v; - } - } - } -#endif - - // Sum remaining elements - for (; start < end; ++start) - s += get(start); - - return s; + return arr.is_compressed() ? m_integer_compressor.decompress(arr) : false; } -size_t Array::count(int64_t value) const noexcept +bool Array::try_compress(Array& arr) const { - const uint64_t* next = reinterpret_cast(m_data); - size_t value_count = 0; - const size_t end = m_size; - size_t i = 0; - - // static values needed for fast population count - const uint64_t m1 = 0x5555555555555555ULL; - const uint64_t m2 = 0x3333333333333333ULL; - const uint64_t m4 = 0x0f0f0f0f0f0f0f0fULL; - const uint64_t h01 = 0x0101010101010101ULL; - - if (m_width == 0) { - if (value == 0) - return m_size; - return 0; - } - if (m_width == 1) { - if (uint64_t(value) > 1) - return 0; - - const size_t chunkvals = 64; - for (; i + chunkvals <= end; i += chunkvals) { - uint64_t a = next[i / chunkvals]; - if (value == 0) - a = ~a; // reverse - - a -= (a >> 1) & m1; - a = (a & m2) + ((a >> 2) & m2); - a = (a + (a >> 4)) & m4; - a = (a * h01) >> 56; - - // Could use intrinsic instead: - // a = __builtin_popcountll(a); // gcc intrinsic - - value_count += to_size_t(a); - } - } - else if (m_width == 2) { - if (uint64_t(value) > 3) - return 0; - - const uint64_t v = ~0ULL / 0x3 * value; - - // Masks to avoid spillover between segments in cascades - const uint64_t c1 = ~0ULL / 0x3 * 0x1; - - const size_t chunkvals = 32; - for (; i + chunkvals <= end; i += chunkvals) { - uint64_t a = next[i / chunkvals]; - a ^= v; // zero matching bit segments - a |= (a >> 1) & c1; // cascade ones in non-zeroed segments - a &= m1; // isolate single bit in each segment - a ^= m1; // reverse isolated bits - // if (!a) continue; - - // Population count - a = (a & m2) + ((a >> 2) & m2); - a = (a + (a >> 4)) & m4; - a = (a * h01) >> 56; - - value_count += to_size_t(a); - } - } - else if (m_width == 4) { - if (uint64_t(value) > 15) - return 0; - - const uint64_t v = ~0ULL / 0xF * value; - const uint64_t m = ~0ULL / 0xF * 0x1; - - // Masks to avoid spillover between segments in cascades - const uint64_t c1 = ~0ULL / 0xF * 0x7; - const uint64_t c2 = ~0ULL / 0xF * 0x3; - - const size_t chunkvals = 16; - for (; i + chunkvals <= end; i += chunkvals) { - uint64_t a = next[i / chunkvals]; - a ^= v; // zero matching bit segments - a |= (a >> 1) & c1; // cascade ones in non-zeroed segments - a |= (a >> 2) & c2; - a &= m; // isolate single bit in each segment - a ^= m; // reverse isolated bits - - // Population count - a = (a + (a >> 4)) & m4; - a = (a * h01) >> 56; - - value_count += to_size_t(a); - } - } - else if (m_width == 8) { - if (value > 0x7FLL || value < -0x80LL) - return 0; // by casting? - - const uint64_t v = ~0ULL / 0xFF * value; - const uint64_t m = ~0ULL / 0xFF * 0x1; - - // Masks to avoid spillover between segments in cascades - const uint64_t c1 = ~0ULL / 0xFF * 0x7F; - const uint64_t c2 = ~0ULL / 0xFF * 0x3F; - const uint64_t c3 = ~0ULL / 0xFF * 0x0F; - - const size_t chunkvals = 8; - for (; i + chunkvals <= end; i += chunkvals) { - uint64_t a = next[i / chunkvals]; - a ^= v; // zero matching bit segments - a |= (a >> 1) & c1; // cascade ones in non-zeroed segments - a |= (a >> 2) & c2; - a |= (a >> 4) & c3; - a &= m; // isolate single bit in each segment - a ^= m; // reverse isolated bits - - // Population count - a = (a * h01) >> 56; - - value_count += to_size_t(a); - } - } - else if (m_width == 16) { - if (value > 0x7FFFLL || value < -0x8000LL) - return 0; // by casting? - - const uint64_t v = ~0ULL / 0xFFFF * value; - const uint64_t m = ~0ULL / 0xFFFF * 0x1; - - // Masks to avoid spillover between segments in cascades - const uint64_t c1 = ~0ULL / 0xFFFF * 0x7FFF; - const uint64_t c2 = ~0ULL / 0xFFFF * 0x3FFF; - const uint64_t c3 = ~0ULL / 0xFFFF * 0x0FFF; - const uint64_t c4 = ~0ULL / 0xFFFF * 0x00FF; - - const size_t chunkvals = 4; - for (; i + chunkvals <= end; i += chunkvals) { - uint64_t a = next[i / chunkvals]; - a ^= v; // zero matching bit segments - a |= (a >> 1) & c1; // cascade ones in non-zeroed segments - a |= (a >> 2) & c2; - a |= (a >> 4) & c3; - a |= (a >> 8) & c4; - a &= m; // isolate single bit in each segment - a ^= m; // reverse isolated bits - - // Population count - a = (a * h01) >> 56; - - value_count += to_size_t(a); - } - } - else if (m_width == 32) { - int32_t v = int32_t(value); - const int32_t* d = reinterpret_cast(m_data); - for (; i < end; ++i) { - if (d[i] == v) - ++value_count; - } - return value_count; - } - else if (m_width == 64) { - const int64_t* d = reinterpret_cast(m_data); - for (; i < end; ++i) { - if (d[i] == value) - ++value_count; - } - return value_count; - } - - // Check remaining elements - for (; i < end; ++i) - if (value == get(i)) - ++value_count; + return compress_array(arr); +} - return value_count; +bool Array::try_decompress() +{ + return decompress_array(*this); } size_t Array::calc_aligned_byte_size(size_t size, int width) @@ -989,92 +725,48 @@ MemRef Array::clone(MemRef mem, Allocator& alloc, Allocator& target_alloc) MemRef Array::create(Type type, bool context_flag, WidthType width_type, size_t size, int_fast64_t value, Allocator& alloc) { - REALM_ASSERT_7(value, ==, 0, ||, width_type, ==, wtype_Bits); - REALM_ASSERT_7(size, ==, 0, ||, width_type, !=, wtype_Ignore); - - bool is_inner_bptree_node = false, has_refs = false; - switch (type) { - case type_Normal: - break; - case type_InnerBptreeNode: - is_inner_bptree_node = true; - has_refs = true; - break; - case type_HasRefs: - has_refs = true; - break; - } - - int width = 0; - size_t byte_size_0 = header_size; - if (value != 0) { - width = int(bit_width(value)); - byte_size_0 = calc_aligned_byte_size(size, width); // Throws - } - // Adding zero to Array::initial_capacity to avoid taking the - // address of that member - size_t byte_size = std::max(byte_size_0, initial_capacity + 0); - MemRef mem = alloc.alloc(byte_size); // Throws - char* header = mem.get_addr(); - - init_header(header, is_inner_bptree_node, has_refs, context_flag, width_type, width, size, byte_size); - + REALM_ASSERT_DEBUG(value == 0 || width_type == wtype_Bits); + REALM_ASSERT_DEBUG(size == 0 || width_type != wtype_Ignore); + uint8_t width = 0; + if (value != 0) + width = bit_width(value); + auto mem = Node::create_node(size, alloc, context_flag, type, width_type, width); if (value != 0) { + const auto header = mem.get_addr(); char* data = get_data_from_header(header); size_t begin = 0, end = size; REALM_TEMPEX(fill_direct, width, (data, begin, end, value)); } - return mem; } // This is the one installed into the m_vtable->finder slots. -template -bool Array::find_vtable(int64_t value, size_t start, size_t end, size_t baseindex, QueryStateBase* state) const +template +bool Array::find_vtable(const Array& arr, int64_t value, size_t start, size_t end, size_t baseindex, + QueryStateBase* state) { - return ArrayWithFind(*this).find_optimized(value, start, end, baseindex, state); + REALM_TEMPEX2(return ArrayWithFind(arr).find_optimized, cond, arr.m_width, (value, start, end, baseindex, state)); } - -template -struct Array::VTableForWidth { - struct PopulatedVTable : Array::VTable { - PopulatedVTable() - { - getter = &Array::get; - setter = &Array::set; - chunk_getter = &Array::get_chunk; - finder[cond_Equal] = &Array::find_vtable; - finder[cond_NotEqual] = &Array::find_vtable; - finder[cond_Greater] = &Array::find_vtable; - finder[cond_Less] = &Array::find_vtable; - } - }; - static const PopulatedVTable vtable; -}; - -template -const typename Array::VTableForWidth::PopulatedVTable Array::VTableForWidth::vtable; - void Array::update_width_cache_from_header() noexcept { - auto width = get_width_from_header(get_header()); - m_lbound = lbound_for_width(width); - m_ubound = ubound_for_width(width); - - m_width = width; - - REALM_TEMPEX(m_vtable = &VTableForWidth, width, ::vtable); + m_width = get_width_from_header(get_header()); + m_lbound = lbound_for_width(m_width); + m_ubound = ubound_for_width(m_width); + REALM_ASSERT_DEBUG(m_lbound <= m_ubound); + REALM_ASSERT_DEBUG(m_width >= m_lbound); + REALM_ASSERT_DEBUG(m_width <= m_ubound); + REALM_TEMPEX(m_vtable = &VTableForWidth, m_width, ::vtable); m_getter = m_vtable->getter; } // This method reads 8 concecutive values into res[8], starting from index 'ndx'. It's allowed for the 8 values to // exceed array length; in this case, remainder of res[8] will be be set to 0. template -void Array::get_chunk(size_t ndx, int64_t res[8]) const noexcept +void Array::get_chunk(const Array& arr, size_t ndx, int64_t res[8]) noexcept { - REALM_ASSERT_3(ndx, <, m_size); - + auto sz = arr.size(); + REALM_ASSERT_3(ndx, <, sz); size_t i = 0; // if constexpr to avoid producing spurious warnings resulting from @@ -1086,7 +778,7 @@ void Array::get_chunk(size_t ndx, int64_t res[8]) const noexcept // Round m_size down to byte granularity as the trailing bits in the last // byte are uninitialized - size_t bytes_available = m_size / elements_per_byte; + size_t bytes_available = sz / elements_per_byte; // Round start and end to be byte-aligned. Start is rounded down and // end is rounded up as we may read up to 7 unused bits at each end. @@ -1098,7 +790,7 @@ void Array::get_chunk(size_t ndx, int64_t res[8]) const noexcept uint64_t c = 0; for (size_t i = end; i > start; --i) { c <<= 8; - c += *reinterpret_cast(m_data + i - 1); + c += *reinterpret_cast(arr.m_data + i - 1); } // Trim off leading bits which aren't part of the requested range c >>= (ndx - start * elements_per_byte) * w; @@ -1118,31 +810,31 @@ void Array::get_chunk(size_t ndx, int64_t res[8]) const noexcept } } - for (; i + ndx < m_size && i < 8; i++) - res[i] = get(ndx + i); + for (; i + ndx < sz && i < 8; i++) + res[i] = get(arr, ndx + i); for (; i < 8; i++) res[i] = 0; #ifdef REALM_DEBUG - for (int j = 0; j + ndx < m_size && j < 8; j++) { - int64_t expected = get(ndx + j); + for (int j = 0; j + ndx < sz && j < 8; j++) { + int64_t expected = Array::get_universal(arr.m_data, ndx + j); REALM_ASSERT(res[j] == expected); } #endif } template <> -void Array::get_chunk<0>(size_t ndx, int64_t res[8]) const noexcept +void Array::get_chunk<0>(const Array& arr, size_t ndx, int64_t res[8]) noexcept { - REALM_ASSERT_3(ndx, <, m_size); + REALM_ASSERT_3(ndx, <, arr.m_size); memset(res, 0, sizeof(int64_t) * 8); } template -void Array::set(size_t ndx, int64_t value) +void Array::set(Array& arr, size_t ndx, int64_t value) { - set_direct(m_data, ndx, value); + realm::set_direct(arr.m_data, ndx, value); } void Array::_mem_usage(size_t& mem) const noexcept @@ -1247,10 +939,15 @@ void Array::report_memory_usage_2(MemUsageHandler& handler) const void Array::verify() const { #ifdef REALM_DEBUG - REALM_ASSERT(is_attached()); - REALM_ASSERT(m_width == 0 || m_width == 1 || m_width == 2 || m_width == 4 || m_width == 8 || m_width == 16 || - m_width == 32 || m_width == 64); + REALM_ASSERT(is_attached()); + if (!wtype_is_extended(get_header())) { + REALM_ASSERT(m_width == 0 || m_width == 1 || m_width == 2 || m_width == 4 || m_width == 8 || m_width == 16 || + m_width == 32 || m_width == 64); + } + else { + REALM_ASSERT(m_width <= 64); + } if (!get_parent()) return; @@ -1263,35 +960,60 @@ void Array::verify() const size_t Array::lower_bound_int(int64_t value) const noexcept { + if (is_compressed()) + return lower_bound_int_compressed(value); REALM_TEMPEX(return lower_bound, m_width, (m_data, m_size, value)); } size_t Array::upper_bound_int(int64_t value) const noexcept { + if (is_compressed()) + return upper_bound_int_compressed(value); REALM_TEMPEX(return upper_bound, m_width, (m_data, m_size, value)); } - -size_t Array::find_first(int64_t value, size_t start, size_t end) const +size_t Array::lower_bound_int_compressed(int64_t value) const noexcept { - return find_first(value, start, end); + static impl::CompressedDataFetcher encoder; + encoder.ptr = &m_integer_compressor; + return lower_bound(m_data, m_size, value, encoder); } +size_t Array::upper_bound_int_compressed(int64_t value) const noexcept +{ + static impl::CompressedDataFetcher encoder; + encoder.ptr = &m_integer_compressor; + return upper_bound(m_data, m_size, value, encoder); +} int_fast64_t Array::get(const char* header, size_t ndx) noexcept { - const char* data = get_data_from_header(header); - uint_least8_t width = get_width_from_header(header); - return get_direct(data, width, ndx); + // this is very important. Most of the times we end up here + // because we are traversing the cluster, the keys/refs in the cluster + // are not compressed (because there is almost no gain), so the intent + // is avoiding to pollute traversing the cluster as little as possible. + // We need to check the header wtype and only initialise the + // integer compressor, if needed. Otherwise we should just call + // get_direct. On average there should be one more access to the header + // while traversing the cluster tree. + if (REALM_LIKELY(!NodeHeader::wtype_is_extended(header))) { + const char* data = get_data_from_header(header); + uint_least8_t width = get_width_from_header(header); + return get_direct(data, width, ndx); + } + // Ideally, we would not want to construct a compressor every time we end up here. + // However the compressor initalization should be fast enough. Creating an array, + // which owns a compressor internally, is the better approach if we intend to access + // the same data over and over again. The compressor basically caches the most important + // information about the layuot of the data itself. + IntegerCompressor s_compressor; + s_compressor.init(header); + return s_compressor.get(ndx); } - std::pair Array::get_two(const char* header, size_t ndx) noexcept { - const char* data = get_data_from_header(header); - uint_least8_t width = get_width_from_header(header); - std::pair p = ::get_two(data, width, ndx); - return std::make_pair(p.first, p.second); + return std::make_pair(get(header, ndx), get(header, ndx + 1)); } bool QueryStateCount::match(size_t, Mixed) noexcept @@ -1337,7 +1059,6 @@ bool QueryStateFindAll>::match(size_t index) noexcept ++m_match_count; int64_t key_value = (m_key_values ? m_key_values->get(index) : index) + m_key_offset; m_keys.push_back(ObjKey(key_value)); - return (m_limit > m_match_count); } @@ -1358,3 +1079,12 @@ bool QueryStateFindAll::match(size_t index) noexcept return (m_limit > m_match_count); } + +ref_type ArrayPayload::typed_write(ref_type ref, _impl::ArrayWriterBase& out, Allocator& alloc) +{ + Array arr(alloc); + arr.init_from_ref(ref); + // By default we are not compressing + constexpr bool compress = false; + return arr.write(out, true, out.only_modified, compress); +} diff --git a/src/realm/array.hpp b/src/realm/array.hpp index af6fee1efa6..858915fe27f 100644 --- a/src/realm/array.hpp +++ b/src/realm/array.hpp @@ -21,8 +21,10 @@ #include #include +#include #include #include +#include namespace realm { @@ -90,12 +92,8 @@ class QueryStateFindFirst : public QueryStateBase { class Array : public Node, public ArrayParent { public: /// Create an array accessor in the unattached state. - explicit Array(Allocator& allocator) noexcept - : Node(allocator) - { - } - - ~Array() noexcept override {} + explicit Array(Allocator& allocator) noexcept; + virtual ~Array() noexcept = default; /// Create a new integer array of the specified type and size, and filled /// with the specified value, and attach this accessor to it. This does not @@ -126,6 +124,8 @@ class Array : public Node, public ArrayParent { init_from_ref(ref); } + MemRef get_mem() const noexcept; + /// Called in the context of Group::commit() to ensure that attached /// accessors stay valid across a commit. Please note that this works only /// for non-transactional commits. Accessors obtained during a transaction @@ -174,21 +174,23 @@ class Array : public Node, public ArrayParent { void set_as_ref(size_t ndx, ref_type ref); template - void set(size_t ndx, int64_t value); + static void set(Array&, size_t ndx, int64_t value); int64_t get(size_t ndx) const noexcept; + std::vector get_all(size_t b, size_t e) const; + template - int64_t get(size_t ndx) const noexcept; + static int64_t get(const Array& arr, size_t ndx) noexcept; void get_chunk(size_t ndx, int64_t res[8]) const noexcept; template - void get_chunk(size_t ndx, int64_t res[8]) const noexcept; + static void get_chunk(const Array&, size_t ndx, int64_t res[8]) noexcept; ref_type get_as_ref(size_t ndx) const noexcept; - RefOrTagged get_as_ref_or_tagged(size_t ndx) const noexcept; + void set(size_t ndx, RefOrTagged); void add(RefOrTagged); void ensure_minimum_width(RefOrTagged); @@ -198,12 +200,21 @@ class Array : public Node, public ArrayParent { void alloc(size_t init_size, size_t new_width) { - REALM_ASSERT_3(m_width, ==, get_width_from_header(get_header())); - REALM_ASSERT_3(m_size, ==, get_size_from_header(get_header())); + // Node::alloc is the one that triggers copy on write. If we call alloc for a B + // array we have a bug in our machinery, the array should have been decompressed + // way before calling alloc. + const auto header = get_header(); + REALM_ASSERT_3(m_width, ==, get_width_from_header(header)); + REALM_ASSERT_3(m_size, ==, get_size_from_header(header)); Node::alloc(init_size, new_width); update_width_cache_from_header(); } + bool is_empty() const noexcept + { + return size() == 0; + } + /// Remove the element at the specified index, and move elements at higher /// indexes to the next lower index. /// @@ -322,6 +333,8 @@ class Array : public Node, public ArrayParent { /// by doing a linear search for short sequences. size_t lower_bound_int(int64_t value) const noexcept; size_t upper_bound_int(int64_t value) const noexcept; + size_t lower_bound_int_compressed(int64_t value) const noexcept; + size_t upper_bound_int_compressed(int64_t value) const noexcept; //@} int64_t get_sum(size_t start = 0, size_t end = size_t(-1)) const @@ -351,6 +364,18 @@ class Array : public Node, public ArrayParent { /// (idempotency). void destroy_deep() noexcept; + /// check if the array is encoded (in B format) + inline bool is_compressed() const; + + inline const IntegerCompressor& integer_compressor() const; + + /// used only for testing, encode the array passed as argument + bool try_compress(Array&) const; + + /// used only for testing, decode the array, on which this method is invoked. If the array is not encoded, this is + /// a NOP + bool try_decompress(); + /// Shorthand for `destroy_deep(MemRef(ref, alloc), alloc)`. static void destroy_deep(ref_type ref, Allocator& alloc) noexcept; @@ -379,13 +404,19 @@ class Array : public Node, public ArrayParent { /// /// \param only_if_modified Set to `false` to always write, or to `true` to /// only write the array if it has been modified. - ref_type write(_impl::ArrayWriterBase& out, bool deep, bool only_if_modified) const; + ref_type write(_impl::ArrayWriterBase& out, bool deep, bool only_if_modified, bool compress_in_flight) const; /// Same as non-static write() with `deep` set to true. This is for the /// cases where you do not already have an array accessor available. - static ref_type write(ref_type, Allocator&, _impl::ArrayWriterBase&, bool only_if_modified); + /// Compression may be attempted if `compress_in_flight` is true. + /// This should be avoided if you rely on the size of the array beeing unchanged. + static ref_type write(ref_type, Allocator&, _impl::ArrayWriterBase&, bool only_if_modified, + bool compress_in_flight); - size_t find_first(int64_t value, size_t begin = 0, size_t end = size_t(-1)) const; + inline size_t find_first(int64_t value, size_t begin = 0, size_t end = size_t(-1)) const + { + return find_first(value, begin, end); + } // Wrappers for backwards compatibility and for simple use without // setting up state initialization etc @@ -400,11 +431,18 @@ class Array : public Node, public ArrayParent { // todo, would be nice to avoid this in order to speed up find_first loops QueryStateFindFirst state; Finder finder = m_vtable->finder[cond::condition]; - (this->*finder)(value, start, end, 0, &state); + finder(*this, value, start, end, 0, &state); + return state.m_state; + } - return static_cast(state.m_state); + template + bool find(int64_t value, size_t start, size_t end, size_t baseIndex, QueryStateBase* state) const + { + Finder finder = m_vtable->finder[cond::condition]; + return finder(*this, value, start, end, baseIndex, state); } + /// Get the specified element without the cost of constructing an /// array instance. If an array instance is already available, or /// you need to get multiple values, then this method will be @@ -463,7 +501,16 @@ class Array : public Node, public ArrayParent { Array& operator=(const Array&) = delete; // not allowed Array(const Array&) = delete; // not allowed + /// Takes a 64-bit value and returns the minimum number of bits needed + /// to fit the value. For alignment this is rounded up to nearest + /// log2. Possible results {0, 1, 2, 4, 8, 16, 32, 64} + static uint8_t bit_width(int64_t value); + protected: + friend class NodeTree; + void copy_on_write(); + void copy_on_write(size_t min_size); + // This returns the minimum value ("lower bound") of the representable values // for the given bit width. Valid widths are 0, 1, 2, 4, 8, 16, 32, and 64. static constexpr int_fast64_t lbound_for_width(size_t width) noexcept; @@ -489,7 +536,7 @@ class Array : public Node, public ArrayParent { /// It is an error to specify a non-zero value unless the width /// type is wtype_Bits. It is also an error to specify a non-zero /// size if the width type is wtype_Ignore. - static MemRef create(Type, bool context_flag, WidthType, size_t size, int_fast64_t value, Allocator&); + static MemRef create(Type, bool, WidthType, size_t, int_fast64_t, Allocator&); // Overriding method in ArrayParent void update_child_ref(size_t, ref_type) override; @@ -501,14 +548,17 @@ class Array : public Node, public ArrayParent { protected: // Getters and Setters for adaptive-packed arrays - typedef int64_t (Array::*Getter)(size_t) const; // Note: getters must not throw - typedef void (Array::*Setter)(size_t, int64_t); - typedef bool (Array::*Finder)(int64_t, size_t, size_t, size_t, QueryStateBase*) const; - typedef void (Array::*ChunkGetter)(size_t, int64_t res[8]) const; // Note: getters must not throw + typedef int64_t (*Getter)(const Array&, size_t); // Note: getters must not throw + typedef void (*Setter)(Array&, size_t, int64_t); + typedef bool (*Finder)(const Array&, int64_t, size_t, size_t, size_t, QueryStateBase*); + typedef void (*ChunkGetter)(const Array&, size_t, int64_t res[8]); // Note: getters must not throw + + typedef std::vector (*GetterAll)(const Array&, size_t, size_t); // Note: getters must not throw struct VTable { Getter getter; ChunkGetter chunk_getter; + GetterAll getter_all; Setter setter; Finder finder[cond_VTABLE_FINDER_COUNT]; // one for each active function pointer }; @@ -516,17 +566,12 @@ class Array : public Node, public ArrayParent { struct VTableForWidth; // This is the one installed into the m_vtable->finder slots. - template - bool find_vtable(int64_t value, size_t start, size_t end, size_t baseindex, QueryStateBase* state) const; + template + static bool find_vtable(const Array&, int64_t value, size_t start, size_t end, size_t baseindex, + QueryStateBase* state); template - int64_t get_universal(const char* const data, const size_t ndx) const; - -protected: - /// Takes a 64-bit value and returns the minimum number of bits needed - /// to fit the value. For alignment this is rounded up to nearest - /// log2. Posssible results {0, 1, 2, 4, 8, 16, 32, 64} - static size_t bit_width(int64_t value); + static int64_t get_universal(const char* const data, const size_t ndx); protected: Getter m_getter = nullptr; // cached to avoid indirection @@ -540,9 +585,14 @@ class Array : public Node, public ArrayParent { bool m_has_refs; // Elements whose first bit is zero are refs to subarrays. bool m_context_flag; // Meaning depends on context. + IntegerCompressor m_integer_compressor; + // compress/decompress this array + bool compress_array(Array&) const; + bool decompress_array(Array& arr) const; + private: ref_type do_write_shallow(_impl::ArrayWriterBase&) const; - ref_type do_write_deep(_impl::ArrayWriterBase&, bool only_if_modified) const; + ref_type do_write_deep(_impl::ArrayWriterBase&, bool only_if_modified, bool compress) const; void _mem_usage(size_t& mem) const noexcept; @@ -550,14 +600,87 @@ class Array : public Node, public ArrayParent { void report_memory_usage_2(MemUsageHandler&) const; #endif + +private: friend class Allocator; friend class SlabAlloc; friend class GroupWriter; friend class ArrayWithFind; + friend class IntegerCompressor; + friend class PackedCompressor; + friend class FlexCompressor; +}; + +class TempArray : public Array { +public: + TempArray(size_t sz, Type type = Type::type_HasRefs, bool cf = false) + : Array(Allocator::get_default()) + { + create(type, cf, sz); + } + ~TempArray() + { + destroy(); + } + ref_type write(_impl::ArrayWriterBase& out) + { + return Array::write(out, false, false, false); + } }; // Implementation: +inline Array::Array(Allocator& allocator) noexcept + : Node(allocator) +{ +} + +inline bool Array::is_compressed() const +{ + const auto enc = m_integer_compressor.get_encoding(); + return enc == NodeHeader::Encoding::Flex || enc == NodeHeader::Encoding::Packed; +} + +inline const IntegerCompressor& Array::integer_compressor() const +{ + return m_integer_compressor; +} + +inline int64_t Array::get(size_t ndx) const noexcept +{ + REALM_ASSERT_DEBUG(is_attached()); + REALM_ASSERT_DEBUG_EX(ndx < m_size, ndx, m_size); + return m_getter(*this, ndx); + + // Two ideas that are not efficient but may be worth looking into again: + /* + // Assume correct width is found early in REALM_TEMPEX, which is the case for B tree offsets that + // are probably either 2^16 long. Turns out to be 25% faster if found immediately, but 50-300% slower + // if found later + REALM_TEMPEX(return get, (ndx)); + */ + /* + // Slightly slower in both of the if-cases. Also needs an matchcount m_size check too, to avoid + // reading beyond array. + if (m_width >= 8 && m_size > ndx + 7) + return get<64>(ndx >> m_shift) & m_widthmask; + else + return (this->*(m_vtable->getter))(ndx); + */ +} + +inline std::vector Array::get_all(size_t b, size_t e) const +{ + REALM_ASSERT_DEBUG(is_compressed()); + return m_vtable->getter_all(*this, b, e); +} + +template +inline int64_t Array::get(const Array& arr, size_t ndx) noexcept +{ + REALM_ASSERT_DEBUG(arr.is_attached()); + return get_universal(arr.m_data, ndx); +} constexpr inline int_fast64_t Array::lbound_for_width(size_t width) noexcept { @@ -658,7 +781,6 @@ inline void Array::create(Type type, bool context_flag, size_t length, int_fast6 init_from_mem(mem); } - inline Array::Type Array::get_type() const noexcept { if (m_is_inner_bptree_node) { @@ -674,41 +796,44 @@ inline Array::Type Array::get_type() const noexcept inline void Array::get_chunk(size_t ndx, int64_t res[8]) const noexcept { REALM_ASSERT_DEBUG(ndx < m_size); - (this->*(m_vtable->chunk_getter))(ndx, res); + m_vtable->chunk_getter(*this, ndx, res); } template -int64_t Array::get_universal(const char* data, size_t ndx) const +inline int64_t Array::get_universal(const char* data, size_t ndx) { - if (w == 0) { - return 0; - } - else if (w == 1) { - size_t offset = ndx >> 3; - return (data[offset] >> (ndx & 7)) & 0x01; + if (w == 64) { + size_t offset = ndx << 3; + return *reinterpret_cast(data + offset); } - else if (w == 2) { - size_t offset = ndx >> 2; - return (data[offset] >> ((ndx & 3) << 1)) & 0x03; + else if (w == 32) { + size_t offset = ndx << 2; + return *reinterpret_cast(data + offset); } - else if (w == 4) { - size_t offset = ndx >> 1; - return (data[offset] >> ((ndx & 1) << 2)) & 0x0F; + else if (w == 16) { + size_t offset = ndx << 1; + return *reinterpret_cast(data + offset); } else if (w == 8) { return *reinterpret_cast(data + ndx); } - else if (w == 16) { - size_t offset = ndx * 2; - return *reinterpret_cast(data + offset); + else if (w == 4) { + size_t offset = ndx >> 1; + auto d = data[offset]; + return (d >> ((ndx & 1) << 2)) & 0x0F; } - else if (w == 32) { - size_t offset = ndx * 4; - return *reinterpret_cast(data + offset); + else if (w == 2) { + size_t offset = ndx >> 2; + auto d = data[offset]; + return (d >> ((ndx & 3) << 1)) & 0x03; } - else if (w == 64) { - size_t offset = ndx * 8; - return *reinterpret_cast(data + offset); + else if (w == 1) { + size_t offset = ndx >> 3; + auto d = data[offset]; + return (d >> (ndx & 7)) & 0x01; + } + else if (w == 0) { + return 0; } else { REALM_ASSERT_DEBUG(false); @@ -716,35 +841,6 @@ int64_t Array::get_universal(const char* data, size_t ndx) const } } -template -int64_t Array::get(size_t ndx) const noexcept -{ - return get_universal(m_data, ndx); -} - -inline int64_t Array::get(size_t ndx) const noexcept -{ - REALM_ASSERT_DEBUG(is_attached()); - REALM_ASSERT_DEBUG_EX(ndx < m_size, ndx, m_size); - return (this->*m_getter)(ndx); - - // Two ideas that are not efficient but may be worth looking into again: - /* - // Assume correct width is found early in REALM_TEMPEX, which is the case for B tree offsets that - // are probably either 2^16 long. Turns out to be 25% faster if found immediately, but 50-300% slower - // if found later - REALM_TEMPEX(return get, (ndx)); - */ - /* - // Slightly slower in both of the if-cases. Also needs an matchcount m_size check too, to avoid - // reading beyond array. - if (m_width >= 8 && m_size > ndx + 7) - return get<64>(ndx >> m_shift) & m_widthmask; - else - return (this->*(m_vtable->getter))(ndx); - */ -} - inline int64_t Array::front() const noexcept { return get(0); @@ -833,33 +929,6 @@ inline void Array::destroy_deep() noexcept m_data = nullptr; } -inline ref_type Array::write(_impl::ArrayWriterBase& out, bool deep, bool only_if_modified) const -{ - REALM_ASSERT(is_attached()); - - if (only_if_modified && m_alloc.is_read_only(m_ref)) - return m_ref; - - if (!deep || !m_has_refs) - return do_write_shallow(out); // Throws - - return do_write_deep(out, only_if_modified); // Throws -} - -inline ref_type Array::write(ref_type ref, Allocator& alloc, _impl::ArrayWriterBase& out, bool only_if_modified) -{ - if (only_if_modified && alloc.is_read_only(ref)) - return ref; - - Array array(alloc); - array.init_from_ref(ref); - - if (!array.m_has_refs) - return array.do_write_shallow(out); // Throws - - return array.do_write_deep(out, only_if_modified); // Throws -} - inline void Array::add(int_fast64_t value) { insert(m_size, value); @@ -942,8 +1011,7 @@ inline void Array::adjust(size_t begin, size_t end, int_fast64_t diff) inline size_t Array::get_byte_size() const noexcept { const char* header = get_header_from_data(m_data); - WidthType wtype = Node::get_wtype_from_header(header); - size_t num_bytes = NodeHeader::calc_byte_size(wtype, m_size, m_width); + size_t num_bytes = NodeHeader::get_byte_size_from_header(header); REALM_ASSERT_7(m_alloc.is_read_only(m_ref), ==, true, ||, num_bytes, <=, get_capacity_from_header(header)); @@ -971,7 +1039,6 @@ inline size_t Array::get_max_byte_size(size_t num_elems) noexcept return header_size + num_elems * max_bytes_per_elem; } - inline void Array::update_child_ref(size_t child_ndx, ref_type new_ref) { set(child_ndx, new_ref); @@ -989,6 +1056,73 @@ inline void Array::ensure_minimum_width(int_fast64_t value) do_ensure_minimum_width(value); } +inline ref_type Array::write(_impl::ArrayWriterBase& out, bool deep, bool only_if_modified, + bool compress_in_flight) const +{ + REALM_ASSERT_DEBUG(is_attached()); + // The default allocator cannot be trusted wrt is_read_only(): + REALM_ASSERT_DEBUG(!only_if_modified || &m_alloc != &Allocator::get_default()); + if (only_if_modified && m_alloc.is_read_only(m_ref)) + return m_ref; + + if (!deep || !m_has_refs) { + // however - creating an array using ANYTHING BUT the default allocator during commit is also wrong.... + // it only works by accident, because the whole slab area is reinitialized after commit. + // We should have: Array encoded_array{Allocator::get_default()}; + Array compressed_array{Allocator::get_default()}; + if (compress_in_flight && compress_array(compressed_array)) { +#ifdef REALM_DEBUG + const auto encoding = compressed_array.m_integer_compressor.get_encoding(); + REALM_ASSERT_DEBUG(encoding == Encoding::Flex || encoding == Encoding::Packed); + REALM_ASSERT_DEBUG(size() == compressed_array.size()); + for (size_t i = 0; i < compressed_array.size(); ++i) { + REALM_ASSERT_DEBUG(get(i) == compressed_array.get(i)); + } +#endif + auto ref = compressed_array.do_write_shallow(out); + compressed_array.destroy(); + return ref; + } + return do_write_shallow(out); // Throws + } + + return do_write_deep(out, only_if_modified, compress_in_flight); // Throws +} + +inline ref_type Array::write(ref_type ref, Allocator& alloc, _impl::ArrayWriterBase& out, bool only_if_modified, + bool compress_in_flight) +{ + // The default allocator cannot be trusted wrt is_read_only(): + REALM_ASSERT_DEBUG(!only_if_modified || &alloc != &Allocator::get_default()); + if (only_if_modified && alloc.is_read_only(ref)) + return ref; + + Array array(alloc); + array.init_from_ref(ref); + REALM_ASSERT_DEBUG(array.is_attached()); + + if (!array.m_has_refs) { + Array compressed_array{Allocator::get_default()}; + if (compress_in_flight && array.compress_array(compressed_array)) { +#ifdef REALM_DEBUG + const auto encoding = compressed_array.m_integer_compressor.get_encoding(); + REALM_ASSERT_DEBUG(encoding == Encoding::Flex || encoding == Encoding::Packed); + REALM_ASSERT_DEBUG(array.size() == compressed_array.size()); + for (size_t i = 0; i < compressed_array.size(); ++i) { + REALM_ASSERT_DEBUG(array.get(i) == compressed_array.get(i)); + } +#endif + auto ref = compressed_array.do_write_shallow(out); + compressed_array.destroy(); + return ref; + } + else { + return array.do_write_shallow(out); // Throws + } + } + return array.do_write_deep(out, only_if_modified, compress_in_flight); // Throws +} + } // namespace realm diff --git a/src/realm/array_aggregate_optimizations.cpp b/src/realm/array_aggregate_optimizations.cpp new file mode 100644 index 00000000000..6242e6853dd --- /dev/null +++ b/src/realm/array_aggregate_optimizations.cpp @@ -0,0 +1,369 @@ +/************************************************************************* + * + * Copyright 2023 Realm Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + **************************************************************************/ + +#include +#include + +using namespace realm; + +int64_t Array::sum(size_t start, size_t end) const +{ + REALM_TEMPEX(return sum, m_width, (start, end)); +} + +template +int64_t Array::sum(size_t start, size_t end) const +{ + if (end == size_t(-1)) + end = m_size; + + REALM_ASSERT_EX(end <= m_size && start <= end, start, end, m_size); + + if (start == end) + return 0; + + int64_t s = 0; + + // Sum manually until 128 bit aligned + for (; (start < end) && (((size_t(m_data) & 0xf) * 8 + start * w) % 128 != 0); start++) { + s += get(*this, start); + } + + if (w == 1 || w == 2 || w == 4) { + // Sum of bitwidths less than a byte (which are always positive) + // uses a divide and conquer algorithm that is a variation of popolation count: + // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel + + // static values needed for fast sums + const uint64_t m2 = 0x3333333333333333ULL; + const uint64_t m4 = 0x0f0f0f0f0f0f0f0fULL; + const uint64_t h01 = 0x0101010101010101ULL; + + int64_t* data = reinterpret_cast(m_data + start * w / 8); + size_t chunks = (end - start) * w / 8 / sizeof(int64_t); + + for (size_t t = 0; t < chunks; t++) { + if (w == 1) { +#if 0 +#if defined(USE_SSE42) && defined(_MSC_VER) && defined(REALM_PTR_64) + s += __popcnt64(data[t]); +#elif !defined(_MSC_VER) && defined(USE_SSE42) && defined(REALM_PTR_64) + s += __builtin_popcountll(data[t]); +#else + uint64_t a = data[t]; + const uint64_t m1 = 0x5555555555555555ULL; + a -= (a >> 1) & m1; + a = (a & m2) + ((a >> 2) & m2); + a = (a + (a >> 4)) & m4; + a = (a * h01) >> 56; + s += a; +#endif +#endif + s += fast_popcount64(data[t]); + } + else if (w == 2) { + uint64_t a = data[t]; + a = (a & m2) + ((a >> 2) & m2); + a = (a + (a >> 4)) & m4; + a = (a * h01) >> 56; + + s += a; + } + else if (w == 4) { + uint64_t a = data[t]; + a = (a & m4) + ((a >> 4) & m4); + a = (a * h01) >> 56; + s += a; + } + } + start += sizeof(int64_t) * 8 / no0(w) * chunks; + } + +#ifdef REALM_COMPILER_SSE + if (sseavx<42>()) { + // 2000 items summed 500000 times, 8/16/32 bits, miliseconds: + // Naive, templated get<>: 391 371 374 + // SSE: 97 148 282 + + if ((w == 8 || w == 16 || w == 32) && end - start > sizeof(__m128i) * 8 / no0(w)) { + __m128i* data = reinterpret_cast<__m128i*>(m_data + start * w / 8); + __m128i sum_result = {0}; + __m128i sum2; + + size_t chunks = (end - start) * w / 8 / sizeof(__m128i); + + for (size_t t = 0; t < chunks; t++) { + if (w == 8) { + /* + // 469 ms AND disadvantage of handling max 64k elements before overflow + __m128i vl = _mm_cvtepi8_epi16(data[t]); + __m128i vh = data[t]; + vh.m128i_i64[0] = vh.m128i_i64[1]; + vh = _mm_cvtepi8_epi16(vh); + sum_result = _mm_add_epi16(sum_result, vl); + sum_result = _mm_add_epi16(sum_result, vh); + */ + + /* + // 424 ms + __m128i vl = _mm_unpacklo_epi8(data[t], _mm_set1_epi8(0)); + __m128i vh = _mm_unpackhi_epi8(data[t], _mm_set1_epi8(0)); + sum_result = _mm_add_epi32(sum_result, _mm_madd_epi16(vl, _mm_set1_epi16(1))); + sum_result = _mm_add_epi32(sum_result, _mm_madd_epi16(vh, _mm_set1_epi16(1))); + */ + + __m128i vl = _mm_cvtepi8_epi16(data[t]); // sign extend lower words 8->16 + __m128i vh = data[t]; + vh = _mm_srli_si128(vh, 8); // v >>= 64 + vh = _mm_cvtepi8_epi16(vh); // sign extend lower words 8->16 + __m128i sum1 = _mm_add_epi16(vl, vh); + __m128i sumH = _mm_cvtepi16_epi32(sum1); + __m128i sumL = _mm_srli_si128(sum1, 8); // v >>= 64 + sumL = _mm_cvtepi16_epi32(sumL); + sum_result = _mm_add_epi32(sum_result, sumL); + sum_result = _mm_add_epi32(sum_result, sumH); + } + else if (w == 16) { + // todo, can overflow for array size > 2^32 + __m128i vl = _mm_cvtepi16_epi32(data[t]); // sign extend lower words 16->32 + __m128i vh = data[t]; + vh = _mm_srli_si128(vh, 8); // v >>= 64 + vh = _mm_cvtepi16_epi32(vh); // sign extend lower words 16->32 + sum_result = _mm_add_epi32(sum_result, vl); + sum_result = _mm_add_epi32(sum_result, vh); + } + else if (w == 32) { + __m128i v = data[t]; + __m128i v0 = _mm_cvtepi32_epi64(v); // sign extend lower dwords 32->64 + v = _mm_srli_si128(v, 8); // v >>= 64 + __m128i v1 = _mm_cvtepi32_epi64(v); // sign extend lower dwords 32->64 + sum_result = _mm_add_epi64(sum_result, v0); + sum_result = _mm_add_epi64(sum_result, v1); + + /* + __m128i m = _mm_set1_epi32(0xc000); // test if overflow could happen (still need + underflow test). + __m128i mm = _mm_and_si128(data[t], m); + zz = _mm_or_si128(mm, zz); + sum_result = _mm_add_epi32(sum_result, data[t]); + */ + } + } + start += sizeof(__m128i) * 8 / no0(w) * chunks; + + // prevent taking address of 'state' to make the compiler keep it in SSE register in above loop + // (vc2010/gcc4.6) + sum2 = sum_result; + + // Avoid aliasing bug where sum2 might not yet be initialized when accessed by get_universal + char sum3[sizeof sum2]; + memcpy(&sum3, &sum2, sizeof sum2); + + // Sum elements of sum + for (size_t t = 0; t < sizeof(__m128i) * 8 / ((w == 8 || w == 16) ? 32 : 64); ++t) { + int64_t v = get_universal < (w == 8 || w == 16) ? 32 : 64 > (reinterpret_cast(&sum3), t); + s += v; + } + } + } +#endif + + // Sum remaining elements + for (; start < end; ++start) + s += get(*this, start); + + return s; +} + +size_t Array::count(int64_t value) const noexcept +{ + // This is not used anywhere in the code, I believe we can delete this + // since the query logic does not use this + const uint64_t* next = reinterpret_cast(m_data); + size_t value_count = 0; + const size_t end = m_size; + size_t i = 0; + + // static values needed for fast population count + const uint64_t m1 = 0x5555555555555555ULL; + const uint64_t m2 = 0x3333333333333333ULL; + const uint64_t m4 = 0x0f0f0f0f0f0f0f0fULL; + const uint64_t h01 = 0x0101010101010101ULL; + + if (m_width == 0) { + if (value == 0) + return m_size; + return 0; + } + if (m_width == 1) { + if (uint64_t(value) > 1) + return 0; + + const size_t chunkvals = 64; + for (; i + chunkvals <= end; i += chunkvals) { + uint64_t a = next[i / chunkvals]; + if (value == 0) + a = ~a; // reverse + + a -= (a >> 1) & m1; + a = (a & m2) + ((a >> 2) & m2); + a = (a + (a >> 4)) & m4; + a = (a * h01) >> 56; + + // Could use intrinsic instead: + // a = __builtin_popcountll(a); // gcc intrinsic + + value_count += to_size_t(a); + } + } + else if (m_width == 2) { + if (uint64_t(value) > 3) + return 0; + + const uint64_t v = ~0ULL / 0x3 * value; + + // Masks to avoid spillover between segments in cascades + const uint64_t c1 = ~0ULL / 0x3 * 0x1; + + const size_t chunkvals = 32; + for (; i + chunkvals <= end; i += chunkvals) { + uint64_t a = next[i / chunkvals]; + a ^= v; // zero matching bit segments + a |= (a >> 1) & c1; // cascade ones in non-zeroed segments + a &= m1; // isolate single bit in each segment + a ^= m1; // reverse isolated bits + // if (!a) continue; + + // Population count + a = (a & m2) + ((a >> 2) & m2); + a = (a + (a >> 4)) & m4; + a = (a * h01) >> 56; + + value_count += to_size_t(a); + } + } + else if (m_width == 4) { + if (uint64_t(value) > 15) + return 0; + + const uint64_t v = ~0ULL / 0xF * value; + const uint64_t m = ~0ULL / 0xF * 0x1; + + // Masks to avoid spillover between segments in cascades + const uint64_t c1 = ~0ULL / 0xF * 0x7; + const uint64_t c2 = ~0ULL / 0xF * 0x3; + + const size_t chunkvals = 16; + for (; i + chunkvals <= end; i += chunkvals) { + uint64_t a = next[i / chunkvals]; + a ^= v; // zero matching bit segments + a |= (a >> 1) & c1; // cascade ones in non-zeroed segments + a |= (a >> 2) & c2; + a &= m; // isolate single bit in each segment + a ^= m; // reverse isolated bits + + // Population count + a = (a + (a >> 4)) & m4; + a = (a * h01) >> 56; + + value_count += to_size_t(a); + } + } + else if (m_width == 8) { + if (value > 0x7FLL || value < -0x80LL) + return 0; // by casting? + + const uint64_t v = ~0ULL / 0xFF * value; + const uint64_t m = ~0ULL / 0xFF * 0x1; + + // Masks to avoid spillover between segments in cascades + const uint64_t c1 = ~0ULL / 0xFF * 0x7F; + const uint64_t c2 = ~0ULL / 0xFF * 0x3F; + const uint64_t c3 = ~0ULL / 0xFF * 0x0F; + + const size_t chunkvals = 8; + for (; i + chunkvals <= end; i += chunkvals) { + uint64_t a = next[i / chunkvals]; + a ^= v; // zero matching bit segments + a |= (a >> 1) & c1; // cascade ones in non-zeroed segments + a |= (a >> 2) & c2; + a |= (a >> 4) & c3; + a &= m; // isolate single bit in each segment + a ^= m; // reverse isolated bits + + // Population count + a = (a * h01) >> 56; + + value_count += to_size_t(a); + } + } + else if (m_width == 16) { + if (value > 0x7FFFLL || value < -0x8000LL) + return 0; // by casting? + + const uint64_t v = ~0ULL / 0xFFFF * value; + const uint64_t m = ~0ULL / 0xFFFF * 0x1; + + // Masks to avoid spillover between segments in cascades + const uint64_t c1 = ~0ULL / 0xFFFF * 0x7FFF; + const uint64_t c2 = ~0ULL / 0xFFFF * 0x3FFF; + const uint64_t c3 = ~0ULL / 0xFFFF * 0x0FFF; + const uint64_t c4 = ~0ULL / 0xFFFF * 0x00FF; + + const size_t chunkvals = 4; + for (; i + chunkvals <= end; i += chunkvals) { + uint64_t a = next[i / chunkvals]; + a ^= v; // zero matching bit segments + a |= (a >> 1) & c1; // cascade ones in non-zeroed segments + a |= (a >> 2) & c2; + a |= (a >> 4) & c3; + a |= (a >> 8) & c4; + a &= m; // isolate single bit in each segment + a ^= m; // reverse isolated bits + + // Population count + a = (a * h01) >> 56; + + value_count += to_size_t(a); + } + } + else if (m_width == 32) { + int32_t v = int32_t(value); + const int32_t* d = reinterpret_cast(m_data); + for (; i < end; ++i) { + if (d[i] == v) + ++value_count; + } + return value_count; + } + else if (m_width == 64) { + const int64_t* d = reinterpret_cast(m_data); + for (; i < end; ++i) { + if (d[i] == value) + ++value_count; + } + return value_count; + } + + // Check remaining elements + for (; i < end; ++i) + if (value == get(i)) + ++value_count; + + return value_count; +} diff --git a/src/realm/array_basic_tpl.hpp b/src/realm/array_basic_tpl.hpp index 4c8e0ae2319..fc1e1c04df1 100644 --- a/src/realm/array_basic_tpl.hpp +++ b/src/realm/array_basic_tpl.hpp @@ -40,16 +40,12 @@ inline MemRef BasicArray::create_array(size_t init_size, Allocator& allocator // Adding zero to Array::initial_capacity to avoid taking the // address of that member size_t byte_size = std::max(byte_size_0, Node::initial_capacity + 0); // Throws - - MemRef mem = allocator.alloc(byte_size); // Throws - - bool is_inner_bptree_node = false; - bool has_refs = false; - bool context_flag = false; - int width = sizeof(T); - init_header(mem.get_addr(), is_inner_bptree_node, has_refs, context_flag, wtype_Multiply, width, init_size, - byte_size); - + MemRef mem = allocator.alloc(byte_size); // Throws + uint8_t flags = 0; + const int width = sizeof(T) * 8; // element width is in bits now + const auto header = mem.get_addr(); + init_header(header, Encoding::WTypMult, flags, width, init_size); + set_capacity_in_header(byte_size, header); return mem; } diff --git a/src/realm/array_blobs_small.cpp b/src/realm/array_blobs_small.cpp index bca4d012a1f..4e93f40c5f4 100644 --- a/src/realm/array_blobs_small.cpp +++ b/src/realm/array_blobs_small.cpp @@ -91,7 +91,8 @@ void ArraySmallBlobs::erase(size_t ndx) REALM_ASSERT_3(ndx, <, m_offsets.size()); size_t start = ndx ? to_size_t(m_offsets.get(ndx - 1)) : 0; - size_t end = to_size_t(m_offsets.get(ndx)); + auto offset = m_offsets.get(ndx); + size_t end = to_size_t(offset); m_blob.erase(start, end); m_offsets.erase(ndx); diff --git a/src/realm/array_blobs_small.hpp b/src/realm/array_blobs_small.hpp index 8db3467a209..e1a08e43e4f 100644 --- a/src/realm/array_blobs_small.hpp +++ b/src/realm/array_blobs_small.hpp @@ -176,7 +176,8 @@ inline BinaryData ArraySmallBlobs::get(size_t ndx) const noexcept } else { size_t begin = ndx ? to_size_t(m_offsets.get(ndx - 1)) : 0; - size_t end = to_size_t(m_offsets.get(ndx)); + auto offset = m_offsets.get(ndx); + size_t end = to_size_t(offset); BinaryData bd = BinaryData(m_blob.get(begin), end - begin); // Old database file (non-nullable column should never return null) diff --git a/src/realm/array_direct.hpp b/src/realm/array_direct.hpp index d595363c762..4b92141bf55 100644 --- a/src/realm/array_direct.hpp +++ b/src/realm/array_direct.hpp @@ -19,63 +19,59 @@ #ifndef REALM_ARRAY_DIRECT_HPP #define REALM_ARRAY_DIRECT_HPP +#include #include #include // clang-format off /* wid == 16/32 likely when accessing offsets in B tree */ #define REALM_TEMPEX(fun, wid, arg) \ - if (wid == 16) {fun<16> arg;} \ - else if (wid == 32) {fun<32> arg;} \ - else if (wid == 0) {fun<0> arg;} \ - else if (wid == 1) {fun<1> arg;} \ - else if (wid == 2) {fun<2> arg;} \ - else if (wid == 4) {fun<4> arg;} \ - else if (wid == 8) {fun<8> arg;} \ - else if (wid == 64) {fun<64> arg;} \ - else {REALM_ASSERT_DEBUG(false); fun<0> arg;} +if (wid == 16) {fun<16> arg;} \ +else if (wid == 32) {fun<32> arg;} \ +else if (wid == 0) {fun<0> arg;} \ +else if (wid == 1) {fun<1> arg;} \ +else if (wid == 2) {fun<2> arg;} \ +else if (wid == 4) {fun<4> arg;} \ +else if (wid == 8) {fun<8> arg;} \ +else if (wid == 64) {fun<64> arg;} \ +else {REALM_ASSERT_DEBUG(false); fun<0> arg;} #define REALM_TEMPEX2(fun, targ, wid, arg) \ - if (wid == 16) {fun arg;} \ - else if (wid == 32) {fun arg;} \ - else if (wid == 0) {fun arg;} \ - else if (wid == 1) {fun arg;} \ - else if (wid == 2) {fun arg;} \ - else if (wid == 4) {fun arg;} \ - else if (wid == 8) {fun arg;} \ - else if (wid == 64) {fun arg;} \ - else {REALM_ASSERT_DEBUG(false); fun arg;} +if (wid == 16) {fun arg;} \ +else if (wid == 32) {fun arg;} \ +else if (wid == 0) {fun arg;} \ +else if (wid == 1) {fun arg;} \ +else if (wid == 2) {fun arg;} \ +else if (wid == 4) {fun arg;} \ +else if (wid == 8) {fun arg;} \ +else if (wid == 64) {fun arg;} \ +else {REALM_ASSERT_DEBUG(false); fun arg;} #define REALM_TEMPEX3(fun, targ1, wid, targ3, arg) \ - if (wid == 16) {fun arg;} \ - else if (wid == 32) {fun arg;} \ - else if (wid == 0) {fun arg;} \ - else if (wid == 1) {fun arg;} \ - else if (wid == 2) {fun arg;} \ - else if (wid == 4) {fun arg;} \ - else if (wid == 8) {fun arg;} \ - else if (wid == 64) {fun arg;} \ - else {REALM_ASSERT_DEBUG(false); fun arg;} +if (wid == 16) {fun arg;} \ +else if (wid == 32) {fun arg;} \ +else if (wid == 0) {fun arg;} \ +else if (wid == 1) {fun arg;} \ +else if (wid == 2) {fun arg;} \ +else if (wid == 4) {fun arg;} \ +else if (wid == 8) {fun arg;} \ +else if (wid == 64) {fun arg;} \ +else {REALM_ASSERT_DEBUG(false); fun arg;} #define REALM_TEMPEX4(fun, targ1, targ3, targ4, wid, arg) \ - if (wid == 16) {fun arg;} \ - else if (wid == 32) {fun arg;} \ - else if (wid == 0) {fun arg;} \ - else if (wid == 1) {fun arg;} \ - else if (wid == 2) {fun arg;} \ - else if (wid == 4) {fun arg;} \ - else if (wid == 8) {fun arg;} \ - else if (wid == 64) {fun arg;} \ - else {REALM_ASSERT_DEBUG(false); fun arg;} +if (wid == 16) {fun arg;} \ +else if (wid == 32) {fun arg;} \ +else if (wid == 0) {fun arg;} \ +else if (wid == 1) {fun arg;} \ +else if (wid == 2) {fun arg;} \ +else if (wid == 4) {fun arg;} \ +else if (wid == 8) {fun arg;} \ +else if (wid == 64) {fun arg;} \ +else {REALM_ASSERT_DEBUG(false); fun arg;} // clang-format on namespace realm { -/// Takes a 64-bit value and returns the minimum number of bits needed -/// to fit the value. For alignment this is rounded up to nearest -/// log2. Posssible results {0, 1, 2, 4, 8, 16, 32, 64} -size_t bit_width(int64_t value); - // Direct access methods template @@ -168,15 +164,15 @@ int64_t get_direct(const char* data, size_t ndx) noexcept return *reinterpret_cast(data + ndx); } if (w == 16) { - size_t offset = ndx * 2; + size_t offset = ndx << 1; return *reinterpret_cast(data + offset); } if (w == 32) { - size_t offset = ndx * 4; + size_t offset = ndx << 2; return *reinterpret_cast(data + offset); } if (w == 64) { - size_t offset = ndx * 8; + size_t offset = ndx << 3; return *reinterpret_cast(data + offset); } REALM_ASSERT_DEBUG(false); @@ -188,6 +184,229 @@ inline int64_t get_direct(const char* data, size_t width, size_t ndx) noexcept REALM_TEMPEX(return get_direct, width, (data, ndx)); } +// An iterator for getting a 64 bit word from any (byte-address+bit-offset) address. +class UnalignedWordIter { +public: + UnalignedWordIter(const uint64_t* data, size_t bit_offset) + : m_word_ptr(data + (bit_offset >> 6)) + , m_in_word_offset(bit_offset & 0x3F) + { + } + // 'num_bits' number of bits which must be read + // WARNING returned word may be garbage above the first 'num_bits' bits. + uint64_t consume(size_t num_bits) + { + auto first_word = m_word_ptr[0]; + uint64_t result = first_word >> m_in_word_offset; + // note: above shifts in zeroes + if (m_in_word_offset + num_bits > 64) { + // if we're here, in_word_offset > 0 + auto first_word_size = 64 - m_in_word_offset; + auto second_word = m_word_ptr[1]; + result |= second_word << first_word_size; + // note: above shifts in zeroes below the bits we want + } + _bump(num_bits); + return result; + } + uint64_t consume_with_unsafe_prefetch(size_t num_bits) + { + auto first_word = m_word_ptr[0]; + uint64_t result = first_word >> m_in_word_offset; + // note: above shifts in zeroes + auto first_word_size = 64 - m_in_word_offset; + auto second_word = m_word_ptr[1]; + REALM_ASSERT_DEBUG(num_bits <= 64); + if (num_bits > first_word_size) + result |= second_word << first_word_size; + // note: above shifts in zeroes below the bits we want + _bump(num_bits); + return result; + } + +private: + const uint64_t* m_word_ptr; + unsigned m_in_word_offset; + + // bump the iterator the specified number of bits + void _bump(size_t num_bits) + { + auto total_offset = m_in_word_offset + num_bits; + m_word_ptr += total_offset >> 6; + m_in_word_offset = total_offset & 0x3F; + } +}; + +// Read a bit field of up to 64 bits. +// - Any alignment and size is supported +// - The start of the 'data' area must be 64 bit aligned in all cases. +// - For fields of 64-bit or less, the first 64-bit word is filled with the zero-extended +// value of the bitfield. +// iterator useful for scanning arrays faster than by indexing each element +// supports arrays of pairs by differentiating field size and step size. +class BfIterator { + friend class FlexCompressor; + friend class PackedCompressor; + +public: + BfIterator() = default; + BfIterator(const BfIterator&) = default; + BfIterator(BfIterator&&) = default; + BfIterator& operator=(const BfIterator&) = default; + BfIterator& operator=(BfIterator&&) = default; + BfIterator(uint64_t* data_area, size_t initial_offset, uint8_t field_size, uint8_t step_size, size_t index) + : data_area(data_area) + , field_size(field_size) + , step_size(step_size) + , offset(initial_offset) + { + if (field_size < 64) + mask = (1ULL << field_size) - 1; + move(index); + } + + inline uint64_t get_full_word_with_value() const + { + const auto in_word_position = field_position & 0x3F; + const auto first_word = first_word_ptr[0]; + uint64_t result = first_word >> in_word_position; + // note: above shifts in zeroes above the bitfield + if (in_word_position + field_size > 64) { + // if we're here, in_word_position > 0 + const auto first_word_size = 64 - in_word_position; + const auto second_word = first_word_ptr[1]; + return result | second_word << first_word_size; + // note: above shifts in zeroes below the bits we want + } + return result; + } + + inline uint64_t get_value() const + { + auto result = get_full_word_with_value(); + // discard any bits above the field we want + if (field_size < 64) + result &= mask; + return result; + } + + // get unaligned word - this should not be called if the next word extends beyond + // end of array. For that particular case, you must use get_last_unaligned_word instead. + inline uint64_t get_unaligned_word() const + { + const auto in_word_position = field_position & 0x3F; + const auto first_word = first_word_ptr[0]; + if (in_word_position == 0) + return first_word; + uint64_t result = first_word >> in_word_position; + // note: above shifts in zeroes above the bitfield + const auto first_word_size = 64 - in_word_position; + const auto second_word = first_word_ptr[1]; + result |= second_word << first_word_size; + // note: above shifts in zeroes below the bits we want + return result; + } + + inline uint64_t get_last_unaligned_word() const + { + const auto in_word_position = field_position & 0x3F; + const auto first_word = first_word_ptr[0]; + const uint64_t result = first_word >> in_word_position; + // note: above shifts in zeroes above the bitfield + return result; + } + + void set_value(uint64_t value) const + { + const auto in_word_position = field_position & 0x3F; + auto first_word = first_word_ptr[0]; + uint64_t mask = 0ULL - 1ULL; + if (field_size < 64) { + mask = (1ULL << field_size) - 1; + value &= mask; + } + // zero out field in first word: + const auto first_word_mask = ~(mask << in_word_position); + first_word &= first_word_mask; + // or in relevant part of value + first_word |= value << in_word_position; + first_word_ptr[0] = first_word; + if (in_word_position + field_size > 64) { + // bitfield crosses word boundary. + // discard the lowest bits of value (it has been written to the first word) + const auto bits_written_to_first_word = 64 - in_word_position; + // bit_written_to_first_word must be lower than 64, so shifts based on it are well defined + value >>= bits_written_to_first_word; + const auto second_word_mask = mask >> bits_written_to_first_word; + auto second_word = first_word_ptr[1]; + // zero out the field in second word, then or in the (high part of) value + second_word &= ~second_word_mask; + second_word |= value; + first_word_ptr[1] = second_word; + } + } + inline void operator++() + { + const auto next_field_position = field_position + step_size; + if ((next_field_position >> 6) > (field_position >> 6)) { + first_word_ptr = data_area + (next_field_position >> 6); + } + field_position = next_field_position; + } + + inline void move(size_t index) + { + field_position = offset + index * step_size; + first_word_ptr = data_area + (field_position >> 6); + } + + inline uint64_t operator*() const + { + return get_value(); + } + +private: + friend bool operator<(const BfIterator&, const BfIterator&); + uint64_t* data_area = nullptr; + uint64_t* first_word_ptr = nullptr; + size_t field_position = 0; + uint8_t field_size = 0; + uint8_t step_size = 0; // may be different than field_size if used for arrays of pairs + size_t offset = 0; + uint64_t mask = 0; +}; + + +inline bool operator<(const BfIterator& a, const BfIterator& b) +{ + REALM_ASSERT(a.data_area == b.data_area); + return a.field_position < b.field_position; +} + +inline uint64_t read_bitfield(uint64_t* data_area, size_t field_position, uint8_t width) +{ + BfIterator it(data_area, field_position, width, width, 0); + return *it; +} + +inline void write_bitfield(uint64_t* data_area, size_t field_position, uint8_t width, uint64_t value) +{ + BfIterator it(data_area, field_position, width, width, 0); + it.set_value(value); +} + +inline int64_t sign_extend_field_by_mask(uint64_t sign_mask, uint64_t value) +{ + uint64_t sign_extension = 0ULL - (value & sign_mask); + return value | sign_extension; +} + +inline int64_t sign_extend_value(size_t width, uint64_t value) +{ + uint64_t sign_mask = 1ULL << (width - 1); + uint64_t sign_extension = 0ULL - (value & sign_mask); + return value | sign_extension; +} template inline std::pair get_two(const char* data, size_t ndx) noexcept @@ -200,6 +419,245 @@ inline std::pair get_two(const char* data, size_t width, size_ REALM_TEMPEX(return get_two, width, (data, ndx)); } +/* Subword parallel search + + The following provides facilities for subword parallel search for bitfields of any size. + To simplify, the first bitfield must be aligned within the word: it must occupy the lowest + bits of the word. + + In general the metods here return a vector with the most significant bit in each field + marking that a condition was met when comparing the corresponding pair of fields in two + vectors. Checking if any field meets a condition is as simple as comparing the return + vector against 0. Finding the first to meet a condition is also supported. + + Vectors are "split" into fields according to a MSB vector, wich indicates the most + significant bit of each field. The MSB must be passed in as an argument to most + bit field comparison functions. It can be generated by the field_sign_bit template. + + The simplest condition to test is any_field_NE(A,B), where A and B are words. + This condition should be true if any bitfield in A is not equal to the corresponding + field in B. + + This is almost as simple as a direct word compare, but needs to take into account that + we may want to have part of the words undefined. + */ +constexpr uint8_t num_fields_table[65] = {0, 64, 32, 21, 16, 12, 10, 9, // 0-7 + 8, 7, 6, 5, 5, 4, 4, 4, // 8-15 + 4, 3, 3, 3, 3, 3, 2, 2, // 16-23 + 2, 2, 2, 2, 2, 2, 2, 2, // 24-31 + 2, 1, 1, 1, 1, 1, 1, 1, // 32-39 + 1, 1, 1, 1, 1, 1, 1, 1, // 40-47 + 1, 1, 1, 1, 1, 1, 1, 1, // 48-55 + 1, 1, 1, 1, 1, 1, 1, 1, // 56-63 + 1}; + +constexpr uint8_t num_bits_table[65] = {64, 64, 64, 63, 64, 60, 60, 63, // 0-7 + 64, 63, 60, 55, 60, 52, 56, 60, // 8-15 + 64, 51, 54, 57, 60, 63, 44, 46, // 16-23 + 48, 50, 52, 54, 56, 58, 60, 62, // 24-31 + 64, 33, 34, 35, 36, 37, 38, 39, // 32-39 + 40, 41, 42, 43, 44, 45, 46, 47, // 40-47 + 48, 49, 50, 51, 52, 53, 54, 55, // 48-55 + 56, 57, 58, 59, 60, 61, 62, 63, // 56-63 + 64}; + +inline uint8_t num_fields_for_width(uint8_t width) +{ + REALM_ASSERT_DEBUG(width); + const auto retval = num_fields_table[width]; +#ifdef REALM_DEBUG + REALM_ASSERT_DEBUG(width == 0 || retval == int(64 / width)); +#endif + return retval; +} + +inline uint8_t num_bits_for_width(uint8_t width) +{ + REALM_ASSERT_DEBUG(width); + return num_bits_table[width]; +} + +inline uint64_t cares_about(uint8_t width) +{ + return 0xFFFFFFFFFFFFFFFFULL >> (64 - num_bits_table[width]); +} + +// true if any field in A differs from corresponding field in B. If you also want +// to find which fields, use find_all_fields_NE instead. +bool inline any_field_NE(int width, uint64_t A, uint64_t B) +{ + return (A ^ B) & cares_about(width); +} + +// Populate all fields in a vector with a given value of a give width. +// Bits outside of the given field are ignored. +constexpr uint64_t populate(size_t width, uint64_t value) +{ + value &= 0xFFFFFFFFFFFFFFFFULL >> (64 - width); + if (width < 8) { + value |= value << width; + width <<= 1; + value |= value << width; + width <<= 1; + value |= value << width; + width <<= 1; + } + // width now in range 8..64 + if (width < 32) { + value |= value << width; + width <<= 1; + value |= value << width; + width <<= 1; + } + // width now in range 32..128 + if (width < 64) { + value |= value << width; + } + return value; +} + +// provides a set bit in pos 0 of each field, remaining bits zero +constexpr uint64_t field_bit0(int width) +{ + return populate(width, 1); +} + +// provides a set sign-bit in each field, remaining bits zero +constexpr uint64_t field_sign_bit(int width) +{ + return populate(width, 1ULL << (width - 1)); +} + +constexpr uint32_t inverse_width[65] = { + 65536 * 64 / 1, // never used + 65536 * 64 / 1, 65536 * 64 / 2, 65536 * 64 / 3, 65536 * 64 / 4, 65536 * 64 / 5, 65536 * 64 / 6, + 65536 * 64 / 7, 65536 * 64 / 8, 65536 * 64 / 9, 65536 * 64 / 10, 65536 * 64 / 11, 65536 * 64 / 12, + 65536 * 64 / 13, 65536 * 64 / 14, 65536 * 64 / 15, 65536 * 64 / 16, 65536 * 64 / 17, 65536 * 64 / 18, + 65536 * 64 / 19, 65536 * 64 / 20, 65536 * 64 / 21, 65536 * 64 / 22, 65536 * 64 / 23, 65536 * 64 / 24, + 65536 * 64 / 25, 65536 * 64 / 26, 65536 * 64 / 27, 65536 * 64 / 28, 65536 * 64 / 29, 65536 * 64 / 30, + 65536 * 64 / 31, 65536 * 64 / 32, 65536 * 64 / 33, 65536 * 64 / 34, 65536 * 64 / 35, 65536 * 64 / 36, + 65536 * 64 / 37, 65536 * 64 / 38, 65536 * 64 / 39, 65536 * 64 / 40, 65536 * 64 / 41, 65536 * 64 / 42, + 65536 * 64 / 43, 65536 * 64 / 44, 65536 * 64 / 45, 65536 * 64 / 46, 65536 * 64 / 47, 65536 * 64 / 48, + 65536 * 64 / 49, 65536 * 64 / 50, 65536 * 64 / 51, 65536 * 64 / 52, 65536 * 64 / 53, 65536 * 64 / 54, + 65536 * 64 / 55, 65536 * 64 / 56, 65536 * 64 / 57, 65536 * 64 / 58, 65536 * 64 / 59, 65536 * 64 / 60, + 65536 * 64 / 61, 65536 * 64 / 62, 65536 * 64 / 63, 65536 * 64 / 64, +}; + +inline size_t countr_zero(uint64_t vector) +{ + unsigned long where; +#if defined(_WIN64) + if (_BitScanForward64(&where, vector)) + return static_cast(where); + return 0; +#elif defined(_WIN32) + uint32_t low = vector & 0xFFFFFFFF; + if (low) { + bool scan_ok = _BitScanForward(&where, low); + REALM_ASSERT_DEBUG(scan_ok); + return where; + } + else { + low = vector >> 32; + bool scan_ok = _BitScanForward(&where, low); + REALM_ASSERT_DEBUG(scan_ok); + return 32 + where; + } +#else + where = __builtin_ctzll(vector); + return static_cast(where); +#endif +} + +inline size_t first_field_marked(size_t width, uint64_t vector) +{ + const auto lz = countr_zero(vector); + const auto field = (lz * inverse_width[width]) >> 22; + REALM_ASSERT_DEBUG(width != 0); + REALM_ASSERT_DEBUG(field == (lz / width)); + return field; +} + +template +size_t parallel_subword_find(VectorCompare vector_compare, const uint64_t* data, size_t offset, uint8_t width, + uint64_t MSBs, uint64_t search_vector, size_t start, size_t end) +{ + const auto field_count = num_fields_for_width(width); + const auto bit_count_pr_iteration = num_bits_for_width(width); + const size_t fast_scan_limit = 4 * bit_count_pr_iteration; + // use signed to make it easier to ascertain correctness of loop condition below + auto total_bit_count_left = (end - start) * width; + REALM_ASSERT_DEBUG(end >= start); + UnalignedWordIter it(data, offset + start * width); + uint64_t found_vector = 0; + while (total_bit_count_left >= fast_scan_limit) { + // unrolling 2x + const auto word0 = it.consume_with_unsafe_prefetch(bit_count_pr_iteration); + const auto word1 = it.consume_with_unsafe_prefetch(bit_count_pr_iteration); + auto found_vector0 = vector_compare(MSBs, word0, search_vector); + auto found_vector1 = vector_compare(MSBs, word1, search_vector); + if (found_vector0) { + const auto sub_word_index = first_field_marked(width, found_vector0); + return start + sub_word_index; + } + if (found_vector1) { + const auto sub_word_index = first_field_marked(width, found_vector1); + return start + field_count + sub_word_index; + } + total_bit_count_left -= 2 * bit_count_pr_iteration; + start += 2 * field_count; + } + + // One word at a time + while (total_bit_count_left >= bit_count_pr_iteration) { + const auto word = it.consume(bit_count_pr_iteration); + found_vector = vector_compare(MSBs, word, search_vector); + if (found_vector) { + const auto sub_word_index = first_field_marked(width, found_vector); + return start + sub_word_index; + } + total_bit_count_left -= bit_count_pr_iteration; + start += field_count; + } + + // final subword, may be partial + if (total_bit_count_left) { + // limit lookahead to avoid touching memory beyond array + const auto word = it.consume(total_bit_count_left); + found_vector = vector_compare(MSBs, word, search_vector); + auto last_word_mask = 0xFFFFFFFFFFFFFFFFULL >> (64 - total_bit_count_left); + found_vector &= last_word_mask; + if (found_vector) { + const auto sub_word_index = first_field_marked(width, found_vector); + return start + sub_word_index; + } + } + return end; +} + + +namespace impl { + +template +inline int64_t default_fetcher(const char* data, size_t ndx) +{ + return get_direct(data, ndx); +} + +template +struct CompressedDataFetcher { + + int64_t operator()(const char*, size_t ndx) const + { + return ptr->get(ndx); + } + const T* ptr; +}; + +// Lower and Upper bound are mainly used in the B+tree implementation, +// but also for indexing, we can exploit these functions when the array +// is encoded, just providing a way for fetching the data. +// In this case the width is going to be ignored. // Lower/upper bound in sorted sequence // ------------------------------------ @@ -222,8 +680,9 @@ inline std::pair get_two(const char* data, size_t width, size_ // // We currently use binary search. See for example // http://www.tbray.org/ongoing/When/200x/2003/03/22/Binary. -template -inline size_t lower_bound(const char* data, size_t size, int64_t value) noexcept +template +inline size_t lower_bound(const char* data, size_t start, size_t end, int64_t value, + F fetcher = default_fetcher) noexcept { // The binary search used here is carefully optimized. Key trick is to use a single // loop controlling variable (size) instead of high/low pair, and to keep updates @@ -233,7 +692,11 @@ inline size_t lower_bound(const char* data, size_t size, int64_t value) noexcept // might be slightly faster if we used branches instead. The loop unrolling yields // a final 5-20% speedup depending on circumstances. - size_t low = 0; + // size_t low = 0; + REALM_ASSERT_DEBUG(end >= start); + size_t size = end - start; + // size_t low = 0; + size_t low = start; while (size >= 8) { // The following code (at X, Y and Z) is 3 times manually unrolled instances of (A) below. @@ -244,7 +707,7 @@ inline size_t lower_bound(const char* data, size_t size, int64_t value) noexcept size_t other_half = size - half; size_t probe = low + half; size_t other_low = low + other_half; - int64_t v = get_direct(data, probe); + int64_t v = fetcher(data, probe); size = half; low = (v < value) ? other_low : low; @@ -253,7 +716,7 @@ inline size_t lower_bound(const char* data, size_t size, int64_t value) noexcept other_half = size - half; probe = low + half; other_low = low + other_half; - v = get_direct(data, probe); + v = fetcher(data, probe); size = half; low = (v < value) ? other_low : low; @@ -262,7 +725,7 @@ inline size_t lower_bound(const char* data, size_t size, int64_t value) noexcept other_half = size - half; probe = low + half; other_low = low + other_half; - v = get_direct(data, probe); + v = fetcher(data, probe); size = half; low = (v < value) ? other_low : low; } @@ -295,7 +758,7 @@ inline size_t lower_bound(const char* data, size_t size, int64_t value) noexcept size_t other_half = size - half; size_t probe = low + half; size_t other_low = low + other_half; - int64_t v = get_direct(data, probe); + int64_t v = fetcher(data, probe); size = half; // for max performance, the line below should compile into a conditional // move instruction. Not all compilers do this. To maximize chance @@ -308,16 +771,20 @@ inline size_t lower_bound(const char* data, size_t size, int64_t value) noexcept } // See lower_bound() -template -inline size_t upper_bound(const char* data, size_t size, int64_t value) noexcept +template +inline size_t upper_bound(const char* data, size_t start, size_t end, int64_t value, + F fetcher = default_fetcher) noexcept { - size_t low = 0; + REALM_ASSERT_DEBUG(end >= start); + size_t size = end - start; + // size_t low = 0; + size_t low = start; while (size >= 8) { size_t half = size / 2; size_t other_half = size - half; size_t probe = low + half; size_t other_low = low + other_half; - int64_t v = get_direct(data, probe); + int64_t v = fetcher(data, probe); size = half; low = (value >= v) ? other_low : low; @@ -325,7 +792,7 @@ inline size_t upper_bound(const char* data, size_t size, int64_t value) noexcept other_half = size - half; probe = low + half; other_low = low + other_half; - v = get_direct(data, probe); + v = fetcher(data, probe); size = half; low = (value >= v) ? other_low : low; @@ -333,7 +800,7 @@ inline size_t upper_bound(const char* data, size_t size, int64_t value) noexcept other_half = size - half; probe = low + half; other_low = low + other_half; - v = get_direct(data, probe); + v = fetcher(data, probe); size = half; low = (value >= v) ? other_low : low; } @@ -343,13 +810,41 @@ inline size_t upper_bound(const char* data, size_t size, int64_t value) noexcept size_t other_half = size - half; size_t probe = low + half; size_t other_low = low + other_half; - int64_t v = get_direct(data, probe); + int64_t v = fetcher(data, probe); size = half; low = (value >= v) ? other_low : low; }; return low; } +} // namespace impl + +template +inline size_t lower_bound(const char* data, size_t size, int64_t value) noexcept +{ + return impl::lower_bound(data, 0, size, value, impl::default_fetcher); +} + +template +inline size_t lower_bound(const char* data, size_t size, int64_t value, + const impl::CompressedDataFetcher& fetcher) noexcept +{ + return impl::lower_bound(data, 0, size, value, fetcher); +} + +template +inline size_t upper_bound(const char* data, size_t size, int64_t value) noexcept +{ + return impl::upper_bound(data, 0, size, value, impl::default_fetcher); +} + +template +inline size_t upper_bound(const char* data, size_t size, int64_t value, + const impl::CompressedDataFetcher& fetcher) noexcept +{ + return impl::upper_bound(data, 0, size, value, fetcher); +} + } // namespace realm #endif /* ARRAY_TPL_HPP_ */ diff --git a/src/realm/array_integer.cpp b/src/realm/array_integer.cpp index 51c4e665562..7cf99ff8dff 100644 --- a/src/realm/array_integer.cpp +++ b/src/realm/array_integer.cpp @@ -24,6 +24,12 @@ using namespace realm; +ArrayInteger::ArrayInteger(Allocator& allocator) noexcept + : Array(allocator) +{ + m_is_inner_bptree_node = false; +} + Mixed ArrayInteger::get_any(size_t ndx) const { return Mixed(get(ndx)); @@ -125,7 +131,6 @@ void ArrayIntNull::replace_nulls_with(int64_t new_null) } } - void ArrayIntNull::avoid_null_collision(int64_t value) { if (m_width == 64) { diff --git a/src/realm/array_integer.hpp b/src/realm/array_integer.hpp index 77fa59779d3..9c31e5f6bf9 100644 --- a/src/realm/array_integer.hpp +++ b/src/realm/array_integer.hpp @@ -29,16 +29,10 @@ namespace realm { class ArrayInteger : public Array, public ArrayPayload { public: using value_type = int64_t; - - using Array::add; using Array::find_first; - using Array::get; - using Array::insert; - using Array::move; - using Array::set; explicit ArrayInteger(Allocator&) noexcept; - ~ArrayInteger() noexcept override {} + ~ArrayInteger() noexcept override = default; static value_type default_value(bool) { @@ -70,8 +64,15 @@ class ArrayInteger : public Array, public ArrayPayload { } template bool find(value_type value, size_t start, size_t end, QueryStateBase* state) const; - size_t find_first_in_range(int64_t from, int64_t to, size_t start, size_t end) const; + + template + static ref_type typed_write(ref_type ref, T& out, Allocator& alloc) + { + Array arr(alloc); + arr.init_from_ref(ref); + return arr.write(out, false, out.only_modified, out.compress); + } }; class ArrayIntNull : public Array, public ArrayPayload { @@ -142,6 +143,14 @@ class ArrayIntNull : public Array, public ArrayPayload { size_t find_first(value_type value, size_t begin = 0, size_t end = npos) const; size_t find_first_in_range(int64_t from, int64_t to, size_t start, size_t end) const; + template + static ref_type typed_write(ref_type ref, T& out, Allocator& alloc) + { + Array arr(alloc); + arr.init_from_ref(ref); + return arr.write(out, false, out.only_modified, out.compress); + } + protected: void avoid_null_collision(int64_t value); @@ -158,12 +167,6 @@ class ArrayIntNull : public Array, public ArrayPayload { // Implementation: -inline ArrayInteger::ArrayInteger(Allocator& allocator) noexcept - : Array(allocator) -{ - m_is_inner_bptree_node = false; -} - inline ArrayIntNull::ArrayIntNull(Allocator& allocator) noexcept : Array(allocator) { diff --git a/src/realm/array_integer_tpl.hpp b/src/realm/array_integer_tpl.hpp index 5b97e257593..83f03f0e7d5 100644 --- a/src/realm/array_integer_tpl.hpp +++ b/src/realm/array_integer_tpl.hpp @@ -27,9 +27,10 @@ namespace realm { template bool ArrayInteger::find(value_type value, size_t start, size_t end, QueryStateBase* state) const { - return ArrayWithFind(*this).find(value, start, end, 0, state); + return Array::find(value, start, end, 0, state); } + inline bool ArrayIntNull::find_impl(int cond, value_type value, size_t start, size_t end, QueryStateBase* state) const { switch (cond) { @@ -74,9 +75,7 @@ bool ArrayIntNull::find_impl(value_type opt_value, size_t start, size_t end, Que value = *opt_value; } } - - // Fall back to plain Array find. - return ArrayWithFind(*this).find(value, start2, end2, baseindex2, state); + return Array::find(value, start2, end2, baseindex2, state); } else { cond c; diff --git a/src/realm/array_mixed.cpp b/src/realm/array_mixed.cpp index 6b07f423b06..3f65498c844 100644 --- a/src/realm/array_mixed.cpp +++ b/src/realm/array_mixed.cpp @@ -18,6 +18,8 @@ #include #include +#include +#include using namespace realm; @@ -275,6 +277,7 @@ size_t ArrayMixed::find_first(Mixed value, size_t begin, size_t end) const noexc DataType type = value.get_type(); if (end == realm::npos) end = size(); + for (size_t i = begin; i < end; i++) { if (Mixed::data_types_are_comparable(this->get_type(i), type) && get(i) == value) { return i; @@ -331,6 +334,89 @@ void ArrayMixed::verify() const // TODO: Implement } +ref_type ArrayMixed::typed_write(ref_type top_ref, _impl::ArrayWriterBase& out, Allocator& alloc) +{ + if (out.only_modified && alloc.is_read_only(top_ref)) + return top_ref; + + ArrayRef top(alloc); + top.init_from_ref(top_ref); + size_t sz = top.size(); + TempArray written_leaf(sz); + + /* + Mixed stores things using different arrays. We need to take into account this in order to + understand what we need to compress and what we can instead leave not compressed. + + The main subarrays are: + + composite array : index 0 + int array : index 1 + pair_int array: index 2 + string array: index 3 + ref array: index 4 + key array: index 5 + + Description of each array: + 1. composite array: the data stored here is either a small int (< 32 bits) or an offset to one of + the other arrays where the actual data is. + 2. int and pair int arrays, they are used for storing integers, timestamps, floats, doubles, + decimals, links. In general we can compress them, but we need to be careful, controlling the col_type + should prevent compressing data that we want to leave in the current format. + 3. string array is for strings and binary data (no compression for now) + 4. ref array is actually storing refs to collections. they can only be BPlusTree or + BPlusTree. + 5. key array stores unique identifiers for collections in mixed (integers that can be compressed) + */ + Array composite(alloc); + composite.init_from_ref(top.get_as_ref(0)); + written_leaf.set_as_ref(0, composite.write(out, true, out.only_modified, false)); + for (size_t i = 1; i < sz; ++i) { + auto ref = top.get(i); + ref_type new_ref = ref; + if (ref && !(out.only_modified && alloc.is_read_only(ref))) { + if (i < 3) { // int, and pair_int + // integer arrays + new_ref = Array::write(ref, alloc, out, out.only_modified, out.compress); + } + else if (i == 4) { // collection in mixed + ArrayRef arr_ref(alloc); + arr_ref.init_from_ref(ref); + auto ref_sz = arr_ref.size(); + TempArray written_ref_leaf(ref_sz); + + for (size_t k = 0; k < ref_sz; k++) { + ref_type new_sub_ref = 0; + if (auto sub_ref = arr_ref.get(k)) { + auto header = alloc.translate(sub_ref); + // Now we have to find out if the nested collection is a + // dictionary or a list. If the top array has a size of 2 + // and it is not a BplusTree inner node, then it is a dictionary + if (NodeHeader::get_size_from_header(header) == 2 && + !NodeHeader::get_is_inner_bptree_node_from_header(header)) { + new_sub_ref = Dictionary::typed_write(sub_ref, out, alloc); + } + else { + new_sub_ref = BPlusTree::typed_write(sub_ref, out, alloc); + } + } + written_ref_leaf.set_as_ref(k, new_sub_ref); + } + new_ref = written_ref_leaf.write(out); + } + else if (i == 5) { // unique keys associated to collections in mixed + new_ref = Array::write(ref, alloc, out, out.only_modified, out.compress); + } + else { + // all the rest we don't want to compress it, at least for now (strings will be needed) + new_ref = Array::write(ref, alloc, out, out.only_modified, false); + } + } + written_leaf.set(i, new_ref); + } + return written_leaf.write(out); +} + void ArrayMixed::ensure_array_accessor(Array& arr, size_t ndx_in_parent) const { if (!arr.is_attached()) { diff --git a/src/realm/array_mixed.hpp b/src/realm/array_mixed.hpp index ab71e254c63..a0de93b8339 100644 --- a/src/realm/array_mixed.hpp +++ b/src/realm/array_mixed.hpp @@ -100,6 +100,7 @@ class ArrayMixed : public ArrayPayload, private Array { int64_t get_key(size_t ndx) const; void verify() const; + static ref_type typed_write(ref_type ref, _impl::ArrayWriterBase& outs, Allocator& alloc); private: enum { diff --git a/src/realm/array_timestamp.cpp b/src/realm/array_timestamp.cpp index dba68a185d2..62dbc0879e0 100644 --- a/src/realm/array_timestamp.cpp +++ b/src/realm/array_timestamp.cpp @@ -18,6 +18,7 @@ #include #include +#include using namespace realm; @@ -261,4 +262,25 @@ void ArrayTimestamp::verify() const REALM_ASSERT(m_seconds.size() == m_nanoseconds.size()); #endif } + +ref_type ArrayTimestamp::typed_write(ref_type ref, _impl::ArrayWriterBase& out, Allocator& alloc) +{ + // timestamps could be compressed, but the formats we support at the moment are not producing + // noticeable gains. + Array top(alloc); + top.init_from_ref(ref); + REALM_ASSERT_DEBUG(top.size() == 2); + + TempArray written_top(2); + + auto rot0 = top.get_as_ref_or_tagged(0); + auto rot1 = top.get_as_ref_or_tagged(1); + REALM_ASSERT_DEBUG(rot0.is_ref() && rot0.get_as_ref()); + REALM_ASSERT_DEBUG(rot1.is_ref() && rot1.get_as_ref()); + written_top.set_as_ref(0, Array::write(rot0.get_as_ref(), alloc, out, out.only_modified, false)); + written_top.set_as_ref(1, Array::write(rot1.get_as_ref(), alloc, out, out.only_modified, false)); + + return written_top.write(out); +} + } // namespace realm diff --git a/src/realm/array_timestamp.hpp b/src/realm/array_timestamp.hpp index 385f1b1e530..e2982934080 100644 --- a/src/realm/array_timestamp.hpp +++ b/src/realm/array_timestamp.hpp @@ -109,6 +109,7 @@ class ArrayTimestamp : public ArrayPayload, private Array { size_t find_first_in_range(Timestamp from, Timestamp to, size_t start, size_t end) const; void verify() const; + static ref_type typed_write(ref_type ref, _impl::ArrayWriterBase& out, Allocator& alloc); private: ArrayIntNull m_seconds; diff --git a/src/realm/array_unsigned.cpp b/src/realm/array_unsigned.cpp index e1aac8dbf80..938fe5aece8 100644 --- a/src/realm/array_unsigned.cpp +++ b/src/realm/array_unsigned.cpp @@ -71,6 +71,7 @@ inline uint64_t ArrayUnsigned::_get(size_t ndx, uint8_t width) const return reinterpret_cast(m_data)[ndx]; } return get_direct(m_data, width, ndx); + REALM_UNREACHABLE(); } void ArrayUnsigned::create(size_t initial_size, uint64_t ubound_value) @@ -168,7 +169,8 @@ size_t ArrayUnsigned::upper_bound(uint64_t value) const noexcept void ArrayUnsigned::insert(size_t ndx, uint64_t value) { REALM_ASSERT_DEBUG(m_width >= 8); - bool do_expand = value > m_ubound; + + bool do_expand = value > (uint64_t)m_ubound; const uint8_t old_width = m_width; const uint8_t new_width = do_expand ? bit_width(value) : m_width; const auto old_size = m_size; @@ -215,6 +217,7 @@ void ArrayUnsigned::insert(size_t ndx, uint64_t value) void ArrayUnsigned::erase(size_t ndx) { REALM_ASSERT_DEBUG(m_width >= 8); + copy_on_write(); // Throws size_t w = m_width >> 3; diff --git a/src/realm/array_unsigned.hpp b/src/realm/array_unsigned.hpp index f1926ec7fc0..3e13b35e8dd 100644 --- a/src/realm/array_unsigned.hpp +++ b/src/realm/array_unsigned.hpp @@ -19,7 +19,7 @@ #ifndef REALM_ARRAY_UNSIGNED_HPP #define REALM_ARRAY_UNSIGNED_HPP -#include +#include namespace realm { @@ -81,13 +81,13 @@ class ArrayUnsigned : public Node { } private: - uint_least8_t m_width = 0; // Size of an element (meaning depend on type of array). - uint64_t m_ubound; // max number that can be stored with current m_width + uint_least8_t m_width = 0; + uint64_t m_ubound = 0; // max is 0xFFFFFFFFFFFFFFFFLL void init_from_mem(MemRef mem) noexcept { - Node::init_from_mem(mem); - set_width(get_width_from_header(get_header())); + auto header = Node::init_from_mem(mem); + set_width(get_width_from_header(header)); } void adjust(size_t ndx, int64_t diff) diff --git a/src/realm/array_with_find.cpp b/src/realm/array_with_find.cpp index e33513ef28e..2cf528a5c47 100644 --- a/src/realm/array_with_find.cpp +++ b/src/realm/array_with_find.cpp @@ -34,32 +34,6 @@ void ArrayWithFind::find_all(IntegerColumn* result, int64_t value, size_t col_of return; } - -bool ArrayWithFind::find(int cond, int64_t value, size_t start, size_t end, size_t baseindex, - QueryStateBase* state) const -{ - if (cond == cond_Equal) { - return find(value, start, end, baseindex, state); - } - if (cond == cond_NotEqual) { - return find(value, start, end, baseindex, state); - } - if (cond == cond_Greater) { - return find(value, start, end, baseindex, state); - } - if (cond == cond_Less) { - return find(value, start, end, baseindex, state); - } - if (cond == cond_None) { - return find(value, start, end, baseindex, state); - } - else if (cond == cond_LeftNotNull) { - return find(value, start, end, baseindex, state); - } - REALM_ASSERT_DEBUG(false); - return false; -} - size_t ArrayWithFind::first_set_bit(uint32_t v) const { // (v & -v) is UB when v is INT_MIN @@ -79,5 +53,15 @@ size_t ArrayWithFind::first_set_bit64(int64_t v) const return first_set_bit(v1) + 32; } +bool ArrayWithFind::find_all_will_match(size_t start2, size_t end, size_t baseindex, QueryStateBase* state) const +{ + REALM_ASSERT_DEBUG(state->match_count() < state->limit()); + size_t process = state->limit() - state->match_count(); + size_t end2 = end - start2 > process ? start2 + process : end; + for (; start2 < end2; start2++) + if (!state->match(start2 + baseindex)) + return false; + return true; +} } // namespace realm diff --git a/src/realm/array_with_find.hpp b/src/realm/array_with_find.hpp index 81d86d47e44..b35ed85e808 100644 --- a/src/realm/array_with_find.hpp +++ b/src/realm/array_with_find.hpp @@ -89,8 +89,6 @@ class ArrayWithFind { } // Main finding function - used for find_first, find_all, sum, max, min, etc. - bool find(int cond, int64_t value, size_t start, size_t end, size_t baseindex, QueryStateBase* state) const; - template bool find(int64_t value, size_t start, size_t end, size_t baseindex, QueryStateBase* state) const; @@ -161,7 +159,6 @@ class ArrayWithFind { private: const Array& m_array; - template bool find_all_will_match(size_t start, size_t end, size_t baseindex, QueryStateBase* state) const; }; //************************************************************************************* @@ -276,19 +273,6 @@ uint64_t ArrayWithFind::cascade(uint64_t a) const } } -template -REALM_NOINLINE bool ArrayWithFind::find_all_will_match(size_t start2, size_t end, size_t baseindex, - QueryStateBase* state) const -{ - REALM_ASSERT_DEBUG(state->match_count() < state->limit()); - size_t process = state->limit() - state->match_count(); - size_t end2 = end - start2 > process ? start2 + process : end; - for (; start2 < end2; start2++) - if (!state->match(start2 + baseindex)) - return false; - return true; -} - // This is the main finding function for Array. Other finding functions are just // wrappers around this one. Search for 'value' using condition cond (Equal, // NotEqual, Less, etc) and call QueryStateBase::match() for each match. Break and @@ -318,7 +302,7 @@ bool ArrayWithFind::find_optimized(int64_t value, size_t start, size_t end, size // optimization if all items are guaranteed to match (such as cond == NotEqual && value == 100 && m_ubound == 15) if (c.will_match(value, lbound, ubound)) { - return find_all_will_match(start2, end, baseindex, state); + return find_all_will_match(start2, end, baseindex, state); } // finder cannot handle this bitwidth @@ -567,14 +551,18 @@ inline bool ArrayWithFind::compare_equality(int64_t value, size_t start, size_t QueryStateBase* state) const { REALM_ASSERT_DEBUG(start <= m_array.m_size && (end <= m_array.m_size || end == size_t(-1)) && start <= end); + REALM_ASSERT_DEBUG(width == m_array.m_width); - size_t ee = round_up(start, 64 / no0(width)); + auto v = 64 / no0(width); + size_t ee = round_up(start, v); ee = ee > end ? end : ee; - for (; start < ee; ++start) - if (eq ? (m_array.get(start) == value) : (m_array.get(start) != value)) { + for (; start < ee; ++start) { + auto v = Array::get(m_array, start); + if (eq ? (v == value) : (v != value)) { if (!state->match(start + baseindex)) return false; } + } if (start >= end) return true; @@ -624,7 +612,7 @@ inline bool ArrayWithFind::compare_equality(int64_t value, size_t start, size_t } while (start < end) { - if (eq ? m_array.get(start) == value : m_array.get(start) != value) { + if (eq ? Array::get(m_array, start) == value : Array::get(m_array, start) != value) { if (!state->match(start + baseindex)) { return false; } @@ -903,8 +891,8 @@ bool ArrayWithFind::compare_relation(int64_t value, size_t start, size_t end, si size_t ee = round_up(start, 64 / no0(bitwidth)); ee = ee > end ? end : ee; for (; start < ee; start++) { - if (gt ? (m_array.get(start) > value) : (m_array.get(start) < value)) { - if (!state->match(start + baseindex, m_array.get(start))) + if (gt ? (Array::get(m_array, start) > value) : (Array::get(m_array, start) < value)) { + if (!state->match(start + baseindex, Array::get(m_array, start))) return false; } } @@ -969,7 +957,7 @@ bool ArrayWithFind::compare_relation(int64_t value, size_t start, size_t end, si // Test unaligned end and/or values of width > 16 manually while (start < end) { - if (gt ? m_array.get(start) > value : m_array.get(start) < value) { + if (gt ? Array::get(m_array, start) > value : Array::get(m_array, start) < value) { if (!state->match(start + baseindex)) return false; } diff --git a/src/realm/backup_restore.cpp b/src/realm/backup_restore.cpp index 8070642af0d..164ece35907 100644 --- a/src/realm/backup_restore.cpp +++ b/src/realm/backup_restore.cpp @@ -34,13 +34,14 @@ using VersionList = BackupHandler::VersionList; using VersionTimeList = BackupHandler::VersionTimeList; // Note: accepted versions should have new versions added at front -const VersionList BackupHandler::accepted_versions_ = {24, 23, 22, 21, 20, 11, 10}; +const VersionList BackupHandler::accepted_versions_ = {25, 24, 23, 22, 21, 20, 11, 10}; // the pair is // we keep backup files in 3 months. static constexpr int three_months = 3 * 31 * 24 * 60 * 60; -const VersionTimeList BackupHandler::delete_versions_{{23, three_months}, {22, three_months}, {21, three_months}, - {20, three_months}, {11, three_months}, {10, three_months}}; +const VersionTimeList BackupHandler::delete_versions_{{24, three_months}, {23, three_months}, {22, three_months}, + {21, three_months}, {20, three_months}, {11, three_months}, + {10, three_months}}; // helper functions diff --git a/src/realm/bplustree.cpp b/src/realm/bplustree.cpp index da25593abe9..02ea994b303 100644 --- a/src/realm/bplustree.cpp +++ b/src/realm/bplustree.cpp @@ -20,6 +20,7 @@ #include #include #include +#include using namespace realm; @@ -831,6 +832,41 @@ std::unique_ptr BPlusTreeBase::create_root_from_ref(ref_type ref) } } +// this should only be called for a column_type which we can safely compress. +ref_type BPlusTreeBase::typed_write(ref_type ref, _impl::ArrayWriterBase& out, Allocator& alloc, + TypedWriteFunc leaf_write_func) +{ + if (out.only_modified && alloc.is_read_only(ref)) + return ref; + + if (!NodeHeader::get_is_inner_bptree_node_from_header(alloc.translate(ref))) { + // leaf + return leaf_write_func(ref, out, alloc); + } + + Array node(alloc); + node.init_from_ref(ref); + REALM_ASSERT_DEBUG(node.has_refs()); + TempArray written_node(node.size(), NodeHeader::type_InnerBptreeNode, node.get_context_flag()); + for (unsigned j = 0; j < node.size(); ++j) { + RefOrTagged rot = node.get_as_ref_or_tagged(j); + if (rot.is_ref() && rot.get_as_ref()) { + if (j == 0) { + // keys (ArrayUnsigned me thinks) + Array a(alloc); + a.init_from_ref(rot.get_as_ref()); + written_node.set_as_ref(j, a.write(out, false, out.only_modified, false)); + } + else { + written_node.set_as_ref(j, BPlusTreeBase::typed_write(rot.get_as_ref(), out, alloc, leaf_write_func)); + } + } + else + written_node.set(j, rot); + } + return written_node.write(out); +} + size_t BPlusTreeBase::size_from_header(const char* header) { auto node_size = Array::get_size_from_header(header); diff --git a/src/realm/bplustree.hpp b/src/realm/bplustree.hpp index 6db1ffafc34..5f763892b7a 100644 --- a/src/realm/bplustree.hpp +++ b/src/realm/bplustree.hpp @@ -126,6 +126,8 @@ class BPlusTreeLeaf : public BPlusTreeNode { /*****************************************************************************/ class BPlusTreeBase { public: + using TypedWriteFunc = ref_type (*)(ref_type, _impl::ArrayWriterBase&, Allocator&); + BPlusTreeBase(Allocator& alloc) : m_alloc(alloc) { @@ -216,6 +218,8 @@ class BPlusTreeBase { m_root->verify(); } + static ref_type typed_write(ref_type, _impl::ArrayWriterBase&, Allocator&, TypedWriteFunc); + protected: template struct LeafTypeTrait { @@ -556,6 +560,11 @@ class BPlusTree : public BPlusTreeBase { } } + static ref_type typed_write(ref_type ref, _impl::ArrayWriterBase& out, Allocator& alloc) + { + return BPlusTreeBase::typed_write(ref, out, alloc, LeafArray::typed_write); + } + protected: LeafNode m_leaf_cache; @@ -685,6 +694,7 @@ ColumnAverageType bptree_average(const BPlusTree& tree, size_t* return_cnt *return_cnt = cnt; return avg; } + } // namespace realm #endif /* REALM_BPLUSTREE_HPP */ diff --git a/src/realm/cluster.cpp b/src/realm/cluster.cpp index 69f22925543..1734993ebce 100644 --- a/src/realm/cluster.cpp +++ b/src/realm/cluster.cpp @@ -1556,4 +1556,161 @@ void Cluster::remove_backlinks(const Table* origin_table, ObjKey origin_key, Col } } +namespace { + +template +static void switch_on_type(ColKey ck, Fn&& fn) +{ + bool is_optional = ck.is_nullable(); + auto type = ck.get_type(); + switch (type) { + case col_type_Int: + return is_optional ? fn((util::Optional*)0) : fn((int64_t*)0); + case col_type_Bool: + return is_optional ? fn((util::Optional*)0) : fn((bool*)0); + case col_type_Float: + return is_optional ? fn((util::Optional*)0) : fn((float*)0); + case col_type_Double: + return is_optional ? fn((util::Optional*)0) : fn((double*)0); + case col_type_String: + return fn((StringData*)0); + case col_type_Binary: + return fn((BinaryData*)0); + case col_type_Timestamp: + return fn((Timestamp*)0); + case col_type_Link: + return fn((ObjKey*)0); + case col_type_ObjectId: + return is_optional ? fn((util::Optional*)0) : fn((ObjectId*)0); + case col_type_Decimal: + return fn((Decimal128*)0); + case col_type_UUID: + return is_optional ? fn((util::Optional*)0) : fn((UUID*)0); + case col_type_Mixed: + return fn((Mixed*)0); + default: + REALM_COMPILER_HINT_UNREACHABLE(); + } +} + +} // namespace + +ref_type Cluster::typed_write(ref_type ref, _impl::ArrayWriterBase& out) const +{ + REALM_ASSERT_DEBUG(ref == get_mem().get_ref()); + bool only_modified = out.only_modified; + if (only_modified && m_alloc.is_read_only(ref)) + return ref; + REALM_ASSERT_DEBUG(!get_is_inner_bptree_node_from_header(get_header())); + REALM_ASSERT_DEBUG(!get_context_flag_from_header(get_header())); + TempArray written_cluster(size()); + for (size_t j = 0; j < size(); ++j) { + RefOrTagged leaf_rot = get_as_ref_or_tagged(j); + // Handle nulls + if (!leaf_rot.is_ref() || !leaf_rot.get_as_ref()) { + written_cluster.set(j, leaf_rot); + continue; + } + // prune subtrees which should not be written: + if (only_modified && m_alloc.is_read_only(leaf_rot.get_as_ref())) { + written_cluster.set(j, leaf_rot); + continue; + } + // from here: this leaf exists and needs to be written. + ref = leaf_rot.get_as_ref(); + ref_type new_ref = ref; + if (j == 0) { + // Keys (ArrayUnsigned me thinks, so don't compress) + Array leaf(m_alloc); + leaf.init_from_ref(ref); + new_ref = leaf.write(out, false, only_modified, false); + } + else { + // Columns + auto col_key = out.table->m_leaf_ndx2colkey[j - 1]; + auto col_type = col_key.get_type(); + if (col_key.is_collection()) { + ArrayRef arr_ref(m_alloc); + arr_ref.init_from_ref(ref); + auto sz = arr_ref.size(); + TempArray written_ref_leaf(sz); + + for (size_t k = 0; k < sz; k++) { + ref_type new_sub_ref = 0; + // Now we have to find out if the nested collection is a + // dictionary or a list. If the top array has a size of 2 + // and it is not a BplusTree inner node, then it is a dictionary + if (auto sub_ref = arr_ref.get(k)) { + if (col_key.is_dictionary()) { + new_sub_ref = Dictionary::typed_write(sub_ref, out, m_alloc); + } + else { + // List or set - Can be handled the same way + // For some reason, switch_on_type() would not compile on Windows + // switch_on_type(col_key, [&](auto t) { + // using U = std::decay_t; + // new_sub_ref = BPlusTree::typed_write(sub_ref, out, m_alloc); + // }); + switch (col_type) { + case col_type_Int: + new_sub_ref = BPlusTree::typed_write(sub_ref, out, m_alloc); + break; + case col_type_Bool: + new_sub_ref = BPlusTree::typed_write(sub_ref, out, m_alloc); + break; + case col_type_Float: + new_sub_ref = BPlusTree::typed_write(sub_ref, out, m_alloc); + break; + case col_type_Double: + new_sub_ref = BPlusTree::typed_write(sub_ref, out, m_alloc); + break; + case col_type_String: + new_sub_ref = BPlusTree::typed_write(sub_ref, out, m_alloc); + break; + case col_type_Binary: + new_sub_ref = BPlusTree::typed_write(sub_ref, out, m_alloc); + break; + case col_type_Timestamp: + new_sub_ref = BPlusTree::typed_write(sub_ref, out, m_alloc); + break; + case col_type_Link: + new_sub_ref = BPlusTree::typed_write(sub_ref, out, m_alloc); + break; + case col_type_ObjectId: + new_sub_ref = BPlusTree::typed_write(sub_ref, out, m_alloc); + break; + case col_type_Decimal: + new_sub_ref = BPlusTree::typed_write(sub_ref, out, m_alloc); + break; + case col_type_UUID: + new_sub_ref = BPlusTree::typed_write(sub_ref, out, m_alloc); + break; + case col_type_Mixed: + new_sub_ref = BPlusTree::typed_write(sub_ref, out, m_alloc); + break; + default: + REALM_COMPILER_HINT_UNREACHABLE(); + } + } + } + written_ref_leaf.set_as_ref(k, new_sub_ref); + } + new_ref = written_ref_leaf.write(out); + } + else if (col_type == col_type_BackLink) { + Array leaf(m_alloc); + leaf.init_from_ref(ref); + new_ref = leaf.write(out, true, only_modified, false); + } + else { + switch_on_type(col_key, [&](auto t) { + using U = std::decay_t; + new_ref = ColumnTypeTraits::cluster_leaf_type::typed_write(ref, out, m_alloc); + }); + } + } + written_cluster.set_as_ref(j, new_ref); + } + return written_cluster.write(out); +} } // namespace realm diff --git a/src/realm/cluster.hpp b/src/realm/cluster.hpp index 3d23db67ebc..564f3aa107e 100644 --- a/src/realm/cluster.hpp +++ b/src/realm/cluster.hpp @@ -212,6 +212,7 @@ class ClusterNode : public Array { { return m_offset; } + virtual ref_type typed_write(ref_type ref, _impl::ArrayWriterBase& out) const = 0; protected: #if REALM_MAX_BPNODE_SIZE > 256 @@ -228,7 +229,7 @@ class ClusterNode : public Array { uint64_t get(size_t ndx) const { - return (m_data != nullptr) ? ArrayUnsigned::get(ndx) : uint64_t(ndx); + return is_attached() ? ArrayUnsigned::get(ndx) : uint64_t(ndx); } }; @@ -320,6 +321,7 @@ class Cluster : public ClusterNode { void verify() const; void dump_objects(int64_t key_offset, std::string lead) const override; + virtual ref_type typed_write(ref_type ref, _impl::ArrayWriterBase& out) const override; static void remove_backlinks(const Table* origin_table, ObjKey origin_key, ColKey col, const std::vector& keys, CascadeState& state); static void remove_backlinks(const Table* origin_table, ObjKey origin_key, ColKey col, diff --git a/src/realm/cluster_tree.cpp b/src/realm/cluster_tree.cpp index 7cb1ee37cf2..9c6d7fb1fe7 100644 --- a/src/realm/cluster_tree.cpp +++ b/src/realm/cluster_tree.cpp @@ -120,6 +120,51 @@ class ClusterNodeInner : public ClusterNode { void dump_objects(int64_t key_offset, std::string lead) const override; + virtual ref_type typed_write(ref_type ref, _impl::ArrayWriterBase& out) const override + { + REALM_ASSERT_DEBUG(ref == get_mem().get_ref()); + if (out.only_modified && m_alloc.is_read_only(ref)) { + return ref; + } + REALM_ASSERT_DEBUG(get_is_inner_bptree_node_from_header(get_header())); + REALM_ASSERT_DEBUG(!get_context_flag_from_header(get_header())); + REALM_ASSERT_DEBUG(has_refs()); + TempArray written_node(size(), type_InnerBptreeNode); + for (unsigned j = 0; j < size(); ++j) { + RefOrTagged rot = get_as_ref_or_tagged(j); + if (rot.is_ref() && rot.get_as_ref()) { + if (out.only_modified && m_alloc.is_read_only(rot.get_as_ref())) { + written_node.set(j, rot); + continue; + } + if (j == 0) { + // keys (ArrayUnsigned, me thinks) + Array array_unsigned(m_alloc); + array_unsigned.init_from_ref(rot.get_as_ref()); + written_node.set_as_ref(j, array_unsigned.write(out, false, out.only_modified, false)); + } + else { + auto header = m_alloc.translate(rot.get_as_ref()); + MemRef m(header, rot.get_as_ref(), m_alloc); + if (get_is_inner_bptree_node_from_header(header)) { + ClusterNodeInner inner_node(m_alloc, m_tree_top); + inner_node.init(m); + written_node.set_as_ref(j, inner_node.typed_write(rot.get_as_ref(), out)); + } + else { + Cluster cluster(j, m_alloc, m_tree_top); + cluster.init(m); + written_node.set_as_ref(j, cluster.typed_write(rot.get_as_ref(), out)); + } + } + } + else { // not a ref, just copy value over + written_node.set(j, rot); + } + } + return written_node.write(out); + } + private: static constexpr size_t s_key_ref_index = 0; static constexpr size_t s_sub_tree_depth_index = 1; diff --git a/src/realm/cluster_tree.hpp b/src/realm/cluster_tree.hpp index ee1fc5ae478..1b0d05c759b 100644 --- a/src/realm/cluster_tree.hpp +++ b/src/realm/cluster_tree.hpp @@ -190,6 +190,12 @@ class ClusterTree { } void verify() const; + ref_type typed_write(ref_type ref, _impl::ArrayWriterBase& out) const + { + REALM_ASSERT_DEBUG(m_root); + return m_root->typed_write(ref, out); + } + protected: friend class Obj; friend class Cluster; diff --git a/src/realm/db.cpp b/src/realm/db.cpp index ba0fbb8000b..6198cac10ae 100644 --- a/src/realm/db.cpp +++ b/src/realm/db.cpp @@ -2559,6 +2559,10 @@ void DB::low_level_commit(uint_fast64_t new_version, Transaction& transaction, b // Add 4k to ensure progress on small commits size_t work_limit = commit_size / 2 + out.get_free_list_size() + 0x1000; transaction.cow_outliers(out.get_evacuation_progress(), limit, work_limit); + // moving blocks around may have left table accessors with stale data, + // and we need them to work later in the process when we determine which + // arrays to compress during writing, so make sure they're up to date: + transaction.update_table_accessors(); } ref_type new_top_ref; @@ -2757,7 +2761,7 @@ TransactionRef DB::start_write(bool nonblocking) end_write_on_correct_thread(); throw; } - + tr->update_allocator_wrappers(true); return tr; } @@ -2802,7 +2806,8 @@ void DB::async_request_write_mutex(TransactionRef& tr, util::UniqueFunction { SharedInfo* m_info = nullptr; bool m_wait_for_change_enabled = true; // Initially wait_for_change is enabled bool m_write_transaction_open GUARDED_BY(m_mutex) = false; + bool m_allow_flexible_schema; std::string m_db_path; int m_file_format_version = 0; util::InterprocessMutex m_writemutex; diff --git a/src/realm/db_options.hpp b/src/realm/db_options.hpp index a11eded2352..68e2c8098f3 100644 --- a/src/realm/db_options.hpp +++ b/src/realm/db_options.hpp @@ -106,6 +106,9 @@ struct DBOptions { /// will clear and reinitialize the file. bool clear_on_invalid_file = false; + /// Allow setting properties not supported by a specific column on an object + bool allow_flexible_schema = false; + /// sys_tmp_dir will be used if the temp_dir is empty when creating DBOptions. /// It must be writable and allowed to create pipe/fifo file on it. /// set_sys_tmp_dir is not a thread-safe call and it is only supposed to be called once diff --git a/src/realm/dictionary.cpp b/src/realm/dictionary.cpp index 147d0b95c25..63851d0e72b 100644 --- a/src/realm/dictionary.cpp +++ b/src/realm/dictionary.cpp @@ -1214,6 +1214,36 @@ LinkCollectionPtr Dictionary::clone_as_obj_list() const return nullptr; } +ref_type Dictionary::typed_write(ref_type ref, _impl::ArrayWriterBase& out, Allocator& alloc) +{ + if (out.only_modified && alloc.is_read_only(ref)) + return ref; + + ArrayRef dict_top(alloc); + dict_top.init_from_ref(ref); + REALM_ASSERT_DEBUG(dict_top.size() == 2); + TempArray written_dict_top(2); + + // We have to find out what kind of keys we are using - strings or ints + // Btw - ints is only used in tests. Can probably be removed at some point + auto key_ref = dict_top.get(0); + auto header = alloc.translate(key_ref); + if (!NodeHeader::get_hasrefs_from_header(header) && + NodeHeader::get_wtype_from_header(header) != Array::wtype_Multiply) { + // Key type int. + REALM_ASSERT(!NodeHeader::get_is_inner_bptree_node_from_header(header)); + written_dict_top.set_as_ref(0, BPlusTree::typed_write(key_ref, out, alloc)); + } + else { + written_dict_top.set_as_ref(0, BPlusTree::typed_write(key_ref, out, alloc)); + } + + auto values_ref = dict_top.get_as_ref(1); + written_dict_top.set_as_ref(1, BPlusTree::typed_write(values_ref, out, alloc)); + + return written_dict_top.write(out); +} + /************************* DictionaryLinkValues *************************/ DictionaryLinkValues::DictionaryLinkValues(const Obj& obj, ColKey col_key) diff --git a/src/realm/dictionary.hpp b/src/realm/dictionary.hpp index a01c7fd8b8e..be8765f5a8f 100644 --- a/src/realm/dictionary.hpp +++ b/src/realm/dictionary.hpp @@ -232,6 +232,8 @@ class Dictionary final : public CollectionBaseImpl, public Colle LinkCollectionPtr clone_as_obj_list() const final; + static ref_type typed_write(ref_type ref, _impl::ArrayWriterBase& out, Allocator& alloc); + private: using Base::set_collection; diff --git a/src/realm/exec/CMakeLists.txt b/src/realm/exec/CMakeLists.txt index 4897f62a52a..969b45a9e10 100644 --- a/src/realm/exec/CMakeLists.txt +++ b/src/realm/exec/CMakeLists.txt @@ -24,6 +24,28 @@ if (EMSCRIPTEN) set_target_properties(RealmTrawler PROPERTIES EXCLUDE_FROM_ALL TRUE) endif() +add_executable(ClickBench EXCLUDE_FROM_ALL clickbench.cpp ) +set_target_properties(ClickBench PROPERTIES + OUTPUT_NAME "clickbench" + DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX} +) +if (EMSCRIPTEN) + set_target_properties(ClickBench PROPERTIES EXCLUDE_FROM_ALL TRUE) +endif() + +target_link_libraries(ClickBench Storage) + +add_executable(ClickQuery EXCLUDE_FROM_ALL clickquery.cpp ) +set_target_properties(ClickQuery PROPERTIES + OUTPUT_NAME "clickquery" + DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX} +) +if (EMSCRIPTEN) + set_target_properties(ClickQuery PROPERTIES EXCLUDE_FROM_ALL TRUE) +endif() + +target_link_libraries(ClickQuery Storage) + add_executable(RealmEnumerate realm_enumerate.cpp) set_target_properties(RealmEnumerate PROPERTIES OUTPUT_NAME "realm-enumerate" diff --git a/src/realm/exec/clickbench.cpp b/src/realm/exec/clickbench.cpp new file mode 100644 index 00000000000..54c4b13e318 --- /dev/null +++ b/src/realm/exec/clickbench.cpp @@ -0,0 +1,375 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace realm; + +template +class Mailbox { +public: + void send(T* val) + { + std::unique_lock lck(m_mutex); + m_list.push_back(val); + m_cv.notify_one(); + } + T* receive() + { + std::unique_lock lck(m_mutex); + while (m_list.empty()) + m_cv.wait(lck); + T* ret = m_list.front(); + m_list.pop_front(); + return ret; + } + +private: + std::deque m_list; + std::mutex m_mutex; + std::condition_variable m_cv; +}; + +// remove this when enumerated strings are supported: +#define type_EnumString type_String + +static void create_table(TransactionRef tr) +{ + auto t = tr->add_table("Hits"); + t->add_column(type_Int, "WatchID"); + t->add_column(type_Int, "JavaEnable"); + t->add_column(type_EnumString, "Title", true); + t->add_column(type_Int, "GoodEvent"); + t->add_column(type_Timestamp, "EventTime"); + t->add_column(type_Timestamp, "EventDate"); + t->add_column(type_Int, "CounterID"); + t->add_column(type_Int, "ClientIP"); + t->add_column(type_Int, "RegionID"); + t->add_column(type_Int, "UserID"); + t->add_column(type_Int, "CounterClass"); + t->add_column(type_Int, "OS"); + t->add_column(type_Int, "UserAgent"); + t->add_column(type_EnumString, "URL", true); + t->add_column(type_EnumString, "Referer", true); + t->add_column(type_Int, "IsRefresh"); + t->add_column(type_Int, "RefererCategoryID"); + t->add_column(type_Int, "RefererRegionID"); + t->add_column(type_Int, "URLCategoryID"); + t->add_column(type_Int, "URLRegionID"); + t->add_column(type_Int, "ResolutionWidth"); + t->add_column(type_Int, "ResolutionHeight"); + t->add_column(type_Int, "ResolutionDepth"); + t->add_column(type_Int, "FlashMajor"); + t->add_column(type_Int, "FlashMinor"); + t->add_column(type_EnumString, "FlashMinor2", true); + t->add_column(type_Int, "NetMajor"); + t->add_column(type_Int, "NetMinor"); + t->add_column(type_Int, "UserAgentMajor"); + t->add_column(type_EnumString, "UserAgentMinor", true); + t->add_column(type_Int, "CookieEnable"); + t->add_column(type_Int, "JavascriptEnable"); + t->add_column(type_Int, "IsMobile"); + t->add_column(type_Int, "MobilePhone"); + t->add_column(type_EnumString, "MobilePhoneModel", true); + t->add_column(type_EnumString, "Params", true); + t->add_column(type_Int, "IPNetworkID"); + t->add_column(type_Int, "TraficSourceID"); + t->add_column(type_Int, "SearchEngineID"); + t->add_column(type_EnumString, "SearchPhrase", true); + t->add_column(type_Int, "AdvEngineID"); + t->add_column(type_Int, "IsArtifical"); + t->add_column(type_Int, "WindowClientWidth"); + t->add_column(type_Int, "WindowClientHeight"); + t->add_column(type_Int, "ClientTimeZone"); + t->add_column(type_Timestamp, "ClientEventTime"); + t->add_column(type_Int, "SilverlightVersion1"); + t->add_column(type_Int, "SilverlightVersion2"); + t->add_column(type_Int, "SilverlightVersion3"); + t->add_column(type_Int, "SilverlightVersion4"); + t->add_column(type_EnumString, "PageCharset", true); + t->add_column(type_Int, "CodeVersion"); + t->add_column(type_Int, "IsLink"); + t->add_column(type_Int, "IsDownload"); + t->add_column(type_Int, "IsNotBounce"); + t->add_column(type_Int, "FUniqID"); + t->add_column(type_EnumString, "OriginalURL", true); + t->add_column(type_Int, "HID"); + t->add_column(type_Int, "IsOldCounter"); + t->add_column(type_Int, "IsEvent"); + t->add_column(type_Int, "IsParameter"); + t->add_column(type_Int, "DontCountHits"); + t->add_column(type_Int, "WithHash"); + t->add_column(type_EnumString, "HitColor", true); + t->add_column(type_Timestamp, "LocalEventTime"); + t->add_column(type_Int, "Age"); + t->add_column(type_Int, "Sex"); + t->add_column(type_Int, "Income"); + t->add_column(type_Int, "Interests"); + t->add_column(type_Int, "Robotness"); + t->add_column(type_Int, "RemoteIP"); + t->add_column(type_Int, "WindowName"); + t->add_column(type_Int, "OpenerName"); + t->add_column(type_Int, "HistoryLength"); + t->add_column(type_EnumString, "BrowserLanguage", true); + t->add_column(type_EnumString, "BrowserCountry", true); + t->add_column(type_EnumString, "SocialNetwork", true); + t->add_column(type_EnumString, "SocialAction", true); + t->add_column(type_Int, "HTTPError"); + t->add_column(type_Int, "SendTiming"); + t->add_column(type_Int, "DNSTiming"); + t->add_column(type_Int, "ConnectTiming"); + t->add_column(type_Int, "ResponseStartTiming"); + t->add_column(type_Int, "ResponseEndTiming"); + t->add_column(type_Int, "FetchTiming"); + t->add_column(type_Int, "SocialSourceNetworkID"); + t->add_column(type_EnumString, "SocialSourcePage", true); + t->add_column(type_Int, "ParamPrice"); + t->add_column(type_EnumString, "ParamOrderID", true); + t->add_column(type_EnumString, "ParamCurrency", true); + t->add_column(type_Int, "ParamCurrencyID"); + t->add_column(type_EnumString, "OpenstatServiceName", true); + t->add_column(type_EnumString, "OpenstatCampaignID", true); + t->add_column(type_EnumString, "OpenstatAdID", true); + t->add_column(type_EnumString, "OpenstatSourceID", true); + t->add_column(type_EnumString, "UTMSource", true); + t->add_column(type_EnumString, "UTMMedium", true); + t->add_column(type_EnumString, "UTMCampaign", true); + t->add_column(type_EnumString, "UTMContent", true); + t->add_column(type_EnumString, "UTMTerm", true); + t->add_column(type_EnumString, "FromTag", true); + t->add_column(type_Int, "HasGCLID"); + t->add_column(type_Int, "RefererHash"); + t->add_column(type_Int, "URLHash"); + t->add_column(type_Int, "CLID"); + tr->commit(); +} + +static int strtoi(const char* p, char** endp) +{ + int ret = 0; + while (*p >= '0' && *p <= '9') { + ret *= 10; + ret += *p - '0'; + ++p; + } + *endp = const_cast(p); + return ret; +} + +inline int64_t epoch_days_fast(int y, int m, int d) +{ + const uint32_t year_base = 4800; /* Before min year, multiple of 400. */ + const int32_t m_adj = m - 3; /* March-based month. */ + const uint32_t carry = (m_adj > m) ? 1 : 0; + const uint32_t adjust = carry ? 12 : 0; + const uint32_t y_adj = y + year_base - carry; + const uint32_t month_days = ((m_adj + adjust) * 62719 + 769) / 2048; + const uint32_t leap_days = y_adj / 4 - y_adj / 100 + y_adj / 400; + return y_adj * 365 + leap_days + month_days + (d - 1) - 2472632; +} + +static Timestamp get_timestamp(const char* str) +{ + char* p; + int year = int(strtoi(str, &p)); + if (*p == '-') { + p++; + int mon = int(strtoi(p, &p)); + if (*p == '-') { + p++; + int day = int(strtoi(p, &p)); + time_t hms = 0; + if (*p == ' ' || *p == 'T') { + p++; + int h = int(strtoi(p, &p)); + int m = 0; + int s = 0; + if (*p == ':') { + p++; + m = int(strtoi(p, &p)); + if (*p == ':') { + p++; + s = int(strtoi(p, &p)); + } + } + hms = (h * 3600) + (m * 60) + s; + } + if (*p == '\0') { + return Timestamp(epoch_days_fast(year, mon, day) * 86400 + hms, 0); + } + } + } + return Timestamp(); +} + +struct BufferedValues { + std::vector buffer{256}; + std::vector values{256}; +}; + +Mailbox mbx; +Mailbox resp; + +static void parse_file(const char* filename) +{ + std::ifstream inp(filename); + + auto buf = resp.receive(); + auto str = buf->buffer.begin(); + auto it = buf->values.begin(); + auto end = buf->values.end(); + while (std::getline(inp, *str)) { + char* tok = str->data(); + for (FieldValue& val : *it) { + char* end = strchr(tok, '\t'); + if (end) { + *end = '\0'; + } + switch (val.col_key.get_type()) { + case col_type_Int: + val.value = Mixed(int64_t(strtoll(tok, nullptr, 10))); + break; + case col_type_String: + val.value = strlen(tok) ? Mixed(tok) : Mixed(); + break; + /* + case col_type_EnumString: + val.value = strlen(tok) ? Mixed(tok) : Mixed(); + break; + */ + case col_type_Timestamp: + val.value = Mixed(get_timestamp(tok)); + break; + default: + break; + } + tok = end + 1; + } + ++it; + ++str; + if (it == end) { + mbx.send(buf); + buf = resp.receive(); + str = buf->buffer.begin(); + it = buf->values.begin(); + end = buf->values.end(); + } + } + buf->values.erase(it, end); + if (buf->values.size()) { + mbx.send(buf); + } + mbx.send(nullptr); +} + +static void import(const char* filename) +{ + DBOptions options; + auto db = DB::create(make_in_realm_history(), "hits.realm"); + create_table(db->start_write()); + auto tr = db->start_write(); + auto t = tr->get_table("Hits"); + auto col_keys = t->get_column_keys(); + + std::cout << std::endl << "Reading data into realm" << std::endl; + auto time_start = std::chrono::high_resolution_clock::now(); + BufferedValues buf1; + BufferedValues buf2; + for (auto& val : buf1.values) { + for (auto col : col_keys) { + val.insert(col, Mixed()); + } + } + for (auto& val : buf2.values) { + for (auto col : col_keys) { + val.insert(col, Mixed()); + } + } + resp.send(&buf1); + resp.send(&buf2); + std::thread parse_file_thread(parse_file, filename); + + int buf_cnt = 0; + const int bufs_per_commit = 100; + while (auto buf = mbx.receive()) { + for (auto& val : buf->values) { + Obj o = t->create_object(ObjKey(), val); + // verify + for (auto& e : val) { + if (e.col_key.get_type() == col_type_Int) { + auto got_int = o.get(e.col_key); + auto the_int = e.value.get_int(); + REALM_ASSERT(got_int == the_int); + } + if (e.col_key.get_type() == col_type_String) { + auto got_string = o.get(e.col_key); + auto the_string = e.value.get_string(); + REALM_ASSERT(got_string == the_string); + } + } + } + resp.send(buf); + if (buf_cnt++ > bufs_per_commit) { + tr->commit_and_continue_as_read(); + tr->promote_to_write(); + std::cout << '.'; + std::cout.flush(); + buf_cnt = 0; + } + } + tr->commit_and_continue_as_read(); + + parse_file_thread.join(); + auto time_end = std::chrono::high_resolution_clock::now(); + std::cout << "Ingestion complete in " + << std::chrono::duration_cast(time_end - time_start).count() << " msecs" + << std::endl; +} + +static void dump_prop(const char* filename, const char* prop_name) +{ + auto db = DB::create(make_in_realm_history(), filename); + auto tr = db->start_read(); + auto t = tr->get_table("Hits"); + auto col = t->get_column_key(prop_name); + for (auto& o : *t) { + switch (col.get_type()) { + case col_type_Int: + std::cout << o.get(col) << std::endl; + break; + case col_type_String: + std::cout << o.get(col) << std::endl; + break; + /* + case col_type_EnumString: + REALM_ASSERT(false); + break; + */ + case col_type_Timestamp: + std::cout << o.get(col) << std::endl; + break; + default: + break; + } + } +} + +int main(int argc, const char* argv[]) +{ + if (argc == 1) { + import("mill.tsv"); + } + if (argc == 2) { + import(argv[1]); + } + if (argc == 3) { + dump_prop(argv[1], argv[2]); + } +} diff --git a/src/realm/exec/clickquery.cpp b/src/realm/exec/clickquery.cpp new file mode 100644 index 00000000000..e930eb27013 --- /dev/null +++ b/src/realm/exec/clickquery.cpp @@ -0,0 +1,86 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace realm; + +static void import(const char* filename) +{ + DBOptions options; + auto db = DB::create(make_in_realm_history(), filename); + auto tr = db->start_write(); + auto t = tr->get_table("Hits"); + auto time_start = std::chrono::high_resolution_clock::now(); + auto time_end = time_start; + { + std::cout << std::endl << "count of AdvEngineID <> 0" << std::endl; + time_start = std::chrono::high_resolution_clock::now(); + size_t q = 0; + for (int i = 0; i < 10; ++i) { + auto k = t->get_column_key("AdvEngineID"); + q = t->where().not_equal(k, 0).count(); + } + time_end = std::chrono::high_resolution_clock::now(); + std::cout << "result = " << q << " in " + << std::chrono::duration_cast(time_end - time_start).count() << " msecs" + << std::endl; + } + { + std::cout << std::endl << "Query result for AdvEngineID <> 0" << std::endl; + time_start = std::chrono::high_resolution_clock::now(); + TableView q; + for (int i = 0; i < 10; ++i) { + auto k = t->get_column_key("AdvEngineID"); + q = t->where().not_equal(k, 0).find_all(); + } + time_end = std::chrono::high_resolution_clock::now(); + std::cout << "result with size " << q.size() << " in " + << std::chrono::duration_cast(time_end - time_start).count() << " msecs" + << std::endl; + time_start = std::chrono::high_resolution_clock::now(); + size_t count = 0; + for (int i = 0; i < 10; ++i) { + auto limit = q.size(); + auto k = t->get_column_key("AdvEngineID"); + for (size_t i = 0; i < limit; ++i) { + count += q[i].get(k); + } + } + time_end = std::chrono::high_resolution_clock::now(); + std::cout << "Iterating over result to get count " << count << " in " + << std::chrono::duration_cast(time_end - time_start).count() << " msecs" + << std::endl; + } + { + std::cout << std::endl << "Max of EventDate" << std::endl; + time_start = std::chrono::high_resolution_clock::now(); + Mixed q; + for (int i = 0; i < 10; ++i) { + auto k = t->get_column_key("EventDate"); + q = *(t->max(k)); + } + // auto q = t->where().not_equal(k, 0).count(); + time_end = std::chrono::high_resolution_clock::now(); + std::cout << "result = " << q << " in " + << std::chrono::duration_cast(time_end - time_start).count() << " msecs" + << std::endl; + } +} + +int main(int argc, const char* argv[]) +{ + if (argc == 1) { + import("./hits.realm"); + } + if (argc == 2) { + import(argv[1]); + } +} diff --git a/src/realm/group.cpp b/src/realm/group.cpp index 1938b2e54fb..039a76ea535 100644 --- a/src/realm/group.cpp +++ b/src/realm/group.cpp @@ -69,12 +69,13 @@ Group::Group() } -Group::Group(const std::string& file_path, const char* encryption_key) +Group::Group(const std::string& file_path, const char* encryption_key, bool allow_additional_properties) : m_local_alloc(new SlabAlloc) // Throws , m_alloc(*m_local_alloc) , m_top(m_alloc) , m_tables(m_alloc) , m_table_names(m_alloc) + , m_allow_additional_properties(allow_additional_properties) { init_array_parents(); @@ -419,6 +420,7 @@ int Group::read_only_version_check(SlabAlloc& alloc, ref_type top_ref, const std case 0: file_format_ok = (top_ref == 0); break; + case 24: case g_current_file_format_version: file_format_ok = true; break; @@ -490,6 +492,19 @@ void Group::remap_and_update_refs(ref_type new_top_ref, size_t new_file_size, bo update_refs(new_top_ref); } +void Group::update_table_accessors() +{ + for (unsigned j = 0; j < m_table_accessors.size(); ++j) { + Table* table = m_table_accessors[j]; + // this should be filtered further as an optimization + if (table) { + table->refresh_allocator_wrapper(); + table->update_from_parent(); + } + } +} + + void Group::validate_top_array(const Array& arr, const SlabAlloc& alloc, std::optional read_lock_file_size, std::optional read_lock_version) { @@ -747,6 +762,10 @@ Table* Group::do_add_table(StringData name, Table::Type table_type, bool do_repl Table* table = create_table_accessor(j); table->do_set_table_type(table_type); + if (m_allow_additional_properties && name.begins_with(g_class_name_prefix)) { + table->do_add_additional_prop_column(); + } + return table; } @@ -925,23 +944,46 @@ void Group::validate(ObjLink link) const } } +ref_type Group::typed_write_tables(_impl::ArrayWriterBase& out) const +{ + ref_type ref = m_top.get_as_ref(1); + if (out.only_modified && m_alloc.is_read_only(ref)) + return ref; + Array a(m_alloc); + a.init_from_ref(ref); + REALM_ASSERT_DEBUG(a.has_refs()); + TempArray dest(a.size()); + for (unsigned j = 0; j < a.size(); ++j) { + RefOrTagged rot = a.get_as_ref_or_tagged(j); + if (rot.is_tagged()) { + dest.set(j, rot); + } + else { + auto table = do_get_table(j); + REALM_ASSERT_DEBUG(table); + dest.set_as_ref(j, table->typed_write(rot.get_as_ref(), out)); + } + } + return dest.write(out); +} + ref_type Group::DefaultTableWriter::write_names(_impl::OutputStream& out) { - bool deep = true; // Deep - bool only_if_modified = false; // Always - return m_group->m_table_names.write(out, deep, only_if_modified); // Throws + bool deep = true; // Deep + bool only_if_modified = false; // Always + bool compress = false; // true; + return m_group->m_table_names.write(out, deep, only_if_modified, compress); // Throws } ref_type Group::DefaultTableWriter::write_tables(_impl::OutputStream& out) { - bool deep = true; // Deep - bool only_if_modified = false; // Always - return m_group->m_tables.write(out, deep, only_if_modified); // Throws + return m_group->typed_write_tables(out); } auto Group::DefaultTableWriter::write_history(_impl::OutputStream& out) -> HistoryInfo { bool deep = true; // Deep bool only_if_modified = false; // Always + bool compress = false; ref_type history_ref = _impl::GroupFriend::get_history_ref(*m_group); HistoryInfo info; if (history_ref) { @@ -958,7 +1000,7 @@ auto Group::DefaultTableWriter::write_history(_impl::OutputStream& out) -> Histo info.version = history_schema_version; Array history{const_cast(_impl::GroupFriend::get_alloc(*m_group))}; history.init_from_ref(history_ref); - info.ref = history.write(out, deep, only_if_modified); // Throws + info.ref = history.write(out, deep, only_if_modified, compress); // Throws } info.sync_file_id = m_group->get_sync_file_id(); return info; @@ -1033,6 +1075,7 @@ void Group::write(std::ostream& out, int file_format_version, TableWriter& table bool pad_for_encryption, uint_fast64_t version_number) { _impl::OutputStream out_2(out); + out_2.only_modified = false; // Write the file header SlabAlloc::Header streaming_header; @@ -1063,7 +1106,8 @@ void Group::write(std::ostream& out, int file_format_version, TableWriter& table // DB to compact the database by writing only the live data // into a separate file. ref_type names_ref = table_writer.write_names(out_2); // Throws - ref_type tables_ref = table_writer.write_tables(out_2); // Throws + ref_type tables_ref = table_writer.write_tables(out_2); + SlabAlloc new_alloc; new_alloc.attach_empty(); // Throws Array top(new_alloc); @@ -1090,9 +1134,10 @@ void Group::write(std::ostream& out, int file_format_version, TableWriter& table _impl::DeepArrayDestroyGuard dg_3(&version_list); bool deep = true; // Deep bool only_if_modified = false; // Always - ref_type free_list_ref = free_list.write(out_2, deep, only_if_modified); - ref_type size_list_ref = size_list.write(out_2, deep, only_if_modified); - ref_type version_list_ref = version_list.write(out_2, deep, only_if_modified); + bool compress = false; + ref_type free_list_ref = free_list.write(out_2, deep, only_if_modified, compress); + ref_type size_list_ref = size_list.write(out_2, deep, only_if_modified, compress); + ref_type version_list_ref = version_list.write(out_2, deep, only_if_modified, compress); top.add(RefOrTagged::make_ref(free_list_ref)); // Throws top.add(RefOrTagged::make_ref(size_list_ref)); // Throws top.add(RefOrTagged::make_ref(version_list_ref)); // Throws @@ -1125,9 +1170,10 @@ void Group::write(std::ostream& out, int file_format_version, TableWriter& table top.set(2, RefOrTagged::make_tagged(final_file_size)); // Throws // Write the top array - bool deep = false; // Shallow - bool only_if_modified = false; // Always - top.write(out_2, deep, only_if_modified); // Throws + bool deep = false; // Shallow + bool only_if_modified = false; // Always + bool compress = false; + top.write(out_2, deep, only_if_modified, compress); // Throws REALM_ASSERT_3(size_t(out_2.get_ref_of_next_array()), ==, final_file_size); dg_top.reset(nullptr); // Destroy now @@ -1266,6 +1312,11 @@ class TransactAdvancer : public _impl::NullInstructionObserver { void Group::update_allocator_wrappers(bool writable) { m_is_writable = writable; + // This is tempting: + // m_alloc.set_read_only(!writable); + // - but m_alloc may refer to the "global" allocator in the DB object. + // Setting it here would cause different transactions to raze for + // changing the shared allocator setting. This is somewhat of a mess. for (size_t i = 0; i < m_table_accessors.size(); ++i) { auto table_accessor = m_table_accessors[i]; if (table_accessor) { diff --git a/src/realm/group.hpp b/src/realm/group.hpp index 9659f550254..24cd6df1b1c 100644 --- a/src/realm/group.hpp +++ b/src/realm/group.hpp @@ -117,7 +117,8 @@ class Group : public ArrayParent { /// types that are derived from FileAccessError, the /// derived exception type is thrown. Note that InvalidDatabase is /// among these derived exception types. - explicit Group(const std::string& file, const char* encryption_key = nullptr); + explicit Group(const std::string& file, const char* encryption_key = nullptr, + bool allow_additional_properties = false); /// Attach this Group instance to the specified memory buffer. /// @@ -515,6 +516,7 @@ class Group : public ArrayParent { m_alloc.enable_debug(enable); } #endif + ref_type typed_write_tables(_impl::ArrayWriterBase& out) const; protected: static constexpr size_t s_table_name_ndx = 0; @@ -598,6 +600,7 @@ class Group : public ArrayParent { mutable int m_num_tables = 0; bool m_attached = false; bool m_is_writable = true; + bool m_allow_additional_properties = false; static std::optional fake_target_file_format; util::UniqueFunction m_notify_handler; @@ -636,7 +639,7 @@ class Group : public ArrayParent { void reset_free_space_tracking(); void remap_and_update_refs(ref_type new_top_ref, size_t new_file_size, bool writable); - + void update_table_accessors(); /// Recursively update refs stored in all cached array /// accessors. This includes cached array accessors in any /// currently attached table accessors. This ensures that the @@ -763,6 +766,9 @@ class Group : public ArrayParent { /// Backlinks in BPlusTree /// Sort order of Strings changed (affects sets and the string index) /// + /// 25 Enhanced layout of NodeHeader to support compression. + /// Integer arrays are stored in a compressed format. + /// /// IMPORTANT: When introducing a new file format version, be sure to review /// the file validity checks in Group::open() and DB::do_open, the file /// format selection logic in @@ -770,7 +776,7 @@ class Group : public ArrayParent { /// upgrade logic in Group::upgrade_file_format(), AND the lists of accepted /// file formats and the version deletion list residing in "backup_restore.cpp" - static constexpr int g_current_file_format_version = 24; + static constexpr int g_current_file_format_version = 25; int get_file_format_version() const noexcept; void set_file_format_version(int) noexcept; @@ -1126,6 +1132,7 @@ class Group::TableWriter { virtual ref_type write_names(_impl::OutputStream&) = 0; virtual ref_type write_tables(_impl::OutputStream&) = 0; virtual HistoryInfo write_history(_impl::OutputStream&) = 0; + virtual ~TableWriter() noexcept {} void set_group(const Group* g) diff --git a/src/realm/group_writer.cpp b/src/realm/group_writer.cpp index 2e6651101fa..22ce7db93ac 100644 --- a/src/realm/group_writer.cpp +++ b/src/realm/group_writer.cpp @@ -41,15 +41,16 @@ class InMemoryWriter : public _impl::ArrayWriterBase { , m_alloc(owner.m_alloc) { } - ref_type write_array(const char* data, size_t size, uint32_t checksum) override + ref_type write_array(const char* data, size_t size, uint32_t checksum, uint32_t checksum_bytes) override { + REALM_ASSERT(checksum_bytes == 4 || checksum_bytes == 2); size_t pos = m_owner.get_free_space(size); // Write the block char* dest_addr = translate(pos); REALM_ASSERT_RELEASE(dest_addr && (reinterpret_cast(dest_addr) & 7) == 0); - memcpy(dest_addr, &checksum, 4); - memcpy(dest_addr + 4, data + 4, size - 4); + memcpy(dest_addr, &checksum, checksum_bytes); + memcpy(dest_addr + checksum_bytes, data + checksum_bytes, size - checksum_bytes); // return ref of the written array ref_type ref = to_ref(pos); return ref; @@ -659,15 +660,17 @@ ref_type GroupWriter::write_group() // that has been release during the current transaction (or since the last // commit), as that would lead to clobbering of the previous database // version. - bool deep = true, only_if_modified = true; + constexpr bool deep = true; + _impl::ArrayWriterBase::compress = true; + _impl::ArrayWriterBase::only_modified = true; std::unique_ptr in_memory_writer; _impl::ArrayWriterBase* writer = this; if (m_alloc.is_in_memory()) { in_memory_writer = std::make_unique(*this); writer = in_memory_writer.get(); } - ref_type names_ref = m_group.m_table_names.write(*writer, deep, only_if_modified); // Throws - ref_type tables_ref = m_group.m_tables.write(*writer, deep, only_if_modified); // Throws + ref_type names_ref = m_group.m_table_names.write(*writer, deep, only_modified, compress); // Throws + ref_type tables_ref = m_group.typed_write_tables(*writer); int_fast64_t value_1 = from_ref(names_ref); int_fast64_t value_2 = from_ref(tables_ref); @@ -680,8 +683,8 @@ ref_type GroupWriter::write_group() if (top.size() > Group::s_hist_ref_ndx) { if (ref_type history_ref = top.get_as_ref(Group::s_hist_ref_ndx)) { Allocator& alloc = top.get_alloc(); - ref_type new_history_ref = Array::write(history_ref, alloc, *writer, only_if_modified); // Throws - top.set(Group::s_hist_ref_ndx, from_ref(new_history_ref)); // Throws + ref_type new_history_ref = Array::write(history_ref, alloc, *writer, only_modified, false); // Throws + top.set(Group::s_hist_ref_ndx, from_ref(new_history_ref)); // Throws } } if (top.size() > Group::s_evacuation_point_ndx) { @@ -703,7 +706,7 @@ ref_type GroupWriter::write_group() for (auto index : m_evacuation_progress) { arr.add(int64_t(index)); } - ref = arr.write(*writer, false, only_if_modified); + ref = arr.write(*writer, false, only_modified, compress); top.set_as_ref(Group::s_evacuation_point_ndx, ref); } else if (ref) { @@ -1337,8 +1340,9 @@ bool inline is_aligned(char* addr) return (as_binary & 7) == 0; } -ref_type GroupWriter::write_array(const char* data, size_t size, uint32_t checksum) +ref_type GroupWriter::write_array(const char* data, size_t size, uint32_t checksum, uint32_t checksum_bytes) { + REALM_ASSERT(checksum_bytes == 4 || checksum_bytes == 2); // Get position of free space to write in (expanding file if needed) size_t pos = get_free_space(size); @@ -1347,8 +1351,8 @@ ref_type GroupWriter::write_array(const char* data, size_t size, uint32_t checks char* dest_addr = window->translate(pos); REALM_ASSERT_RELEASE(is_aligned(dest_addr)); window->encryption_read_barrier(dest_addr, size); - memcpy(dest_addr, &checksum, 4); - memcpy(dest_addr + 4, data + 4, size - 4); + memcpy(dest_addr, &checksum, checksum_bytes); + memcpy(dest_addr + checksum_bytes, data + checksum_bytes, size - checksum_bytes); window->encryption_write_barrier(dest_addr, size); // return ref of the written array ref_type ref = to_ref(pos); diff --git a/src/realm/group_writer.hpp b/src/realm/group_writer.hpp index c0c59145ffa..f17efa59fe7 100644 --- a/src/realm/group_writer.hpp +++ b/src/realm/group_writer.hpp @@ -138,7 +138,7 @@ class GroupWriter : public _impl::ArrayWriterBase { size_t get_file_size() const noexcept; - ref_type write_array(const char*, size_t, uint32_t) override; + ref_type write_array(const char*, size_t, uint32_t, uint32_t) override; #ifdef REALM_DEBUG void dump(); diff --git a/src/realm/impl/array_writer.hpp b/src/realm/impl/array_writer.hpp index 2f18affe7f3..1b2694a8e60 100644 --- a/src/realm/impl/array_writer.hpp +++ b/src/realm/impl/array_writer.hpp @@ -22,10 +22,14 @@ #include namespace realm { +class Table; namespace _impl { class ArrayWriterBase { public: + bool only_modified = true; + bool compress = true; + const Table* table; virtual ~ArrayWriterBase() {} /// Write the specified array data and its checksum into free @@ -33,7 +37,7 @@ class ArrayWriterBase { /// /// Returns the ref (position in the target stream) of the written copy of /// the specified array data. - virtual ref_type write_array(const char* data, size_t size, uint32_t checksum) = 0; + virtual ref_type write_array(const char* data, size_t size, uint32_t checksum, uint32_t checksum_bytes) = 0; }; } // namespace _impl diff --git a/src/realm/impl/output_stream.cpp b/src/realm/impl/output_stream.cpp index 04db91235b6..1b0d870aa2f 100644 --- a/src/realm/impl/output_stream.cpp +++ b/src/realm/impl/output_stream.cpp @@ -39,17 +39,18 @@ void OutputStream::write(const char* data, size_t size) } -ref_type OutputStream::write_array(const char* data, size_t size, uint32_t checksum) +ref_type OutputStream::write_array(const char* data, size_t size, uint32_t checksum, uint32_t checksum_bytes) { REALM_ASSERT(size % 8 == 0); + REALM_ASSERT(checksum_bytes == 4 || checksum_bytes == 2); const char* data_1 = data; size_t size_1 = size; const char* cksum_bytes = reinterpret_cast(&checksum); - m_out.write(cksum_bytes, 4); // Throws - data_1 += 4; - size_1 -= 4; + m_out.write(cksum_bytes, checksum_bytes); // Throws + data_1 += checksum_bytes; + size_1 -= checksum_bytes; do_write(data_1, size_1); // Throws diff --git a/src/realm/impl/output_stream.hpp b/src/realm/impl/output_stream.hpp index eb459900485..ba287f92c30 100644 --- a/src/realm/impl/output_stream.hpp +++ b/src/realm/impl/output_stream.hpp @@ -41,7 +41,7 @@ class OutputStream : public ArrayWriterBase { void write(const char* data, size_t size); - ref_type write_array(const char* data, size_t size, uint32_t checksum) override; + ref_type write_array(const char* data, size_t size, uint32_t checksum, uint32_t checksum_bytes) override; private: ref_type m_next_ref; diff --git a/src/realm/integer_compressor.cpp b/src/realm/integer_compressor.cpp new file mode 100644 index 00000000000..ca8bca153e8 --- /dev/null +++ b/src/realm/integer_compressor.cpp @@ -0,0 +1,308 @@ +/************************************************************************* + * + * Copyright 2023 Realm Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + **************************************************************************/ + +#include +#include +#include +#include +#include +#include + +#include +#include + +using namespace realm; + +namespace { + +template +inline void init_compress_array(Array& arr, size_t byte_size, Arg&&... args) +{ + Allocator& allocator = arr.get_alloc(); + auto mem = allocator.alloc(byte_size); + auto h = mem.get_addr(); + T::init_header(h, std::forward(args)...); + NodeHeader::set_capacity_in_header(byte_size, h); + arr.init_from_mem(mem); +} + +} // namespace + +bool IntegerCompressor::always_compress(const Array& origin, Array& arr, NodeHeader::Encoding encoding) const +{ + using Encoding = NodeHeader::Encoding; + std::vector values; + std::vector indices; + compress_values(origin, values, indices); + if (!values.empty()) { + const uint8_t flags = NodeHeader::get_flags(origin.get_header()); + uint8_t v_width = std::max(Node::signed_to_num_bits(values.front()), Node::signed_to_num_bits(values.back())); + + if (encoding == Encoding::Packed) { + const auto packed_size = NodeHeader::calc_size(indices.size(), v_width, NodeHeader::Encoding::Packed); + init_compress_array(arr, packed_size, flags, v_width, origin.size()); + PackedCompressor::copy_data(origin, arr); + } + else if (encoding == Encoding::Flex) { + uint8_t ndx_width = NodeHeader::unsigned_to_num_bits(values.size()); + const auto flex_size = NodeHeader::calc_size(values.size(), indices.size(), v_width, ndx_width); + init_compress_array(arr, flex_size, flags, v_width, ndx_width, values.size(), + indices.size()); + FlexCompressor::copy_data(arr, values, indices); + } + else { + REALM_UNREACHABLE(); + } + return true; + } + return false; +} + +bool IntegerCompressor::compress(const Array& origin, Array& arr) const +{ + if (origin.m_width < 2 || origin.m_size == 0) + return false; + +#if REALM_COMPRESS + return always_compress(origin, arr, NodeHeader::Encoding::Flex); +#else + std::vector values; + std::vector indices; + compress_values(origin, values, indices); + REALM_ASSERT(!values.empty()); + const auto uncompressed_size = origin.get_byte_size(); + uint8_t ndx_width = NodeHeader::unsigned_to_num_bits(values.size()); + uint8_t v_width = std::max(Node::signed_to_num_bits(values.front()), Node::signed_to_num_bits(values.back())); + const auto packed_size = NodeHeader::calc_size(indices.size(), v_width, NodeHeader::Encoding::Packed); + const auto flex_size = NodeHeader::calc_size(values.size(), indices.size(), v_width, ndx_width); + // heuristic: only compress to packed if gain at least 11.1% + const auto adjusted_packed_size = packed_size + packed_size / 8; + // heuristic: only compress to flex if gain at least 20% + const auto adjusted_flex_size = flex_size + flex_size / 4; + if (adjusted_flex_size < adjusted_packed_size && adjusted_flex_size < uncompressed_size) { + const uint8_t flags = NodeHeader::get_flags(origin.get_header()); + init_compress_array(arr, flex_size, flags, v_width, ndx_width, values.size(), indices.size()); + FlexCompressor::copy_data(arr, values, indices); + return true; + } + else if (adjusted_packed_size < uncompressed_size) { + const uint8_t flags = NodeHeader::get_flags(origin.get_header()); + init_compress_array(arr, packed_size, flags, v_width, origin.size()); + PackedCompressor::copy_data(origin, arr); + return true; + } + return false; +#endif +} + +bool IntegerCompressor::decompress(Array& arr) const +{ + int64_t min_v = std::numeric_limits::max(); + int64_t max_v = std::numeric_limits::min(); + REALM_ASSERT_DEBUG(arr.is_attached()); + auto values_fetcher = [&]() { + const auto sz = arr.size(); + if (is_packed()) { + std::vector res; + res.reserve(sz); + for (size_t i = 0; i < sz; ++i) { + auto val = arr.get(i); + if (val > max_v) + max_v = val; + if (val < min_v) + min_v = val; + res.push_back(val); + } + return res; + } + min_v = FlexCompressor::min(*this); + max_v = FlexCompressor::max(*this); + return FlexCompressor::get_all(*this, 0, sz); + }; + const auto& values = values_fetcher(); + // do the reverse of compressing the array + REALM_ASSERT_DEBUG(!values.empty()); + using Encoding = NodeHeader::Encoding; + const auto flags = NodeHeader::get_flags(arr.get_header()); + const auto size = values.size(); + const auto width = std::max(Array::bit_width(min_v), Array::bit_width(max_v)); + REALM_ASSERT_DEBUG(width == 0 || width == 1 || width == 2 || width == 4 || width == 8 || width == 16 || + width == 32 || width == 64); + // 64 is some slab allocator magic number. + // The padding is needed in order to account for bit width expansion. + const auto byte_size = 64 + NodeHeader::calc_size(size, width, Encoding::WTypBits); + REALM_ASSERT_DEBUG(byte_size % 8 == 0); // nevertheless all the values my be aligned to 8 + + // Create new array with the correct width + const auto mem = arr.get_alloc().alloc(byte_size); + const auto header = mem.get_addr(); + init_header(header, Encoding::WTypBits, flags, width, size); + NodeHeader::set_capacity_in_header(byte_size, header); + + // Destroy old array before initializing + arr.destroy(); + arr.init_from_mem(mem); + + // this is copying the bits straight, without doing any COW, since the array is basically restored, we just need + // to copy the data straight back into it. This makes decompressing the array equivalent to copy on write for + // normal arrays, in fact for a compressed array, we skip COW and we just decompress, getting the same result. + auto setter = arr.m_vtable->setter; + for (size_t ndx = 0; ndx < size; ++ndx) + setter(arr, ndx, values[ndx]); + + // very important: since the ref of the current array has changed, the parent must be informed. + // Otherwise we will lose the link between parent array and child array. + arr.update_parent(); + REALM_ASSERT_DEBUG(width == arr.get_width()); + REALM_ASSERT_DEBUG(arr.size() == values.size()); + + return true; +} + +bool IntegerCompressor::init(const char* h) +{ + m_encoding = NodeHeader::get_encoding(h); + // avoid to check wtype here, it is another access to the header, that we can avoid. + // We just need to know if the encoding is packed or flex. + // This makes Array::init_from_mem faster. + if (REALM_LIKELY(!(is_packed() || is_flex()))) + return false; + + if (is_packed()) { + init_packed(h); + } + else { + init_flex(h); + } + return true; +} +int64_t IntegerCompressor::get_packed(const Array& arr, size_t ndx) +{ + return PackedCompressor::get(arr.m_integer_compressor, ndx); +} + +int64_t IntegerCompressor::get_flex(const Array& arr, size_t ndx) +{ + return FlexCompressor::get(arr.m_integer_compressor, ndx); +} + +std::vector IntegerCompressor::get_all_packed(const Array& arr, size_t begin, size_t end) +{ + return PackedCompressor::get_all(arr.m_integer_compressor, begin, end); +} + +std::vector IntegerCompressor::get_all_flex(const Array& arr, size_t begin, size_t end) +{ + return FlexCompressor::get_all(arr.m_integer_compressor, begin, end); +} + +void IntegerCompressor::get_chunk_packed(const Array& arr, size_t ndx, int64_t res[8]) +{ + PackedCompressor::get_chunk(arr.m_integer_compressor, ndx, res); +} + +void IntegerCompressor::get_chunk_flex(const Array& arr, size_t ndx, int64_t res[8]) +{ + FlexCompressor::get_chunk(arr.m_integer_compressor, ndx, res); +} + +template +bool IntegerCompressor::find_packed(const Array& arr, int64_t val, size_t begin, size_t end, size_t base_index, + QueryStateBase* st) +{ + return PackedCompressor::find_all(arr, val, begin, end, base_index, st); +} + +template +bool IntegerCompressor::find_flex(const Array& arr, int64_t val, size_t begin, size_t end, size_t base_index, + QueryStateBase* st) +{ + return FlexCompressor::find_all(arr, val, begin, end, base_index, st); +} + +void IntegerCompressor::set_vtable(Array& arr) +{ + static const Array::VTable vtable_packed = {get_packed, + get_chunk_packed, + get_all_packed, + nullptr, + { + find_packed, + find_packed, + find_packed, + find_packed, + }}; + static const Array::VTable vtable_flex = {get_flex, + get_chunk_flex, + get_all_flex, + nullptr, + { + find_flex, + find_flex, + find_flex, + find_flex, + }}; + if (is_packed()) { + arr.m_vtable = &vtable_packed; + } + else { + arr.m_vtable = &vtable_flex; + } +} + +int64_t IntegerCompressor::get(size_t ndx) const +{ + if (is_packed()) { + return PackedCompressor::get(*this, ndx); + } + else { + return FlexCompressor::get(*this, ndx); + } +} + +void IntegerCompressor::compress_values(const Array& arr, std::vector& values, + std::vector& indices) const +{ + // The main idea is to compress the values in flex format. If Packed is better it will be chosen by + // IntegerCompressor::compress. The algorithm is O(n lg n), it gives us nice properties, but we could use an + // efficient hash table and try to boost perf during insertion, although leaf arrays are relatively small in + // general (256 entries). The two compresion formats are packed and flex, and the data in the array is re-arranged + // in the following ways (if compressed): + // Packed: || node header || ..... values ..... || + // Flex: || node header || ..... values ..... || ..... indices ..... || + + const auto sz = arr.size(); + REALM_ASSERT_DEBUG(sz > 0); + values.reserve(sz); + indices.reserve(sz); + + for (size_t i = 0; i < sz; ++i) { + auto item = arr.get(i); + values.push_back(item); + } + + std::sort(values.begin(), values.end()); + auto last = std::unique(values.begin(), values.end()); + values.erase(last, values.end()); + + for (size_t i = 0; i < sz; ++i) { + auto pos = std::lower_bound(values.begin(), values.end(), arr.get(i)); + indices.push_back(unsigned(std::distance(values.begin(), pos))); + REALM_ASSERT_DEBUG(values[indices[i]] == arr.get(i)); + } +} diff --git a/src/realm/integer_compressor.hpp b/src/realm/integer_compressor.hpp new file mode 100644 index 00000000000..c177d491125 --- /dev/null +++ b/src/realm/integer_compressor.hpp @@ -0,0 +1,200 @@ +/************************************************************************* + * + * Copyright 2023 Realm Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + **************************************************************************/ + +#ifndef REALM_INTEGER_COMPRESSOR_HPP +#define REALM_INTEGER_COMPRESSOR_HPP + +#include +#include +#include +#include +#include +#include + +namespace realm { + +class Array; +class QueryStateBase; +class IntegerCompressor { +public: + // commit => encode, COW/insert => decode + bool compress(const Array&, Array&) const; + bool decompress(Array&) const; + + bool init(const char*); + void set_vtable(Array&); + + // init from mem B + inline uint64_t* data() const; + inline size_t size() const; + inline NodeHeader::Encoding get_encoding() const; + inline uint8_t v_width() const; + inline uint8_t ndx_width() const; + inline size_t v_size() const; + inline size_t ndx_size() const; + + inline uint64_t v_mask() const; + inline uint64_t ndx_mask() const; + inline uint64_t msb() const; + inline uint64_t ndx_msb() const; + inline uint64_t bitmask_v() const; + inline uint64_t bitmask_ndx() const; + + int64_t get(size_t) const; + +private: + // getting and setting interface specifically for encoding formats + inline void init_packed(const char*); + inline void init_flex(const char*); + + static int64_t get_packed(const Array& arr, size_t ndx); + static int64_t get_flex(const Array& arr, size_t ndx); + + static std::vector get_all_packed(const Array& arr, size_t begin, size_t end); + static std::vector get_all_flex(const Array& arr, size_t begin, size_t end); + + static void get_chunk_packed(const Array& arr, size_t ndx, int64_t res[8]); + static void get_chunk_flex(const Array& arr, size_t ndx, int64_t res[8]); + // query interface + template + static bool find_packed(const Array& arr, int64_t val, size_t begin, size_t end, size_t base_index, + QueryStateBase* st); + template + static bool find_flex(const Array& arr, int64_t val, size_t begin, size_t end, size_t base_index, + QueryStateBase* st); + + // internal impl + void compress_values(const Array&, std::vector&, std::vector&) const; + inline bool is_packed() const; + inline bool is_flex() const; + + // for testing + bool always_compress(const Array&, Array&, Node::Encoding) const; + +private: + using Encoding = NodeHeader::Encoding; + Encoding m_encoding{NodeHeader::Encoding::WTypBits}; + uint64_t* m_data; + uint8_t m_v_width = 0, m_ndx_width = 0; + size_t m_v_size = 0, m_ndx_size = 0; +}; + +inline void IntegerCompressor::init_packed(const char* h) +{ + m_data = (uint64_t*)NodeHeader::get_data_from_header(h); + m_v_width = NodeHeader::get_element_size(h, Encoding::Packed); + m_v_size = NodeHeader::get_num_elements(h, Encoding::Packed); +} + +inline void IntegerCompressor::init_flex(const char* h) +{ + m_data = (uint64_t*)NodeHeader::get_data_from_header(h); + m_v_width = NodeHeader::get_elementA_size(h); + m_v_size = NodeHeader::get_arrayA_num_elements(h); + m_ndx_width = NodeHeader::get_elementB_size(h); + m_ndx_size = NodeHeader::get_arrayB_num_elements(h); +} + +inline uint64_t* IntegerCompressor::data() const +{ + return m_data; +} + +inline bool IntegerCompressor::is_packed() const +{ + return m_encoding == NodeHeader::Encoding::Packed; +} + +inline bool IntegerCompressor::is_flex() const +{ + return m_encoding == NodeHeader::Encoding::Flex; +} + +inline size_t IntegerCompressor::size() const +{ + REALM_ASSERT_DEBUG(is_packed() || is_flex()); + return m_encoding == NodeHeader::Encoding::Packed ? v_size() : ndx_size(); +} + +inline size_t IntegerCompressor::v_size() const +{ + REALM_ASSERT_DEBUG(is_packed() || is_flex()); + return m_v_size; +} + +inline size_t IntegerCompressor::ndx_size() const +{ + REALM_ASSERT_DEBUG(is_flex()); + return m_ndx_size; +} + +inline uint8_t IntegerCompressor::v_width() const +{ + REALM_ASSERT_DEBUG(is_packed() || is_flex()); + return m_v_width; +} + +inline uint8_t IntegerCompressor::ndx_width() const +{ + REALM_ASSERT_DEBUG(is_flex()); + return m_ndx_width; +} + +inline NodeHeader::Encoding IntegerCompressor::get_encoding() const +{ + return m_encoding; +} + +inline uint64_t IntegerCompressor::v_mask() const +{ + REALM_ASSERT_DEBUG(is_packed() || is_flex()); + return 1ULL << (m_v_width - 1); +} + +inline uint64_t IntegerCompressor::ndx_mask() const +{ + REALM_ASSERT_DEBUG(is_flex()); + return 1ULL << (m_ndx_width - 1); +} + +inline uint64_t IntegerCompressor::msb() const +{ + REALM_ASSERT_DEBUG(is_packed() || is_flex()); + return populate(m_v_width, v_mask()); +} + +inline uint64_t IntegerCompressor::ndx_msb() const +{ + REALM_ASSERT_DEBUG(is_flex()); + return populate(m_ndx_width, ndx_mask()); +} + +inline uint64_t IntegerCompressor::bitmask_v() const +{ + REALM_ASSERT_DEBUG(is_packed() || is_flex()); + return 0xFFFFFFFFFFFFFFFFULL >> (64 - m_v_width); +} + +inline uint64_t IntegerCompressor::bitmask_ndx() const +{ + REALM_ASSERT_DEBUG(is_flex()); + return 0xFFFFFFFFFFFFFFFFULL >> (64 - m_ndx_width); +} + +} // namespace realm +#endif // REALM_INTEGER_COMPRESSOR_HPP diff --git a/src/realm/integer_flex_compressor.cpp b/src/realm/integer_flex_compressor.cpp new file mode 100644 index 00000000000..a7d8c8134b0 --- /dev/null +++ b/src/realm/integer_flex_compressor.cpp @@ -0,0 +1,87 @@ +/************************************************************************* + * + * Copyright 2023 Realm Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + **************************************************************************/ + +#include +#include +#include + +#include +#include + +#ifdef REALM_DEBUG +#include +#include +#endif + +using namespace realm; + +void FlexCompressor::init_header(char* h, uint8_t flags, uint8_t v_width, uint8_t ndx_width, size_t v_size, + size_t ndx_size) +{ + using Encoding = NodeHeader::Encoding; + ::init_header(h, Encoding::Flex, flags, v_width, ndx_width, v_size, ndx_size); +} + +void FlexCompressor::copy_data(const Array& arr, const std::vector& values, + const std::vector& indices) +{ + using Encoding = NodeHeader::Encoding; + REALM_ASSERT_DEBUG(arr.is_attached()); + const auto& compressor = arr.integer_compressor(); + REALM_ASSERT_DEBUG(compressor.get_encoding() == Encoding::Flex); + const auto v_width = compressor.v_width(); + const auto ndx_width = compressor.ndx_width(); + const auto v_size = values.size(); + const auto data = (uint64_t*)arr.m_data; + const auto offset = static_cast(v_size * v_width); + BfIterator it_value{data, 0, v_width, v_width, 0}; + BfIterator it_index{data, offset, ndx_width, ndx_width, 0}; + for (size_t i = 0; i < v_size; ++i) { + it_value.set_value(values[i]); + REALM_ASSERT_DEBUG(sign_extend_value(v_width, it_value.get_value()) == values[i]); + ++it_value; + } + for (size_t i = 0; i < indices.size(); ++i) { + REALM_ASSERT_DEBUG(values[indices[i]] == + sign_extend_value(v_width, read_bitfield(data, indices[i] * v_width, v_width))); + it_index.set_value(indices[i]); + REALM_ASSERT_DEBUG(indices[i] == it_index.get_value()); + REALM_ASSERT_DEBUG(values[indices[i]] == + sign_extend_value(v_width, read_bitfield(data, indices[i] * v_width, v_width))); + ++it_index; + } +} + +bool FlexCompressor::find_all_match(size_t start, size_t end, size_t baseindex, QueryStateBase* state) +{ + REALM_ASSERT_DEBUG(state->match_count() < state->limit()); + while (start < end) { + if (!state->match(start + baseindex)) + return false; + start++; + } + return true; +} + +size_t FlexCompressor::lower_bound(size_t size, int64_t value, uint64_t mask, BfIterator& data_iterator) noexcept +{ + return impl::lower_bound(nullptr, 0, size, value, [&](auto, size_t ndx) { + data_iterator.move(ndx); + return sign_extend_field_by_mask(mask, *data_iterator); + }); +} diff --git a/src/realm/integer_flex_compressor.hpp b/src/realm/integer_flex_compressor.hpp new file mode 100644 index 00000000000..3296e920f6d --- /dev/null +++ b/src/realm/integer_flex_compressor.hpp @@ -0,0 +1,286 @@ +/************************************************************************* + * + * Copyright 2023 Realm Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + **************************************************************************/ + +#ifndef FLEX_COMPRESSOR_HPP +#define FLEX_COMPRESSOR_HPP + +#include + +#include +#include +#include + +namespace realm { + +// +// Compress array in Flex format +// Decompress array in WTypeBits formats +// +class FlexCompressor { +public: + // encoding/decoding + static void init_header(char*, uint8_t, uint8_t, uint8_t, size_t, size_t); + static void copy_data(const Array&, const std::vector&, const std::vector&); + // getters/setters + static int64_t get(const IntegerCompressor&, size_t); + static std::vector get_all(const IntegerCompressor&, size_t, size_t); + static void get_chunk(const IntegerCompressor&, size_t, int64_t[8]); + + template + static bool find_all(const Array&, int64_t, size_t, size_t, size_t, QueryStateBase*); + + static int64_t min(const IntegerCompressor&); + static int64_t max(const IntegerCompressor&); + +private: + static bool find_all_match(size_t, size_t, size_t, QueryStateBase*); + static size_t lower_bound(size_t, int64_t, uint64_t, BfIterator&) noexcept; +}; + +inline int64_t FlexCompressor::get(const IntegerCompressor& c, size_t ndx) +{ + const auto offset = c.v_width() * c.v_size(); + const auto ndx_w = c.ndx_width(); + const auto v_w = c.v_width(); + const auto data = c.data(); + BfIterator ndx_iterator{data, offset, ndx_w, ndx_w, ndx}; + BfIterator data_iterator{data, 0, v_w, v_w, static_cast(*ndx_iterator)}; + return sign_extend_field_by_mask(c.v_mask(), *data_iterator); +} + +inline std::vector FlexCompressor::get_all(const IntegerCompressor& c, size_t b, size_t e) +{ + const auto offset = c.v_width() * c.v_size(); + const auto ndx_w = c.ndx_width(); + const auto v_w = c.v_width(); + const auto data = c.data(); + const auto sign_mask = c.v_mask(); + const auto range = (e - b); + const auto starting_bit = offset + b * ndx_w; + const auto bit_per_it = num_bits_for_width(ndx_w); + const auto ndx_mask = 0xFFFFFFFFFFFFFFFFULL >> (64 - ndx_w); + const auto values_per_word = num_fields_for_width(ndx_w); + + // this is very important, x4 faster pre-allocating the array + std::vector res; + res.reserve(range); + + UnalignedWordIter unaligned_ndx_iterator(data, starting_bit); + BfIterator data_iterator{data, 0, v_w, v_w, 0}; + auto remaining_bits = ndx_w * range; + while (remaining_bits >= bit_per_it) { + auto word = unaligned_ndx_iterator.consume(bit_per_it); + for (int i = 0; i < values_per_word; ++i) { + const auto index = word & ndx_mask; + data_iterator.move(static_cast(index)); + const auto sv = sign_extend_field_by_mask(sign_mask, *data_iterator); + res.push_back(sv); + word >>= ndx_w; + } + remaining_bits -= bit_per_it; + } + if (remaining_bits) { + auto last_word = unaligned_ndx_iterator.consume(remaining_bits); + while (remaining_bits) { + const auto index = last_word & ndx_mask; + data_iterator.move(static_cast(index)); + const auto sv = sign_extend_field_by_mask(sign_mask, *data_iterator); + res.push_back(sv); + remaining_bits -= ndx_w; + last_word >>= ndx_w; + } + } + return res; +} + +inline int64_t FlexCompressor::min(const IntegerCompressor& c) +{ + const auto v_w = c.v_width(); + const auto data = c.data(); + const auto sign_mask = c.v_mask(); + BfIterator data_iterator{data, 0, v_w, v_w, 0}; + return sign_extend_field_by_mask(sign_mask, *data_iterator); +} + +inline int64_t FlexCompressor::max(const IntegerCompressor& c) +{ + const auto v_w = c.v_width(); + const auto data = c.data(); + const auto sign_mask = c.v_mask(); + BfIterator data_iterator{data, 0, v_w, v_w, c.v_size() - 1}; + return sign_extend_field_by_mask(sign_mask, *data_iterator); +} + +inline void FlexCompressor::get_chunk(const IntegerCompressor& c, size_t ndx, int64_t res[8]) +{ + auto sz = 8; + std::memset(res, 0, sizeof(int64_t) * sz); + auto supposed_end = ndx + sz; + size_t i = ndx; + size_t index = 0; + for (; i < supposed_end; ++i) { + res[index++] = get(c, i); + } + for (; index < 8; ++index) { + res[index++] = get(c, i++); + } +} + +template +class IndexCond { +public: + using type = T; +}; + +template <> +class IndexCond { +public: + using type = GreaterEqual; +}; + +template +inline bool FlexCompressor::find_all(const Array& arr, int64_t value, size_t start, size_t end, size_t baseindex, + QueryStateBase* state) +{ + static constexpr size_t RANGE_LIMIT = 20; + static constexpr size_t WIDTH_LIMIT = 16; + + REALM_ASSERT_DEBUG(start <= arr.m_size && (end <= arr.m_size || end == size_t(-1)) && start <= end); + Cond c; + + if (end == npos) + end = arr.m_size; + + if (start >= arr.m_size || start >= end) + return true; + + const auto lbound = arr.m_lbound; + const auto ubound = arr.m_ubound; + + if (!c.can_match(value, lbound, ubound)) + return true; + + if (c.will_match(value, lbound, ubound)) { + return find_all_match(start, end, baseindex, state); + } + + REALM_ASSERT_DEBUG(arr.m_width != 0); + + const auto& compressor = arr.integer_compressor(); + const auto v_width = arr.m_width; + const auto v_size = compressor.v_size(); + const auto mask = compressor.v_mask(); + uint64_t* data = (uint64_t*)arr.m_data; + size_t v_start = realm::not_found; + + /**************** Search the values ****************/ + + int64_t modified_value = value; + if constexpr (std::is_same_v) { + modified_value++; // We use GreaterEqual below, so this will effectively be Greater + } + + if (v_size >= RANGE_LIMIT) { + if (v_width <= WIDTH_LIMIT) { + auto search_vector = populate(v_width, modified_value); + v_start = parallel_subword_find(find_all_fields, data, 0, v_width, compressor.msb(), + search_vector, 0, v_size); + } + else { + BfIterator data_iterator{data, 0, v_width, v_width, 0}; + v_start = lower_bound(v_size, modified_value, mask, data_iterator); + } + } + else { + BfIterator data_iterator{data, 0, v_width, v_width, 0}; + size_t idx = 0; + while (idx < v_size) { + if (sign_extend_field_by_mask(mask, *data_iterator) >= modified_value) { + break; + } + data_iterator.move(++idx); + } + v_start = idx; + } + + if constexpr (realm::is_any_v) { + // Check for equality. + if (v_start < v_size) { + BfIterator it{data, 0, v_width, v_width, v_start}; + if (sign_extend_field_by_mask(mask, *it) > value) { + v_start = v_size; // Mark as not found + } + } + } + + /***************** Some early outs *****************/ + + if (v_start == v_size) { + if constexpr (realm::is_any_v) { + return true; // No Matches + } + if constexpr (realm::is_any_v) { + return find_all_match(start, end, baseindex, state); // All matches + } + } + else if (v_start == 0) { + if constexpr (std::is_same_v) { + // No index is less than 0 + return true; // No Matches + } + if constexpr (std::is_same_v) { + // All index is greater than or equal to 0 + return find_all_match(start, end, baseindex, state); + } + } + + /*************** Search the indexes ****************/ + + using U = typename IndexCond::type; + const auto ndx_range = end - start; + const auto ndx_width = compressor.ndx_width(); + const auto v_offset = v_size * v_width; + if (ndx_range >= RANGE_LIMIT) { + auto search_vector = populate(ndx_width, v_start); + while (start < end) { + start = parallel_subword_find(find_all_fields_unsigned, data, v_offset, ndx_width, + compressor.ndx_msb(), search_vector, start, end); + if (start < end) { + if (!state->match(start + baseindex)) + return false; + } + ++start; + } + } + else { + U index_c; + BfIterator ndx_iterator{data, v_offset, ndx_width, ndx_width, start}; + while (start < end) { + if (index_c(int64_t(*ndx_iterator), int64_t(v_start))) { + if (!state->match(start + baseindex)) + return false; + } + ndx_iterator.move(++start); + } + } + + return true; +} + +} // namespace realm +#endif // FLEX_COMPRESSOR_HPP diff --git a/src/realm/integer_packed_compressor.cpp b/src/realm/integer_packed_compressor.cpp new file mode 100644 index 00000000000..2f7646b1b0c --- /dev/null +++ b/src/realm/integer_packed_compressor.cpp @@ -0,0 +1,68 @@ +/************************************************************************* + * + * Copyright 2023 Realm Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + **************************************************************************/ + +#include +#include +#include +#include +#include + +#include +#include + +#ifdef REALM_DEBUG +#include +#include +#endif + +using namespace realm; + +void PackedCompressor::init_header(char* h, uint8_t flags, uint8_t v_width, size_t v_size) +{ + using Encoding = NodeHeader::Encoding; + ::init_header((char*)h, Encoding::Packed, flags, static_cast(v_width), v_size); +} + +void PackedCompressor::copy_data(const Array& origin, Array& arr) +{ + // this can be boosted a little bit, with and size should be known at this stage. + using Encoding = NodeHeader::Encoding; + REALM_ASSERT_DEBUG(arr.is_attached()); + REALM_ASSERT_DEBUG(arr.integer_compressor().get_encoding() == Encoding::Packed); + // we don't need to access the header, init from mem must have been called + const auto v_width = arr.m_width; + const auto v_size = arr.m_size; + auto data = (uint64_t*)arr.m_data; + BfIterator it_value{data, 0, v_width, v_width, 0}; + for (size_t i = 0; i < v_size; ++i) { + it_value.set_value(origin.get(i)); + REALM_ASSERT_DEBUG(sign_extend_value(v_width, it_value.get_value()) == origin.get(i)); + ++it_value; + } +} + +bool PackedCompressor::find_all_match(size_t start, size_t end, size_t baseindex, QueryStateBase* state) +{ + REALM_ASSERT_DEBUG(state->match_count() < state->limit()); + const auto process = state->limit() - state->match_count(); + const auto end2 = end - start > process ? start + process : end; + for (; start < end2; start++) + if (!state->match(start + baseindex)) + return false; + return true; +} diff --git a/src/realm/integer_packed_compressor.hpp b/src/realm/integer_packed_compressor.hpp new file mode 100644 index 00000000000..ce342a582e2 --- /dev/null +++ b/src/realm/integer_packed_compressor.hpp @@ -0,0 +1,222 @@ +/************************************************************************* + * + * Copyright 2024 Realm Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + **************************************************************************/ + +#ifndef PACKED_COMPRESSOR_HPP +#define PACKED_COMPRESSOR_HPP + +#include +#include + +#include +#include + +namespace realm { + +// +// Compress array in Packed format +// Decompress array in WTypeBits formats +// +class PackedCompressor { +public: + // encoding/decoding + static void init_header(char*, uint8_t, uint8_t, size_t); + static void copy_data(const Array&, Array&); + // get or set + static int64_t get(const IntegerCompressor&, size_t); + static std::vector get_all(const IntegerCompressor& c, size_t b, size_t e); + static void get_chunk(const IntegerCompressor&, size_t, int64_t res[8]); + + template + static bool find_all(const Array&, int64_t, size_t, size_t, size_t, QueryStateBase*); + +private: + static bool find_all_match(size_t start, size_t end, size_t baseindex, QueryStateBase* state); + + template + static bool find_parallel(const Array&, int64_t, size_t, size_t, size_t, QueryStateBase*); + + template + static bool find_linear(const Array&, int64_t, size_t, size_t, size_t, QueryStateBase*); + + template + static bool run_parallel_scan(size_t, size_t); +}; + +inline int64_t PackedCompressor::get(const IntegerCompressor& c, size_t ndx) +{ + BfIterator it{c.data(), 0, c.v_width(), c.v_width(), ndx}; + return sign_extend_field_by_mask(c.v_mask(), *it); +} + +inline std::vector PackedCompressor::get_all(const IntegerCompressor& c, size_t b, size_t e) +{ + const auto range = (e - b); + const auto v_w = c.v_width(); + const auto data = c.data(); + const auto sign_mask = c.v_mask(); + const auto starting_bit = b * v_w; + const auto total_bits = starting_bit + (v_w * range); + const auto mask = 0xFFFFFFFFFFFFFFFFULL >> (64 - v_w); + const auto bit_per_it = num_bits_for_width(v_w); + const auto values_per_word = num_fields_for_width(v_w); + + std::vector res; + res.reserve(range); + + UnalignedWordIter unaligned_data_iterator(data, starting_bit); + auto cnt_bits = starting_bit; + while (cnt_bits + bit_per_it < total_bits) { + auto word = unaligned_data_iterator.consume(bit_per_it); + for (int i = 0; i < values_per_word; ++i) { + res.push_back(sign_extend_field_by_mask(sign_mask, word & mask)); + word >>= v_w; + } + cnt_bits += bit_per_it; + } + if (cnt_bits < total_bits) { + auto last_word = unaligned_data_iterator.consume(static_cast(total_bits - cnt_bits)); + while (cnt_bits < total_bits) { + res.push_back(sign_extend_field_by_mask(sign_mask, last_word & mask)); + cnt_bits += v_w; + last_word >>= v_w; + } + } + return res; +} + +inline void PackedCompressor::get_chunk(const IntegerCompressor& c, size_t ndx, int64_t res[8]) +{ + auto sz = 8; + std::memset(res, 0, sizeof(int64_t) * sz); + auto supposed_end = ndx + sz; + size_t i = ndx; + size_t index = 0; + // this can be done better, in one go, retrieve both!!! + for (; i < supposed_end; ++i) { + res[index++] = get(c, i); + } + for (; index < 8; ++index) { + res[index++] = get(c, i++); + } +} + + +template +inline bool PackedCompressor::find_all(const Array& arr, int64_t value, size_t start, size_t end, size_t baseindex, + QueryStateBase* state) +{ + REALM_ASSERT_DEBUG(start <= arr.m_size && (end <= arr.m_size || end == size_t(-1)) && start <= end); + Cond c; + + if (end == npos) + end = arr.m_size; + + if (!(arr.m_size > start && start < end)) + return true; + + const auto lbound = arr.m_lbound; + const auto ubound = arr.m_ubound; + + if (!c.can_match(value, lbound, ubound)) + return true; + + if (c.will_match(value, lbound, ubound)) { + return find_all_match(start, end, baseindex, state); + } + + REALM_ASSERT_DEBUG(arr.m_width != 0); + + if (!run_parallel_scan(arr.m_width, end - start)) + return find_linear(arr, value, start, end, baseindex, state); + + return find_parallel(arr, value, start, end, baseindex, state); +} + +template +inline bool PackedCompressor::find_parallel(const Array& arr, int64_t value, size_t start, size_t end, + size_t baseindex, QueryStateBase* state) +{ + // + // Main idea around find parallel (applicable to flex arrays too). + // Try to find the starting point where the condition can be met, comparing as many values as a single 64bit can + // contain in parallel. Once we have found the starting point, keep matching values as much as we can between + // start and end. + // + // EG: let's store 6, it gets stored in 4 bits (0110). 6 is 4 bits because 110 (6) + sign bit 0. + // Inside 64bits we can fit max 16 times 6. If we go from index 0 to 15 throughout the same 64 bits, we need to + // apply a mask and a shift bits every time, then compare the extracted values. + // This is not the cheapest thing to do. Instead we can compare all values contained within 64 bits in one go, and + // see if there is a match with what we are looking for. Reducing the number of comparison by ~logk(N) where K is + // the width of each single value within a 64 bit word and N is the total number of values stored in the array. + + const auto data = (const uint64_t*)arr.m_data; + const auto width = arr.m_width; + const auto MSBs = arr.integer_compressor().msb(); + const auto search_vector = populate(arr.m_width, value); + while (start < end) { + start = parallel_subword_find(find_all_fields, data, 0, width, MSBs, search_vector, start, end); + if (start < end && !state->match(start + baseindex)) + return false; + ++start; + } + return true; +} + +template +inline bool PackedCompressor::find_linear(const Array& arr, int64_t value, size_t start, size_t end, size_t baseindex, + QueryStateBase* state) +{ + auto compare = [](int64_t a, int64_t b) { + if constexpr (std::is_same_v) + return a == b; + if constexpr (std::is_same_v) + return a != b; + if constexpr (std::is_same_v) + return a > b; + if constexpr (std::is_same_v) + return a < b; + }; + const auto& c = arr.integer_compressor(); + BfIterator it{c.data(), 0, c.v_width(), c.v_width(), start}; + for (; start < end; ++start) { + it.move(start); + const auto sv = sign_extend_field_by_mask(c.v_mask(), *it); + if (compare(sv, value) && !state->match(start + baseindex)) + return false; + } + return true; +} + +template +inline bool PackedCompressor::run_parallel_scan(size_t width, size_t range) +{ + if constexpr (std::is_same_v) { + // we seem to be particularly slow doing parallel scan in packed for NotEqual. + // we are much better with a linear scan. TODO: investigate this. + return false; + } + if constexpr (std::is_same_v) { + return width < 32 && range >= 20; + } + // > and < need a different heuristic + return width <= 20 && range >= 20; +} + +} // namespace realm + +#endif // PACKED_COMPRESSOR_HPP diff --git a/src/realm/node.cpp b/src/realm/node.cpp index cc880689509..63ef4d3962c 100644 --- a/src/realm/node.cpp +++ b/src/realm/node.cpp @@ -26,17 +26,31 @@ using namespace realm; -MemRef Node::create_node(size_t size, Allocator& alloc, bool context_flag, Type type, WidthType width_type, int width) +MemRef Node::create_node(size_t size, Allocator& alloc, bool context_flag, Type type, WidthType width_type, + uint8_t width) { size_t byte_size_0 = calc_byte_size(width_type, size, width); size_t byte_size = std::max(byte_size_0, size_t(initial_capacity)); MemRef mem = alloc.alloc(byte_size); // Throws - char* header = mem.get_addr(); - - init_header(header, type == type_InnerBptreeNode, type != type_Normal, context_flag, width_type, width, size, - byte_size); - + const auto header = mem.get_addr(); + REALM_ASSERT_DEBUG(width_type != WidthType::wtype_Extend); + Encoding encoding{static_cast(width_type)}; + + uint8_t flags = 0; + if (type == type_InnerBptreeNode) + flags |= static_cast(Flags::InnerBPTree) | static_cast(Flags::HasRefs); + if (type != type_Normal) + flags |= static_cast(Flags::HasRefs); + if (context_flag) + flags |= static_cast(Flags::Context); + // width must be passed to init_header in bits, but for wtype_Multiply and wtype_Ignore + // it is provided by the caller of this function in bytes, so convert to bits + if (width_type != wtype_Bits) + width = width * 8; + + init_header(header, encoding, flags, width, size); + set_capacity_in_header(byte_size, header); return mem; } @@ -68,18 +82,21 @@ size_t Node::calc_item_count(size_t bytes, size_t width) const noexcept void Node::alloc(size_t init_size, size_t new_width) { - REALM_ASSERT(is_attached()); - + REALM_ASSERT_DEBUG(is_attached()); + char* header = get_header_from_data(m_data); + REALM_ASSERT_DEBUG(!wtype_is_extended(header)); size_t needed_bytes = calc_byte_len(init_size, new_width); // this method is not public and callers must (and currently do) ensure that // needed_bytes are never larger than max_array_payload. REALM_ASSERT_RELEASE(init_size <= max_array_size); - if (is_read_only()) + if (is_read_only()) { do_copy_on_write(needed_bytes); + // header has changed after copy on write if the array was compressed + header = get_header_from_data(m_data); + } REALM_ASSERT(!m_alloc.is_read_only(m_ref)); - char* header = get_header_from_data(m_data); size_t orig_capacity_bytes = get_capacity_from_header(header); size_t orig_width = get_width_from_header(header); @@ -114,18 +131,28 @@ void Node::alloc(size_t init_size, size_t new_width) // this array instance in a corrupt state update_parent(); // Throws } - - // Update header + // update width (important when we convert from normal uncompressed array into compressed format) if (new_width != orig_width) { - set_width_in_header(int(new_width), header); + set_width_in_header(new_width, header); } set_size_in_header(init_size, header); m_size = init_size; } +void Node::destroy() noexcept +{ + if (!is_attached()) + return; + char* header = get_header_from_data(m_data); + m_alloc.free_(m_ref, header); + m_data = nullptr; +} + void Node::do_copy_on_write(size_t minimum_size) { const char* header = get_header_from_data(m_data); + // only type A arrays should be allowed during copy on write + REALM_ASSERT(!wtype_is_extended(header)); // Calculate size in bytes size_t array_size = calc_byte_size(get_wtype_from_header(header), m_size, get_width_from_header(header)); @@ -140,7 +167,6 @@ void Node::do_copy_on_write(size_t minimum_size) const char* old_end = header + array_size; char* new_begin = mref.get_addr(); realm::safe_copy_n(old_begin, old_end - old_begin, new_begin); - ref_type old_ref = m_ref; // Update internal data @@ -150,7 +176,6 @@ void Node::do_copy_on_write(size_t minimum_size) // Update capacity in header. Uses m_data to find header, so // m_data must be initialized correctly first. set_capacity_in_header(new_size, new_begin); - update_parent(); #if REALM_ENABLE_MEMDEBUG diff --git a/src/realm/node.hpp b/src/realm/node.hpp index 8bea1c9559d..8d606b37708 100644 --- a/src/realm/node.hpp +++ b/src/realm/node.hpp @@ -22,6 +22,8 @@ #include #include +#include + namespace realm { class Mixed; @@ -114,7 +116,7 @@ class Node : public NodeHeader { { } - virtual ~Node() {} + virtual ~Node() = default; /**************************** Initializers *******************************/ @@ -123,10 +125,10 @@ class Node : public NodeHeader { char* init_from_mem(MemRef mem) noexcept { char* header = mem.get_addr(); + REALM_ASSERT_DEBUG(!wtype_is_extended(header)); m_ref = mem.get_ref(); m_data = get_data_from_header(header); m_size = get_size_from_header(header); - return header; } @@ -212,14 +214,7 @@ class Node : public NodeHeader { /// children of that array. See non-static destroy_deep() for an /// alternative. If this accessor is already in the detached state, this /// function has no effect (idempotency). - void destroy() noexcept - { - if (!is_attached()) - return; - char* header = get_header_from_data(m_data); - m_alloc.free_(m_ref, header); - m_data = nullptr; - } + void destroy() noexcept; /// Shorthand for `destroy(MemRef(ref, alloc), alloc)`. static void destroy(ref_type ref, Allocator& alloc) noexcept @@ -234,7 +229,6 @@ class Node : public NodeHeader { alloc.free_(mem); } - /// Setting a new parent affects ownership of the attached array node, if /// any. If a non-null parent is specified, and there was no parent /// originally, then the caller passes ownership to the parent, and vice @@ -245,6 +239,7 @@ class Node : public NodeHeader { m_parent = parent; m_ndx_in_parent = ndx_in_parent; } + void set_ndx_in_parent(size_t ndx) noexcept { m_ndx_in_parent = ndx; @@ -323,7 +318,7 @@ class Node : public NodeHeader { } static MemRef create_node(size_t size, Allocator& alloc, bool context_flag = false, Type type = type_Normal, - WidthType width_type = wtype_Ignore, int width = 1); + WidthType width_type = wtype_Ignore, uint8_t width = 1); void set_header_size(size_t value) noexcept { @@ -333,8 +328,6 @@ class Node : public NodeHeader { // Includes array header. Not necessarily 8-byte aligned. virtual size_t calc_byte_len(size_t num_items, size_t width) const; virtual size_t calc_item_count(size_t bytes, size_t width) const noexcept; - static void init_header(char* header, bool is_inner_bptree_node, bool has_refs, bool context_flag, - WidthType width_type, int width, size_t size, size_t capacity) noexcept; private: friend class NodeTree; @@ -348,6 +341,10 @@ class Node : public NodeHeader { class Spec; class Mixed; +namespace _impl { +class ArrayWriterBase; +} + /// Base class for all nodes holding user data class ArrayPayload { public: @@ -360,24 +357,9 @@ class ArrayPayload { return false; } virtual void set_spec(Spec*, size_t) const {} + static ref_type typed_write(ref_type ref, _impl::ArrayWriterBase& out, Allocator& alloc); }; - -inline void Node::init_header(char* header, bool is_inner_bptree_node, bool has_refs, bool context_flag, - WidthType width_type, int width, size_t size, size_t capacity) noexcept -{ - // Note: Since the header layout contains unallocated bit and/or - // bytes, it is important that we put the entire header into a - // well defined state initially. - std::fill(header, header + header_size, 0); - set_is_inner_bptree_node_in_header(is_inner_bptree_node, header); - set_hasrefs_in_header(has_refs, header); - set_context_flag_in_header(context_flag, header); - set_wtype_in_header(width_type, header); - set_width_in_header(width, header); - set_size_in_header(size, header); - set_capacity_in_header(capacity, header); -} } // namespace realm #endif /* REALM_NODE_HPP */ diff --git a/src/realm/node_header.hpp b/src/realm/node_header.hpp index 453f6100eaf..ca7d5638025 100644 --- a/src/realm/node_header.hpp +++ b/src/realm/node_header.hpp @@ -20,16 +20,35 @@ #define REALM_NODE_HEADER_HPP #include +#include + +namespace { +// helper converting a number of bits into bytes and aligning to 8 byte boundary +static inline size_t align_bits_to8(size_t n) +{ + n = (n + 7) >> 3; + return (n + 7) & ~size_t(7); +} +} // namespace namespace realm { +// The header holds metadata for all allocations. It is 8 bytes. +// A field in byte 5 indicates the type of the allocation. +// +// Up to and including Core v 13, this field would always hold values 0,1 or 2. +// when stored in the file. This value now indicates that the chunk of memory +// must be interpreted according to the methods in NodeHeader. +// const size_t max_array_size = 0x00ffffffL; // Maximum number of elements in an array const size_t max_array_payload_aligned = 0x07ffffc0L; // Maximum number of bytes that the payload of an array can be // Even though the encoding supports arrays with size up to max_array_payload_aligned, // the maximum allocation size is smaller as it must fit within a memory section // (a contiguous virtual address range). This limitation is enforced in SlabAlloc::do_alloc(). + class NodeHeader { + public: enum Type { type_Normal, @@ -49,10 +68,46 @@ class NodeHeader { }; enum WidthType { + // The first 3 encodings where the only one used as far as Core v13. wtype_Bits = 0, // width indicates how many bits every element occupies wtype_Multiply = 1, // width indicates how many bytes every element occupies wtype_Ignore = 2, // each element is 1 byte + wtype_Extend = 3 // the layouts are described in byte 4 of the header. + }; + // Accessing flags. + enum class Flags { // bit positions in flags "byte", used for masking + Context = 1, + HasRefs = 2, + InnerBPTree = 4, + // additional flags can be supported by new layouts, but old layout is full }; + // Possible header encodings (and corresponding memory layouts): + enum class Encoding { + WTypBits = 0, // Corresponds to wtype_Bits + WTypMult = 1, // Corresponds to wtype_Multiply + WTypIgn = 2, // Corresponds to wtype_Ignore + Packed = 3, // wtype is wtype_Extend + Flex = 4 // wtype is wtype_Extend + }; + // * Packed: tightly packed array (any element size <= 64) + // * WTypBits: less tightly packed. Correspond to wtype_Bits + // * WTypMult: less tightly packed. Correspond to wtype_Multiply + // * WTypIgn: single byte elements. Correspond to wtype_Ignore + // encodings with more flexibility but lower number of elements: + // * Flex: Pair of arrays (2 element sizes, 2 element count) + // + // Encodings: bytes: + // name: | b0 | b1 | b2 | b3 | b4:0-2 | b4:3-4 | b4:5-7 | b5 | b6 | b7 | + // oldies | cap/chksum | 'A' | width | wtype | flags | size | + // Packed | cap/chksum | - | width | flags2 | wtype | flags | enc | size | + // Flex | cap/chksum | w_A + size_A | flags2 | wtype | flags | enc | w_B + size_B | + // + // legend: cap = capacity, chksum = checksum, flags = 3 flag bits, flags2 = 3 additional flag bits + // size = number of elements, w_A = bits per A element, w_B = bits per B element + // size_A = number of A elements, size_B = number of B elements, + // enc = the encoding for the array, corresponding to different memory layouts + // For Flex: w + size is 6 bits for element width, 10 bits for number of elements + // static const int header_size = 8; // Number of bytes used by header @@ -74,98 +129,113 @@ class NodeHeader { return get_data_from_header(const_cast(header)); } - static bool get_is_inner_bptree_node_from_header(const char* header) noexcept + // Helpers for NodeHeader::Type + // handles all header formats + static inline bool get_is_inner_bptree_node_from_header(const char* header) noexcept { typedef unsigned char uchar; const uchar* h = reinterpret_cast(header); return (int(h[4]) & 0x80) != 0; } - static bool get_hasrefs_from_header(const char* header) noexcept + static inline bool get_hasrefs_from_header(const char* header) noexcept { typedef unsigned char uchar; const uchar* h = reinterpret_cast(header); return (int(h[4]) & 0x40) != 0; } - static bool get_context_flag_from_header(const char* header) noexcept + static Type get_type_from_header(const char* header) noexcept + { + if (get_is_inner_bptree_node_from_header(header)) + return type_InnerBptreeNode; + if (get_hasrefs_from_header(header)) + return type_HasRefs; + return type_Normal; + } + + static inline bool get_context_flag_from_header(const char* header) noexcept { typedef unsigned char uchar; const uchar* h = reinterpret_cast(header); return (int(h[4]) & 0x20) != 0; } - - static WidthType get_wtype_from_header(const char* header) noexcept + static inline void set_is_inner_bptree_node_in_header(bool value, char* header) noexcept { typedef unsigned char uchar; - const uchar* h = reinterpret_cast(header); - return WidthType((int(h[4]) & 0x18) >> 3); + uchar* h = reinterpret_cast(header); + h[4] = uchar((int(h[4]) & ~0x80) | int(value) << 7); } - static uint_least8_t get_width_from_header(const char* header) noexcept + static inline void set_hasrefs_in_header(bool value, char* header) noexcept { typedef unsigned char uchar; - const uchar* h = reinterpret_cast(header); - return uint_least8_t((1 << (int(h[4]) & 0x07)) >> 1); + uchar* h = reinterpret_cast(header); + h[4] = uchar((int(h[4]) & ~0x40) | int(value) << 6); } - static size_t get_size_from_header(const char* header) noexcept + static inline void set_context_flag_in_header(bool value, char* header) noexcept { typedef unsigned char uchar; - const uchar* h = reinterpret_cast(header); - return (size_t(h[5]) << 16) + (size_t(h[6]) << 8) + h[7]; + uchar* h = reinterpret_cast(header); + h[4] = uchar((int(h[4]) & ~0x20) | int(value) << 5); } - static size_t get_capacity_from_header(const char* header) noexcept + // Helpers for NodeHeader::WidthType: + // handles all header formats + static inline WidthType get_wtype_from_header(const char* header) noexcept { typedef unsigned char uchar; const uchar* h = reinterpret_cast(header); - return (size_t(h[0]) << 19) + (size_t(h[1]) << 11) + (h[2] << 3); + int h4 = h[4]; + return WidthType((h4 & 0x18) >> 3); } - static Type get_type_from_header(const char* header) noexcept + static inline bool wtype_is_extended(const char* header) noexcept { - if (get_is_inner_bptree_node_from_header(header)) - return type_InnerBptreeNode; - if (get_hasrefs_from_header(header)) - return type_HasRefs; - return type_Normal; + return get_wtype_from_header(header) == wtype_Extend; } - static void set_is_inner_bptree_node_in_header(bool value, char* header) noexcept + static inline void set_wtype_in_header(WidthType value, char* header) noexcept { typedef unsigned char uchar; uchar* h = reinterpret_cast(header); - h[4] = uchar((int(h[4]) & ~0x80) | int(value) << 7); + auto h4 = h[4]; + h4 = (h4 & ~0x18) | int(value) << 3; + h[4] = h4; } - static void set_hasrefs_in_header(bool value, char* header) noexcept + static uint8_t unsigned_to_num_bits(uint64_t value) { - typedef unsigned char uchar; - uchar* h = reinterpret_cast(header); - h[4] = uchar((int(h[4]) & ~0x40) | int(value) << 6); + if constexpr (sizeof(size_t) == sizeof(uint64_t)) + return 1 + log2(static_cast(value)); + uint32_t high = value >> 32; + if (high) + return 33 + log2(high); + uint32_t low = value & 0xFFFFFFFFUL; + if (low) + return 1 + log2(low); + return 0; } - static void set_context_flag_in_header(bool value, char* header) noexcept + static inline uint8_t signed_to_num_bits(int64_t value) { - typedef unsigned char uchar; - uchar* h = reinterpret_cast(header); - h[4] = uchar((int(h[4]) & ~0x20) | int(value) << 5); + if (value >= 0) + return 1 + unsigned_to_num_bits(value); + else + return 1 + unsigned_to_num_bits(~value); // <-- is this correct???? } - static void set_wtype_in_header(WidthType value, char* header) noexcept - { - // Indicates how to calculate size in bytes based on width - // 0: bits (width/8) * size - // 1: multiply width * size - // 2: ignore 1 * size - typedef unsigned char uchar; - uchar* h = reinterpret_cast(header); - h[4] = uchar((int(h[4]) & ~0x18) | int(value) << 3); - } - static void set_width_in_header(int value, char* header) noexcept + // Helper functions for old layouts only: + // Handling width and sizes: + static inline uint_least8_t get_width_from_header(const char* header) noexcept; + + static inline size_t get_size_from_header(const char* header) noexcept; + + static inline void set_width_in_header(size_t value, char* header) noexcept { + REALM_ASSERT_DEBUG(!wtype_is_extended(header)); // Pack width in 3 bits (log2) int w = 0; while (value) { @@ -179,8 +249,9 @@ class NodeHeader { h[4] = uchar((int(h[4]) & ~0x7) | w); } - static void set_size_in_header(size_t value, char* header) noexcept + static inline void set_size_in_header(size_t value, char* header) noexcept { + REALM_ASSERT_DEBUG(!wtype_is_extended(header)); REALM_ASSERT_3(value, <=, max_array_size); typedef unsigned char uchar; uchar* h = reinterpret_cast(header); @@ -189,28 +260,134 @@ class NodeHeader { h[7] = uchar(value & 0x000000FF); } - // Note: There is a copy of this function is test_alloc.cpp + + // Note: The wtype must have been set prior to calling this function + static size_t get_capacity_from_header(const char* header) noexcept + { + if (!wtype_is_extended(header)) { + typedef unsigned char uchar; + const uchar* h = reinterpret_cast(header); + return (size_t(h[0]) << 19) + (size_t(h[1]) << 11) + (h[2] << 3); + } + else { + return reinterpret_cast(header)[0] << 3; + } + } + + // Note: There is a (no longer a correct) copy of this function is test_alloc.cpp + // Note 2: The wtype must have been set prior to calling this function static void set_capacity_in_header(size_t value, char* header) noexcept { - REALM_ASSERT_3(value, <=, (0xffffff << 3)); - typedef unsigned char uchar; - uchar* h = reinterpret_cast(header); - h[0] = uchar((value >> 19) & 0x000000FF); - h[1] = uchar((value >> 11) & 0x000000FF); - h[2] = uchar(value >> 3 & 0x000000FF); + if (!wtype_is_extended(header)) { + REALM_ASSERT_3(value, <=, (0xffffff << 3)); + typedef unsigned char uchar; + uchar* h = reinterpret_cast(header); + h[0] = uchar((value >> 19) & 0x000000FF); + h[1] = uchar((value >> 11) & 0x000000FF); + h[2] = uchar(value >> 3 & 0x000000FF); + } + else { + REALM_ASSERT_DEBUG(value < (65536 << 3)); + REALM_ASSERT_DEBUG((value & 0x7) == 0); + (reinterpret_cast(header))[0] = static_cast(value >> 3); + } } + static size_t get_byte_size_from_header(const char* header) noexcept; - static size_t get_byte_size_from_header(const char* header) noexcept + // ^ First 3 must overlap numerically with corresponding wtype_X enum. + static Encoding get_encoding(const char* header) { - size_t size = get_size_from_header(header); - uint_least8_t width = get_width_from_header(header); - WidthType wtype = get_wtype_from_header(header); - size_t num_bytes = calc_byte_size(wtype, size, width); - return num_bytes; + auto wtype = get_wtype_from_header(header); + if (wtype == wtype_Extend) { + const auto h = reinterpret_cast(header); + int encoding = h[5] + 3; + REALM_ASSERT_DEBUG_EX(encoding >= int(Encoding::Packed) && encoding <= int(Encoding::Flex), encoding); + return static_cast(encoding); + } + return Encoding(int(wtype)); + } + static void set_encoding(char* header, Encoding enc) + { + if (enc < Encoding::Packed) { + set_wtype_in_header(static_cast(enc), header); + } + else { + set_wtype_in_header(wtype_Extend, header); + auto h = reinterpret_cast(header); + h[5] = static_cast(enc) - 3; + } + } + static std::string enc_to_string(Encoding enc) + { + switch (enc) { + case Encoding::WTypMult: + return "Mult"; + case Encoding::WTypIgn: + return "Ign"; + case Encoding::WTypBits: + return "Bits"; + case Encoding::Packed: + return "Pack"; + case Encoding::Flex: + return "Flex"; + default: + return "Err"; + } + } + static std::string header_to_string(const char* header) + { + std::string retval = "{" + enc_to_string(get_encoding(header)) + "}"; + return retval; } +private: + friend class Node; + friend class IntegerCompressor; + // Setting element size for encodings with a single element size: + static void inline set_element_size(char* header, uint8_t bits_per_element, Encoding); + // Getting element size for encodings with a single element size: + static inline uint8_t get_element_size(const char* header, Encoding); + // Used only by flex at this stage. + // Setting element sizes for encodings with two element sizes (called A and B) + static inline void set_elementA_size(char* header, uint8_t bits_per_element); + static inline void set_elementB_size(char* header, uint8_t bits_per_element); + // Getting element sizes for encodings with two element sizes (called A and B) + static inline uint8_t get_elementA_size(const char* header); + static inline uint8_t get_elementB_size(const char* header); + // Setting num of elements for encodings with two element sizes (called A and B) + static inline void set_arrayA_num_elements(char* header, size_t num_elements); + static inline void set_arrayB_num_elements(char* header, size_t num_elements); + // Getting number of elements for encodings with two element sizes (called A and B) + static inline size_t get_arrayA_num_elements(const char* header); + static inline size_t get_arrayB_num_elements(const char* header); + // Getting the number of elements in the array(s). All encodings except Flex have one number of elements. + static inline size_t get_num_elements(const char* header, Encoding); + // Setting the number of elements in the array(s). All encodings except Flex have one number of elements. + static inline void set_num_elements(char* header, size_t num_elements, Encoding); + + static inline size_t calc_size(size_t num_elements); + static inline size_t calc_size(size_t num_elements, uint8_t element_size, Encoding); + static inline size_t calc_size(size_t arrayA_num_elements, size_t arrayB_num_elements, uint8_t elementA_size, + uint8_t elementB_size); + static size_t calc_byte_size(WidthType wtype, size_t size, uint_least8_t width) noexcept { + // the width need to be adjusted to nearest power of two: + if (width > 8) { + if (width > 32) + width = 64; + else if (width > 16) + width = 32; + else + width = 16; + } + else { // width <= 8 + if (width > 4) + width = 8; + else if (width > 2) + width = 4; + // else width is already a power of 2 + } size_t num_bytes = 0; switch (wtype) { case wtype_Bits: { @@ -228,16 +405,361 @@ class NodeHeader { case wtype_Ignore: num_bytes = size; break; + default: { + REALM_ASSERT(false); + break; + } } - + num_bytes += header_size; // Ensure 8-byte alignment num_bytes = (num_bytes + 7) & ~size_t(7); + return num_bytes; + } - num_bytes += header_size; + static inline void set_flags(char* header, uint8_t flags) + { + REALM_ASSERT_DEBUG(flags <= 7); + auto h = reinterpret_cast(header); + h[4] = (h[4] & 0b00011111) | flags << 5; + } + static inline uint8_t get_flags(char* header) + { + auto h = reinterpret_cast(header); + return h[4] >> 5; + } - return num_bytes; + static inline void set_flags2(char* header, uint8_t flags) + { + REALM_ASSERT_DEBUG(flags <= 7); + auto h = reinterpret_cast(header); + h[4] = (h[4] & 0b11111000) | flags; + } + static inline uint8_t get_flags2(char* header) + { + auto h = reinterpret_cast(header); + return h[4] & 0b0111; } }; + +inline void NodeHeader::set_element_size(char* header, uint8_t bits_per_element, Encoding encoding) +{ + switch (encoding) { + case NodeHeader::Encoding::Packed: { + REALM_ASSERT_DEBUG(get_encoding(header) == Encoding::Packed); + REALM_ASSERT_DEBUG(bits_per_element <= 64); + (reinterpret_cast(header)[3] = static_cast(bits_per_element)); + } break; + case NodeHeader::Encoding::WTypBits: { + REALM_ASSERT_DEBUG(bits_per_element <= 64); + // TODO: Only powers of two allowed + // TODO: Optimize + NodeHeader::set_wtype_in_header(wtype_Bits, reinterpret_cast(header)); + NodeHeader::set_width_in_header(bits_per_element, reinterpret_cast(header)); + } break; + case NodeHeader::Encoding::WTypMult: { + REALM_ASSERT_DEBUG(bits_per_element <= 64); + REALM_ASSERT_DEBUG((bits_per_element & 0x7) == 0); + // TODO: Only powers of two allowed + // TODO: Optimize + NodeHeader::set_wtype_in_header(wtype_Multiply, reinterpret_cast(header)); + NodeHeader::set_width_in_header(bits_per_element >> 3, reinterpret_cast(header)); + } break; + default: + REALM_UNREACHABLE(); + } +} + +inline uint8_t NodeHeader::get_element_size(const char* header, Encoding encoding) +{ + switch (encoding) { + case NodeHeader::Encoding::Packed: { + REALM_ASSERT_DEBUG(get_encoding(header) == Encoding::Packed); + const auto bits_per_element = (reinterpret_cast(header))[3]; + REALM_ASSERT_DEBUG(bits_per_element <= 64); + return bits_per_element; + } break; + case NodeHeader::Encoding::WTypBits: { + REALM_ASSERT_DEBUG(get_wtype_from_header(header) == wtype_Bits); + const auto bits_per_element = NodeHeader::get_width_from_header(reinterpret_cast(header)); + REALM_ASSERT_DEBUG(bits_per_element <= 64); + return bits_per_element; + } break; + case NodeHeader::Encoding::WTypMult: { + REALM_ASSERT_DEBUG(get_wtype_from_header(header) == wtype_Multiply); + const auto bits_per_element = NodeHeader::get_width_from_header(reinterpret_cast(header)) + << 3; + REALM_ASSERT_DEBUG(bits_per_element <= 64); + return bits_per_element; + } break; + default: + REALM_UNREACHABLE(); + } +} + +inline void NodeHeader::set_elementA_size(char* header, uint8_t bits_per_element) +{ + // we're a bit low on bits for the Flex encoding, so we need to squeeze stuff + REALM_ASSERT_DEBUG(get_encoding(header) == Encoding::Flex); + REALM_ASSERT_DEBUG(bits_per_element <= 64); + REALM_ASSERT_DEBUG(bits_per_element > 0); + uint16_t word = (reinterpret_cast(header))[1]; + word &= ~(0b111111 << 10); + // we only have 6 bits, so store values in range 1-64 as 0-63 + word |= (bits_per_element - 1) << 10; + (reinterpret_cast(header))[1] = word; +} + +inline void NodeHeader::set_elementB_size(char* header, uint8_t bits_per_element) +{ + // we're a bit low on bits for the Flex encoding, so we need to squeeze stuff + REALM_ASSERT_DEBUG(get_encoding(header) == Encoding::Flex); + REALM_ASSERT_DEBUG(bits_per_element <= 64); + REALM_ASSERT_DEBUG(bits_per_element > 0); + uint16_t word = (reinterpret_cast(header))[3]; + word &= ~(0b111111 << 10); + // we only have 6 bits, so store values in range 1-64 as 0-63 + word |= (bits_per_element - 1) << 10; + (reinterpret_cast(header))[3] = word; +} + +inline uint8_t NodeHeader::get_elementA_size(const char* header) +{ + const auto encoding = get_encoding(header); + REALM_ASSERT_DEBUG(encoding == Encoding::Flex); + uint16_t word = (reinterpret_cast(header))[1]; + auto bits_per_element = (word >> 10) & 0b111111; + // we only have 6 bits, so store values in range 1-64 as 0-63 + // this means that Flex cannot support element sizes of 0 + bits_per_element++; + REALM_ASSERT_DEBUG(bits_per_element <= 64); + REALM_ASSERT_DEBUG(bits_per_element > 0); + return bits_per_element; +} + +inline uint8_t NodeHeader::get_elementB_size(const char* header) +{ + REALM_ASSERT_DEBUG(get_encoding(header) == Encoding::Flex); + uint16_t word = (reinterpret_cast(header))[3]; + auto bits_per_element = (word >> 10) & 0b111111; + // same as above + bits_per_element++; + REALM_ASSERT_DEBUG(bits_per_element <= 64); + REALM_ASSERT_DEBUG(bits_per_element > 0); + return bits_per_element; +} + +inline size_t NodeHeader::get_num_elements(const char* header, Encoding encoding) +{ + switch (encoding) { + case NodeHeader::Encoding::Packed: + REALM_ASSERT_DEBUG(get_encoding(header) == Encoding::Packed); + return (reinterpret_cast(header))[3]; + break; + case NodeHeader::Encoding::WTypBits: + case NodeHeader::Encoding::WTypMult: + case NodeHeader::Encoding::WTypIgn: { + REALM_ASSERT_DEBUG(get_wtype_from_header(header) != wtype_Extend); + typedef unsigned char uchar; + const uchar* h = reinterpret_cast(header); + return (size_t(h[5]) << 16) + (size_t(h[6]) << 8) + h[7]; + break; + } + case NodeHeader::Encoding::Flex: + return get_arrayB_num_elements(header); + break; + default: + printf("Encoding %d\n", int(encoding)); + REALM_UNREACHABLE(); + } +} + +inline void NodeHeader::set_num_elements(char* header, size_t num_elements, Encoding encoding) +{ + switch (encoding) { + case NodeHeader::Encoding::Packed: { + REALM_ASSERT_DEBUG(get_encoding(header) == Encoding::Packed); + REALM_ASSERT_DEBUG(num_elements < 0x10000); + (reinterpret_cast(header))[3] = static_cast(num_elements); + } break; + case NodeHeader::Encoding::WTypBits: { + REALM_ASSERT_DEBUG(get_encoding(header) == Encoding::WTypBits); + NodeHeader::set_wtype_in_header(wtype_Bits, header); + NodeHeader::set_size_in_header(num_elements, header); + } break; + case NodeHeader::Encoding::WTypMult: { + REALM_ASSERT_DEBUG(get_encoding(header) == Encoding::WTypMult); + NodeHeader::set_wtype_in_header(wtype_Multiply, header); + NodeHeader::set_size_in_header(num_elements, header); + } break; + case NodeHeader::Encoding::WTypIgn: { + REALM_ASSERT_DEBUG(get_encoding(header) == Encoding::WTypIgn); + NodeHeader::set_wtype_in_header(wtype_Ignore, header); + NodeHeader::set_size_in_header(num_elements, header); + } break; + default: + REALM_UNREACHABLE(); + } +} + +inline void NodeHeader::set_arrayA_num_elements(char* header, size_t num_elements) +{ + REALM_ASSERT_DEBUG(get_encoding(header) == Encoding::Flex); + REALM_ASSERT_DEBUG(num_elements < 0b10000000000); // 10 bits + uint16_t word = (reinterpret_cast(header))[1]; + word &= ~(0b1111111111 << 10); + word |= num_elements << 10; + (reinterpret_cast(header))[1] = word; +} + +inline void NodeHeader::set_arrayB_num_elements(char* header, size_t num_elements) +{ + REALM_ASSERT_DEBUG(get_encoding(header) == Encoding::Flex); + REALM_ASSERT_DEBUG(num_elements < 0b10000000000); // 10 bits + uint16_t word = (reinterpret_cast(header))[3]; + word &= ~(0b1111111111 << 10); + word |= num_elements << 10; + (reinterpret_cast(header))[3] = word; +} + +inline size_t NodeHeader::get_arrayA_num_elements(const char* header) +{ + const auto encoding = get_encoding(header); + REALM_ASSERT_DEBUG(encoding == Encoding::Flex); + const uint16_t word = (reinterpret_cast(header))[1]; + const auto num_elements = word & 0b1111111111; + return num_elements; +} + +inline size_t NodeHeader::get_arrayB_num_elements(const char* header) +{ + REALM_ASSERT_DEBUG(get_encoding(header) == Encoding::Flex); + const uint16_t word = (reinterpret_cast(header))[3]; + const auto num_elements = word & 0b1111111111; + return num_elements; +} + +inline size_t NodeHeader::calc_size(size_t num_elements) +{ + return calc_byte_size(wtype_Ignore, num_elements, 0); +} + +inline size_t NodeHeader::calc_size(size_t num_elements, uint8_t element_size, Encoding encoding) +{ + using Encoding = NodeHeader::Encoding; + switch (encoding) { + case Encoding::Packed: + return NodeHeader::header_size + align_bits_to8(num_elements * element_size); + case Encoding::WTypBits: + return calc_byte_size(wtype_Bits, num_elements, static_cast(element_size)); + case Encoding::WTypMult: + return calc_byte_size(wtype_Multiply, num_elements, static_cast(element_size)); + case Encoding::WTypIgn: + return calc_byte_size(wtype_Ignore, num_elements, 0); + default: + REALM_UNREACHABLE(); + } +} + +inline size_t NodeHeader::calc_size(size_t arrayA_num_elements, size_t arrayB_num_elements, uint8_t elementA_size, + uint8_t elementB_size) +{ + return NodeHeader::header_size + + align_bits_to8(arrayA_num_elements * elementA_size + arrayB_num_elements * elementB_size); +} + +size_t inline NodeHeader::get_byte_size_from_header(const char* header) noexcept +{ + const auto h = header; + + const auto encoding = get_encoding(h); + const auto size = get_num_elements(h, encoding); + switch (encoding) { + case Encoding::WTypBits: + case Encoding::WTypIgn: + case Encoding::WTypMult: { + const auto width = get_width_from_header(header); + return calc_byte_size(WidthType(int(encoding)), size, static_cast(width)); + } + case Encoding::Packed: + return NodeHeader::header_size + align_bits_to8(size * get_element_size(h, encoding)); + case Encoding::Flex: + return NodeHeader::header_size + align_bits_to8(get_arrayA_num_elements(h) * get_elementA_size(h) + + get_arrayB_num_elements(h) * get_elementB_size(h)); + default: + REALM_UNREACHABLE(); + } +} + + +uint_least8_t inline NodeHeader::get_width_from_header(const char* header) noexcept +{ + REALM_ASSERT_DEBUG(!wtype_is_extended(header)); + typedef unsigned char uchar; + const uchar* h = reinterpret_cast(header); + return uint_least8_t((1 << (int(h[4]) & 0x07)) >> 1); +} + +// A little helper: +size_t inline NodeHeader::get_size_from_header(const char* header) noexcept +{ + return get_num_elements(header, get_encoding(header)); +} + } // namespace realm + +namespace { + +static inline void init_header(char* header, realm::NodeHeader::Encoding enc, uint8_t flags, uint8_t bits_pr_elem, + size_t num_elems) +{ + using Encoding = realm::NodeHeader::Encoding; + std::fill(header, header + realm::NodeHeader::header_size, 0); + const auto hb = reinterpret_cast(header); + REALM_ASSERT_DEBUG(enc <= Encoding::Packed); + if (enc < Encoding::Packed) { + // old layout + uint8_t wtype = static_cast(enc); + hb[4] = (flags << 5) | (wtype << 3); + if (enc == Encoding::WTypBits) + realm::NodeHeader::set_width_in_header(bits_pr_elem, reinterpret_cast(header)); + else + realm::NodeHeader::set_width_in_header(bits_pr_elem >> 3, reinterpret_cast(header)); + realm::NodeHeader::set_size_in_header(num_elems, reinterpret_cast(header)); + } + else if (enc == Encoding::Packed) { + hb[2] = 0; + hb[3] = static_cast(bits_pr_elem); + hb[4] = (flags << 5) | (realm::NodeHeader::wtype_Extend << 3); + hb[5] = static_cast(enc) - realm::NodeHeader::wtype_Extend; + const auto hw = reinterpret_cast(header); + hw[3] = static_cast(num_elems); + } +} + +// init the header for flex array. Passing A bit width and size (values) and B bit width and size (indices) +static inline void init_header(char* header, realm::NodeHeader::Encoding enc, uint8_t flags, uint8_t bits_pr_elemA, + uint8_t bits_pr_elemB, size_t num_elemsA, size_t num_elemsB) +{ + std::fill(header, header + realm::NodeHeader::header_size, 0); + const auto hb = reinterpret_cast(header); + REALM_ASSERT_DEBUG(enc == realm::NodeHeader::Encoding::Flex); + REALM_ASSERT_DEBUG(flags < 8); + hb[4] = (flags << 5) | (realm::NodeHeader::wtype_Extend << 3); + hb[5] = + static_cast(realm::NodeHeader::Encoding::Flex) - static_cast(realm::NodeHeader::Encoding::Packed); + const auto hw = reinterpret_cast(header); + REALM_ASSERT_DEBUG(bits_pr_elemA > 0); + REALM_ASSERT_DEBUG(bits_pr_elemB > 0); + REALM_ASSERT_DEBUG(bits_pr_elemA <= 64); + REALM_ASSERT_DEBUG(bits_pr_elemB <= 64); + REALM_ASSERT_DEBUG(num_elemsA < 1024); + REALM_ASSERT_DEBUG(num_elemsB < 1024); + hw[1] = static_cast(((bits_pr_elemA - 1) << 10) | num_elemsA); + hw[3] = static_cast(((bits_pr_elemB - 1) << 10) | num_elemsB); + REALM_ASSERT_DEBUG(realm::NodeHeader::get_encoding(header) == realm::NodeHeader::Encoding::Flex); +} +} // namespace + + #endif /* REALM_NODE_HEADER_HPP */ diff --git a/src/realm/obj.cpp b/src/realm/obj.cpp index 4dba1de45ce..a569957f990 100644 --- a/src/realm/obj.cpp +++ b/src/realm/obj.cpp @@ -549,12 +549,9 @@ int64_t Obj::_get(ColKey::Idx col_ndx) const if (current_version != m_storage_version) { update(); } - ref_type ref = to_ref(Array::get(m_mem.get_addr(), col_ndx.val + 1)); char* header = alloc.translate(ref); - int width = Array::get_width_from_header(header); - char* data = Array::get_data_from_header(header); - REALM_TEMPEX(return get_direct, width, (data, m_row_ndx)); + return Array::get(header, m_row_ndx); } template <> @@ -634,6 +631,37 @@ BinaryData Obj::_get(ColKey::Idx col_ndx) const return ArrayBinary::get(alloc.translate(ref), m_row_ndx, alloc); } +bool Obj::has_property(StringData prop_name) const +{ + if (m_table->get_column_key(prop_name)) + return true; + if (auto ck = m_table->m_additional_prop_col) { + Dictionary dict(*this, ck); + return dict.contains(prop_name); + } + return false; +} + +bool Obj::has_schema_property(StringData prop_name) const +{ + if (m_table->get_column_key(prop_name)) + return true; + return false; +} + +std::vector Obj::get_additional_properties() const +{ + std::vector ret; + + if (auto ck = m_table->m_additional_prop_col) { + Dictionary dict(*this, ck); + dict.for_all_keys([&ret](StringData key) { + ret.push_back(key); + }); + } + return ret; +} + Mixed Obj::get_any(ColKey col_key) const { m_table->check_column(col_key); @@ -679,6 +707,19 @@ Mixed Obj::get_any(ColKey col_key) const return {}; } +Mixed Obj::get_additional_prop(StringData prop_name) const +{ + if (auto ck = m_table->m_additional_prop_col) { + Dictionary dict(*this, ck); + if (auto val = dict.try_get(prop_name)) { + return *val; + } + } + throw InvalidArgument(ErrorCodes::InvalidProperty, + util::format("Property '%1.%2' does not exist", m_table->get_class_name(), prop_name)); + return {}; +} + Mixed Obj::get_primary_key() const { auto col = m_table->get_primary_key_column(); @@ -1110,7 +1151,8 @@ StablePath Obj::get_stable_path() const noexcept void Obj::add_index(Path& path, const CollectionParent::Index& index) const { if (path.empty()) { - path.emplace_back(get_table()->get_column_key(index)); + auto ck = m_table->get_column_key(index); + path.emplace_back(ck); } else { StringData col_name = get_table()->get_column_name(index); @@ -1232,6 +1274,32 @@ Obj& Obj::set(ColKey col_key, Mixed value, bool is_default) return *this; } +Obj& Obj::erase_additional_prop(StringData prop_name) +{ + bool erased = false; + if (auto ck = m_table->m_additional_prop_col) { + Dictionary dict(*this, ck); + erased = dict.try_erase(prop_name); + } + if (!erased) { + throw InvalidArgument(ErrorCodes::InvalidProperty, util::format("Could not erase property: %1", prop_name)); + } + return *this; +} + +Obj& Obj::set_additional_prop(StringData prop_name, const Mixed& value) +{ + if (auto ck = m_table->m_additional_prop_col) { + Dictionary dict(*this, ck); + dict.insert(prop_name, value); + } + else { + throw InvalidArgument(ErrorCodes::InvalidProperty, + util::format("Property '%1.%2' does not exist", m_table->get_class_name(), prop_name)); + } + return *this; +} + Obj& Obj::set_any(ColKey col_key, Mixed value, bool is_default) { if (value.is_null()) { @@ -1986,7 +2054,6 @@ Dictionary Obj::get_dictionary(ColKey col_key) const Obj& Obj::set_collection(ColKey col_key, CollectionType type) { - REALM_ASSERT(col_key.get_type() == col_type_Mixed); if ((col_key.is_dictionary() && type == CollectionType::Dictionary) || (col_key.is_list() && type == CollectionType::List)) { return *this; @@ -1994,11 +2061,34 @@ Obj& Obj::set_collection(ColKey col_key, CollectionType type) if (type == CollectionType::Set) { throw IllegalOperation("Set nested in Mixed is not supported"); } + if (col_key.get_type() != col_type_Mixed) { + throw IllegalOperation("Collection can only be nested in Mixed"); + } set(col_key, Mixed(0, type)); return *this; } +Obj& Obj::set_collection(StringData prop_name, CollectionType type) +{ + if (auto ck = get_column_key(prop_name)) { + return set_collection(ck, type); + } + return set_additional_collection(prop_name, type); +} + +Obj& Obj::set_additional_collection(StringData prop_name, CollectionType type) +{ + if (auto ck = m_table->m_additional_prop_col) { + Dictionary dict(*this, ck); + dict.insert_collection(prop_name, type); + } + else { + throw InvalidArgument(ErrorCodes::InvalidProperty, util::format("Property not found: %1", prop_name)); + } + return *this; +} + DictionaryPtr Obj::get_dictionary_ptr(ColKey col_key) const { return std::make_shared(get_dictionary(col_key)); @@ -2014,14 +2104,33 @@ Dictionary Obj::get_dictionary(StringData col_name) const return get_dictionary(get_column_key(col_name)); } -CollectionPtr Obj::get_collection_ptr(const Path& path) const +CollectionBasePtr Obj::get_collection_ptr(const Path& path) const { REALM_ASSERT(path.size() > 0); // First element in path must be column name auto col_key = path[0].is_col_key() ? path[0].get_col_key() : m_table->get_column_key(path[0].get_key()); - REALM_ASSERT(col_key); + + CollectionBasePtr collection; size_t level = 1; - CollectionBasePtr collection = get_collection_ptr(col_key); + if (col_key) { + collection = get_collection_ptr(col_key); + } + else { + if (auto ck = m_table->m_additional_prop_col) { + auto prop_name = path[0].get_key(); + Dictionary dict(*this, ck); + auto ref = dict.get(prop_name); + if (ref.is_type(type_List)) { + collection = dict.get_list(prop_name); + } + else if (ref.is_type(type_Dictionary)) { + collection = dict.get_dictionary(prop_name); + } + else { + throw InvalidArgument("Wrong path"); + } + } + } while (level < path.size()) { auto& path_elem = path[level]; @@ -2047,7 +2156,7 @@ CollectionPtr Obj::get_collection_ptr(const Path& path) const return collection; } -CollectionPtr Obj::get_collection_by_stable_path(const StablePath& path) const +CollectionBasePtr Obj::get_collection_by_stable_path(const StablePath& path) const { // First element in path is phony column key ColKey col_key = m_table->get_column_key(path[0]); @@ -2111,7 +2220,7 @@ CollectionBasePtr Obj::get_collection_ptr(ColKey col_key) const CollectionBasePtr Obj::get_collection_ptr(StringData col_name) const { - return get_collection_ptr(get_column_key(col_name)); + return get_collection_ptr(Path{{col_name}}); } LinkCollectionPtr Obj::get_linkcollection_ptr(ColKey col_key) const diff --git a/src/realm/obj.hpp b/src/realm/obj.hpp index 67c82a0cada..96525dd8200 100644 --- a/src/realm/obj.hpp +++ b/src/realm/obj.hpp @@ -117,17 +117,30 @@ class Obj { template U get(ColKey col_key) const; + bool has_property(StringData prop_name) const; + bool has_schema_property(StringData prop_name) const; + + std::vector get_additional_properties() const; + Mixed get_any(ColKey col_key) const; Mixed get_any(StringData col_name) const { - return get_any(get_column_key(col_name)); + if (auto ck = get_column_key(col_name)) { + return get_any(ck); + } + return get_additional_prop(col_name); } + Mixed get_additional_prop(StringData col_name) const; + Mixed get_primary_key() const; template U get(StringData col_name) const { - return get(get_column_key(col_name)); + if (auto ck = get_column_key(col_name)) { + return get(ck); + } + return get_additional_prop(col_name).get(); } bool is_unresolved(ColKey col_key) const; @@ -187,17 +200,26 @@ class Obj { // default state. If the object does not exist, create a // new object and link it. (To Be Implemented) Obj clear_linked_object(ColKey col_key); + + Obj& erase_additional_prop(StringData prop_name); Obj& set_any(ColKey col_key, Mixed value, bool is_default = false); Obj& set_any(StringData col_name, Mixed value, bool is_default = false) { - return set_any(get_column_key(col_name), value, is_default); + if (auto ck = get_column_key(col_name)) { + return set_any(ck, value, is_default); + } + return set_additional_prop(col_name, value); } template Obj& set(StringData col_name, U value, bool is_default = false) { - return set(get_column_key(col_name), value, is_default); + if (auto ck = get_column_key(col_name)) { + return set(ck, value, is_default); + } + return set_additional_prop(col_name, Mixed(value)); } + Obj& set_additional_prop(StringData prop_name, const Mixed& value); Obj& set_null(ColKey col_key, bool is_default = false); Obj& set_null(StringData col_name, bool is_default = false) @@ -206,6 +228,7 @@ class Obj { } Obj& set_json(ColKey col_key, StringData json); + Obj& add_int(ColKey col_key, int64_t value); Obj& add_int(StringData col_name, int64_t value) { @@ -248,6 +271,11 @@ class Obj { { return std::dynamic_pointer_cast>(get_collection_ptr(path)); } + template + std::shared_ptr> get_list_ptr(StringData prop_name) const + { + return get_list_ptr(Path{prop_name}); + } template Lst get_list(StringData col_name) const @@ -285,17 +313,24 @@ class Obj { LnkSet get_linkset(StringData col_name) const; LnkSetPtr get_linkset_ptr(ColKey col_key) const; SetBasePtr get_setbase_ptr(ColKey col_key) const; + Dictionary get_dictionary(ColKey col_key) const; Dictionary get_dictionary(StringData col_name) const; Obj& set_collection(ColKey col_key, CollectionType type); + Obj& set_collection(StringData, CollectionType type); + Obj& set_additional_collection(StringData, CollectionType type); DictionaryPtr get_dictionary_ptr(ColKey col_key) const; DictionaryPtr get_dictionary_ptr(const Path& path) const; + DictionaryPtr get_dictionary_ptr(StringData prop_name) const + { + return get_dictionary_ptr(Path{prop_name}); + } CollectionBasePtr get_collection_ptr(ColKey col_key) const; CollectionBasePtr get_collection_ptr(StringData col_name) const; - CollectionPtr get_collection_ptr(const Path& path) const; - CollectionPtr get_collection_by_stable_path(const StablePath& path) const; + CollectionBasePtr get_collection_ptr(const Path& path) const; + CollectionBasePtr get_collection_by_stable_path(const StablePath& path) const; LinkCollectionPtr get_linkcollection_ptr(ColKey col_key) const; void assign_pk_and_backlinks(Obj& other); diff --git a/src/realm/object-store/c_api/config.cpp b/src/realm/object-store/c_api/config.cpp index 7b1b00498c2..2d76bd9368f 100644 --- a/src/realm/object-store/c_api/config.cpp +++ b/src/realm/object-store/c_api/config.cpp @@ -243,3 +243,8 @@ RLM_API void realm_config_set_automatic_backlink_handling(realm_config_t* realm_ { realm_config->automatically_handle_backlinks_in_migrations = enable_automatic_handling; } + +RLM_API void realm_config_set_flexible_schema(realm_config_t* realm_config, bool flexible_schema) noexcept +{ + realm_config->flexible_schema = flexible_schema; +} diff --git a/src/realm/object-store/c_api/object.cpp b/src/realm/object-store/c_api/object.cpp index 44bc55ab548..4971a818c66 100644 --- a/src/realm/object-store/c_api/object.cpp +++ b/src/realm/object-store/c_api/object.cpp @@ -253,31 +253,31 @@ RLM_API realm_object_t* realm_object_from_thread_safe_reference(const realm_t* r }); } -RLM_API bool realm_get_value(const realm_object_t* obj, realm_property_key_t col, realm_value_t* out_value) +RLM_API bool realm_get_value(const realm_object_t* object, realm_property_key_t col, realm_value_t* out_value) { - return realm_get_values(obj, 1, &col, out_value); + return realm_get_values(object, 1, &col, out_value); } -RLM_API bool realm_get_values(const realm_object_t* obj, size_t num_values, const realm_property_key_t* properties, +RLM_API bool realm_get_values(const realm_object_t* object, size_t num_values, const realm_property_key_t* properties, realm_value_t* out_values) { return wrap_err([&]() { - obj->verify_attached(); + object->verify_attached(); - auto o = obj->get_obj(); + auto obj = object->get_obj(); for (size_t i = 0; i < num_values; ++i) { auto col_key = ColKey(properties[i]); if (col_key.is_collection()) { - auto table = o.get_table(); - auto& schema = schema_for_table(obj->get_realm(), table->get_key()); + auto table = obj.get_table(); + auto& schema = schema_for_table(object->get_realm(), table->get_key()); throw PropertyTypeMismatch{schema.name, table->get_column_name(col_key)}; } - auto val = o.get_any(col_key); + auto val = obj.get_any(col_key); if (out_values) { - auto converted = objkey_to_typed_link(val, col_key, *o.get_table()); + auto converted = objkey_to_typed_link(val, col_key, *obj.get_table()); out_values[i] = to_capi(converted); } } @@ -286,18 +286,34 @@ RLM_API bool realm_get_values(const realm_object_t* obj, size_t num_values, cons }); } -RLM_API bool realm_set_value(realm_object_t* obj, realm_property_key_t col, realm_value_t new_value, bool is_default) +RLM_API bool realm_get_value_by_name(const realm_object_t* object, const char* property_name, + realm_value_t* out_value) +{ + return wrap_err([&]() { + object->verify_attached(); + + auto obj = object->get_obj(); + auto val = obj.get_any(property_name); + if (out_value) { + *out_value = to_capi(val); + } + return true; + }); +} + +RLM_API bool realm_set_value(realm_object_t* object, realm_property_key_t col, realm_value_t new_value, + bool is_default) { - return realm_set_values(obj, 1, &col, &new_value, is_default); + return realm_set_values(object, 1, &col, &new_value, is_default); } -RLM_API bool realm_set_values(realm_object_t* obj, size_t num_values, const realm_property_key_t* properties, +RLM_API bool realm_set_values(realm_object_t* object, size_t num_values, const realm_property_key_t* properties, const realm_value_t* values, bool is_default) { return wrap_err([&]() { - obj->verify_attached(); - auto o = obj->get_obj(); - auto table = o.get_table(); + object->verify_attached(); + auto obj = object->get_obj(); + auto table = obj.get_table(); // Perform validation up front to avoid partial updates. This is // unlikely to incur performance overhead because the object itself is @@ -308,12 +324,12 @@ RLM_API bool realm_set_values(realm_object_t* obj, size_t num_values, const real table->check_column(col_key); if (col_key.is_collection()) { - auto& schema = schema_for_table(obj->get_realm(), table->get_key()); + auto& schema = schema_for_table(object->get_realm(), table->get_key()); throw PropertyTypeMismatch{schema.name, table->get_column_name(col_key)}; } auto val = from_capi(values[i]); - check_value_assignable(obj->get_realm(), *table, col_key, val); + check_value_assignable(object->get_realm(), *table, col_key, val); } // Actually write the properties. @@ -321,36 +337,94 @@ RLM_API bool realm_set_values(realm_object_t* obj, size_t num_values, const real for (size_t i = 0; i < num_values; ++i) { auto col_key = ColKey(properties[i]); auto val = from_capi(values[i]); - o.set_any(col_key, val, is_default); + obj.set_any(col_key, val, is_default); } return true; }); } -RLM_API bool realm_set_json(realm_object_t* obj, realm_property_key_t col, const char* json_string) +RLM_API bool realm_set_value_by_name(realm_object_t* object, const char* property_name, realm_value_t new_value) { return wrap_err([&]() { - obj->verify_attached(); - auto o = obj->get_obj(); + object->verify_attached(); + auto obj = object->get_obj(); + obj.set_any(property_name, from_capi(new_value)); + return true; + }); +} + +RLM_API bool realm_has_property(realm_object_t* object, const char* property_name, bool* out_has_property) +{ + return wrap_err([&]() { + object->verify_attached(); + if (out_has_property) { + auto obj = object->get_obj(); + *out_has_property = obj.has_property(property_name); + } + return true; + }); +} + +RLM_API void realm_get_additional_properties(realm_object_t* object, const char** out_prop_names, size_t max, + size_t* out_n) +{ + size_t copied = 0; + wrap_err([&]() { + object->verify_attached(); + auto obj = object->get_obj(); + auto vec = obj.get_additional_properties(); + copied = vec.size(); + if (out_prop_names) { + if (max < copied) { + copied = max; + } + auto it = vec.begin(); + auto to_copy = copied; + while (to_copy--) { + *out_prop_names++ = (*it++).data(); + } + } + return true; + }); + if (out_n) { + *out_n = copied; + } +} + +RLM_API bool realm_erase_additional_property(realm_object_t* object, const char* property_name) +{ + return wrap_err([&]() { + object->verify_attached(); + auto obj = object->get_obj(); + obj.erase_additional_prop(property_name); + return true; + }); +} + +RLM_API bool realm_set_json(realm_object_t* object, realm_property_key_t col, const char* json_string) +{ + return wrap_err([&]() { + object->verify_attached(); + auto obj = object->get_obj(); ColKey col_key(col); if (col_key.get_type() != col_type_Mixed) { - auto table = o.get_table(); - auto& schema = schema_for_table(obj->get_realm(), table->get_key()); + auto table = obj.get_table(); + auto& schema = schema_for_table(object->get_realm(), table->get_key()); throw PropertyTypeMismatch{schema.name, table->get_column_name(col_key)}; } - o.set_json(ColKey(col), json_string); + obj.set_json(ColKey(col), json_string); return true; }); } -RLM_API realm_object_t* realm_set_embedded(realm_object_t* obj, realm_property_key_t col) +RLM_API realm_object_t* realm_set_embedded(realm_object_t* object, realm_property_key_t col) { return wrap_err([&]() { - obj->verify_attached(); - auto& o = obj->get_obj(); - return new realm_object_t({obj->get_realm(), o.create_and_set_linked_object(ColKey(col))}); + object->verify_attached(); + auto& obj = object->get_obj(); + return new realm_object_t({object->get_realm(), obj.create_and_set_linked_object(ColKey(col))}); }); } @@ -380,12 +454,35 @@ RLM_API realm_dictionary_t* realm_set_dictionary(realm_object_t* object, realm_p }); } -RLM_API realm_object_t* realm_get_linked_object(realm_object_t* obj, realm_property_key_t col) +RLM_API realm_list_t* realm_set_list_by_name(realm_object_t* object, const char* property_name) { return wrap_err([&]() { - obj->verify_attached(); - const auto& o = obj->get_obj().get_linked_object(ColKey(col)); - return o ? new realm_object_t({obj->get_realm(), o}) : nullptr; + object->verify_attached(); + + auto& obj = object->get_obj(); + obj.set_collection(property_name, CollectionType::List); + return new realm_list_t{List{object->get_realm(), obj.get_list_ptr(property_name)}}; + }); +} + +RLM_API realm_dictionary_t* realm_set_dictionary_by_name(realm_object_t* object, const char* property_name) +{ + return wrap_err([&]() { + object->verify_attached(); + + auto& obj = object->get_obj(); + obj.set_collection(property_name, CollectionType::Dictionary); + return new realm_dictionary_t{ + object_store::Dictionary{object->get_realm(), obj.get_dictionary_ptr(property_name)}}; + }); +} + +RLM_API realm_object_t* realm_get_linked_object(realm_object_t* object, realm_property_key_t col) +{ + return wrap_err([&]() { + object->verify_attached(); + const auto& obj = object->get_obj().get_linked_object(ColKey(col)); + return obj ? new realm_object_t({object->get_realm(), obj}) : nullptr; }); } @@ -408,6 +505,20 @@ RLM_API realm_list_t* realm_get_list(realm_object_t* object, realm_property_key_ }); } +RLM_API realm_list_t* realm_get_list_by_name(realm_object_t* object, const char* prop_name) +{ + return wrap_err([&]() -> realm_list_t* { + object->verify_attached(); + + const auto& obj = object->get_obj(); + auto collection = obj.get_collection_ptr(StringData(prop_name)); + if (collection->get_collection_type() == CollectionType::List) { + return new realm_list_t{List{object->get_realm(), std::move(collection)}}; + } + return nullptr; + }); +} + RLM_API realm_set_t* realm_get_set(realm_object_t* object, realm_property_key_t key) { return wrap_err([&]() { @@ -445,6 +556,20 @@ RLM_API realm_dictionary_t* realm_get_dictionary(realm_object_t* object, realm_p }); } +RLM_API realm_dictionary_t* realm_get_dictionary_by_name(realm_object_t* object, const char* prop_name) +{ + return wrap_err([&]() -> realm_dictionary_t* { + object->verify_attached(); + + const auto& obj = object->get_obj(); + auto collection = obj.get_collection_ptr(StringData(prop_name)); + if (collection->get_collection_type() == CollectionType::Dictionary) { + return new realm_dictionary_t{object_store::Dictionary{object->get_realm(), std::move(collection)}}; + } + return nullptr; + }); +} + RLM_API char* realm_object_to_string(realm_object_t* object) { return wrap_err([&]() { diff --git a/src/realm/object-store/collection.cpp b/src/realm/object-store/collection.cpp index 6efe683844a..ab42eca58be 100644 --- a/src/realm/object-store/collection.cpp +++ b/src/realm/object-store/collection.cpp @@ -42,6 +42,11 @@ Collection::Collection(const Object& parent_obj, const Property* prop) { } +Collection::Collection(std::shared_ptr r, const Obj& parent_obj, const StringData prop_name) + : Collection(std::shared_ptr(r), parent_obj.get_collection_ptr(prop_name), PropertyType::Mixed) +{ +} + Collection::Collection(std::shared_ptr r, const Obj& parent_obj, ColKey col) : Collection(std::move(r), parent_obj.get_collection_ptr(col), ObjectSchema::from_core_type(col) & ~PropertyType::Collection) diff --git a/src/realm/object-store/collection.hpp b/src/realm/object-store/collection.hpp index e809ee2bab1..649155a6e6e 100644 --- a/src/realm/object-store/collection.hpp +++ b/src/realm/object-store/collection.hpp @@ -45,6 +45,7 @@ class Collection { Collection(std::shared_ptr r, const Obj& parent_obj, ColKey col); Collection(std::shared_ptr r, const CollectionBase& coll); Collection(std::shared_ptr r, CollectionBasePtr coll); + Collection(std::shared_ptr r, const Obj& parent_obj, const StringData prop_name); const std::shared_ptr& get_realm() const { diff --git a/src/realm/object-store/impl/object_accessor_impl.hpp b/src/realm/object-store/impl/object_accessor_impl.hpp index c7371589335..0093a8e5117 100644 --- a/src/realm/object-store/impl/object_accessor_impl.hpp +++ b/src/realm/object-store/impl/object_accessor_impl.hpp @@ -70,15 +70,15 @@ class CppContext { // value present. The property is identified both by the name of the // property and its index within the ObjectScehma's persisted_properties // array. - util::Optional value_for_property(std::any& dict, const Property& prop, + util::Optional value_for_property(std::any& dict, const std::string& name, size_t /* property_index */) const { #if REALM_ENABLE_GEOSPATIAL if (auto geo = std::any_cast(&dict)) { - if (prop.name == Geospatial::c_geo_point_type_col_name) { + if (name == Geospatial::c_geo_point_type_col_name) { return geo->get_type_string(); } - else if (prop.name == Geospatial::c_geo_point_coords_col_name) { + else if (name == Geospatial::c_geo_point_coords_col_name) { std::vector coords; auto&& point = geo->get(); // throws coords.push_back(point.longitude); @@ -88,11 +88,11 @@ class CppContext { } return coords; } - REALM_ASSERT_EX(false, prop.name); // unexpected property type + REALM_ASSERT_EX(false, name); // unexpected property type } #endif auto const& v = util::any_cast(dict); - auto it = v.find(prop.name); + auto it = v.find(name); return it == v.end() ? util::none : util::make_optional(it->second); } @@ -118,6 +118,20 @@ class CppContext { template void enumerate_dictionary(std::any& value, Func&& fn) { +#if REALM_ENABLE_GEOSPATIAL + if (auto geo = std::any_cast(&value)) { + fn(Geospatial::c_geo_point_type_col_name, std::any(geo->get_type_string())); + std::vector coords; + auto&& point = geo->get(); // throws + coords.push_back(point.longitude); + coords.push_back(point.latitude); + if (point.has_altitude()) { + coords.push_back(*point.get_altitude()); + } + fn(Geospatial::c_geo_point_coords_col_name, std::any(coords)); + return; + } +#endif for (auto&& v : util::any_cast(value)) fn(v.first, v.second); } diff --git a/src/realm/object-store/impl/realm_coordinator.cpp b/src/realm/object-store/impl/realm_coordinator.cpp index 3d9f2a95034..06a47e6041c 100644 --- a/src/realm/object-store/impl/realm_coordinator.cpp +++ b/src/realm/object-store/impl/realm_coordinator.cpp @@ -478,6 +478,7 @@ bool RealmCoordinator::open_db() options.durability = m_config.in_memory ? DBOptions::Durability::MemOnly : DBOptions::Durability::Full; options.is_immutable = m_config.immutable(); options.logger = util::Logger::get_default_logger(); + options.allow_flexible_schema = m_config.flexible_schema; if (!m_config.fifo_files_fallback_path.empty()) { options.temp_dir = util::normalize_dir(m_config.fifo_files_fallback_path); diff --git a/src/realm/object-store/index_set.hpp b/src/realm/object-store/index_set.hpp index d61e78221f4..e6f0692e767 100644 --- a/src/realm/object-store/index_set.hpp +++ b/src/realm/object-store/index_set.hpp @@ -391,8 +391,7 @@ template // Inlining this function crashes msvc when targeting arm64 in as of 19.39.33523 __declspec(noinline) #endif -inline void -ChunkedRangeVectorIterator::next_chunk() noexcept + inline void ChunkedRangeVectorIterator::next_chunk() noexcept { ++m_outer; m_inner = m_outer != m_end ? &m_outer->data[0] : nullptr; diff --git a/src/realm/object-store/object.hpp b/src/realm/object-store/object.hpp index 53fa43a2653..8d5a7527f7d 100644 --- a/src/realm/object-store/object.hpp +++ b/src/realm/object-store/object.hpp @@ -206,7 +206,12 @@ class Object { void set_property_value_impl(ContextType& ctx, const Property& property, ValueType value, CreatePolicy policy, bool is_default); template + void set_additional_property_value_impl(ContextType& ctx, StringData prop_name, ValueType value, + CreatePolicy policy); + template ValueType get_property_value_impl(ContextType& ctx, const Property& property) const; + template + ValueType get_additional_property_value_impl(ContextType& ctx, StringData prop_name) const; template static ObjKey get_for_primary_key_in_migration(ContextType& ctx, Table const& table, const Property& primary_prop, diff --git a/src/realm/object-store/object_accessor.hpp b/src/realm/object-store/object_accessor.hpp index 824ee437204..6022db7a0eb 100644 --- a/src/realm/object-store/object_accessor.hpp +++ b/src/realm/object-store/object_accessor.hpp @@ -42,9 +42,13 @@ namespace realm { template void Object::set_property_value(ContextType& ctx, StringData prop_name, ValueType value, CreatePolicy policy) { - auto& property = property_for_name(prop_name); - validate_property_for_setter(property); - set_property_value_impl(ctx, property, value, policy, false); + if (auto prop = m_object_schema->property_for_name(prop_name)) { + validate_property_for_setter(*prop); + set_property_value_impl(ctx, *prop, value, policy, false); + } + else { + set_additional_property_value_impl(ctx, prop_name, value, policy); + } } template @@ -62,7 +66,12 @@ ValueType Object::get_property_value(ContextType& ctx, const Property& property) template ValueType Object::get_property_value(ContextType& ctx, StringData prop_name) const { - return get_property_value_impl(ctx, property_for_name(prop_name)); + if (auto prop = m_object_schema->property_for_name(prop_name)) { + return get_property_value_impl(ctx, *prop); + } + else { + return get_additional_property_value_impl(ctx, prop_name); + } } namespace { @@ -205,6 +214,28 @@ void Object::set_property_value_impl(ContextType& ctx, const Property& property, ctx.did_change(); } +template +void Object::set_additional_property_value_impl(ContextType& ctx, StringData prop_name, ValueType value, + CreatePolicy policy) +{ + Mixed new_val = ctx.template unbox(value, policy); + if (new_val.is_type(type_Dictionary)) { + m_obj.set_additional_collection(prop_name, CollectionType::Dictionary); + object_store::Dictionary dict(m_realm, m_obj.get_collection_ptr(prop_name)); + dict.assign(ctx, value, policy); + ctx.did_change(); + return; + } + if (new_val.is_type(type_List)) { + m_obj.set_additional_collection(prop_name, CollectionType::List); + List list(m_realm, m_obj.get_collection_ptr(prop_name)); + list.assign(ctx, value, policy); + ctx.did_change(); + return; + } + m_obj.set_additional_prop(prop_name, new_val); +} + template ValueType Object::get_property_value_impl(ContextType& ctx, const Property& property) const { @@ -269,6 +300,20 @@ ValueType Object::get_property_value_impl(ContextType& ctx, const Property& prop } } +template +ValueType Object::get_additional_property_value_impl(ContextType& ctx, StringData prop_name) const +{ + verify_attached(); + auto value = m_obj.get_additional_prop(prop_name); + if (value.is_type(type_Dictionary)) { + return ctx.box(object_store::Dictionary(m_realm, m_obj.get_collection_ptr(prop_name))); + } + if (value.is_type(type_List)) { + return ctx.box(List(m_realm, m_obj.get_collection_ptr(prop_name))); + } + return ctx.box(value); +} + template Object Object::create(ContextType& ctx, std::shared_ptr const& realm, StringData object_type, ValueType value, CreatePolicy policy, ObjKey current_obj, Obj* out_row) @@ -319,7 +364,7 @@ Object Object::create(ContextType& ctx, std::shared_ptr const& realm, Obj // or throw an exception if updating is disabled. if (auto primary_prop = object_schema.primary_key_property()) { auto primary_value = - ctx.value_for_property(value, *primary_prop, primary_prop - &object_schema.persisted_properties[0]); + ctx.value_for_property(value, primary_prop->name, primary_prop - &object_schema.persisted_properties[0]); if (!primary_value) primary_value = ctx.default_value_for_property(object_schema, *primary_prop); if (!primary_value && !is_nullable(primary_prop->type)) @@ -372,30 +417,44 @@ Object Object::create(ContextType& ctx, std::shared_ptr const& realm, Obj // that. if (out_row && object_schema.table_type != ObjectSchema::ObjectType::TopLevelAsymmetric) *out_row = obj; - for (size_t i = 0; i < object_schema.persisted_properties.size(); ++i) { - auto& prop = object_schema.persisted_properties[i]; - // If table has primary key, it must have been set during object creation - if (prop.is_primary && skip_primary) - continue; - - auto v = ctx.value_for_property(value, prop, i); - if (!created && !v) - continue; - - bool is_default = false; - if (!v) { - v = ctx.default_value_for_property(object_schema, prop); - is_default = true; + + std::unordered_set props_supplied; + ctx.enumerate_dictionary(value, [&](StringData name, auto&& value) { + if (auto prop = object_schema.property_for_name(name)) { + if (!prop->is_primary || !skip_primary) + object.set_property_value_impl(ctx, *prop, value, policy, false); + props_supplied.insert(name); + } + else { + object.set_additional_property_value_impl(ctx, name, value, policy); } - // We consider null or a missing value to be equivalent to an empty - // array/set for historical reasons; the original implementation did this - // accidentally and it's not worth changing. - if ((!v || ctx.is_null(*v)) && !is_nullable(prop.type) && !is_collection(prop.type)) { - if (prop.is_primary || !ctx.allow_missing(value)) - throw MissingPropertyValueException(object_schema.name, prop.name); + }); + + if (created) { + // assign default values + for (size_t i = 0; i < object_schema.persisted_properties.size(); ++i) { + auto& prop = object_schema.persisted_properties[i]; + // If table has primary key, it must have been set during object creation + if (prop.is_primary && skip_primary) + continue; + + bool already_set = props_supplied.count(prop.name); + if (already_set) + continue; + + bool is_default = true; + auto v = ctx.default_value_for_property(object_schema, prop); + + // We consider null or a missing value to be equivalent to an empty + // array/set for historical reasons; the original implementation did this + // accidentally and it's not worth changing. + if ((!v || ctx.is_null(*v)) && !is_nullable(prop.type) && !is_collection(prop.type)) { + if (prop.is_primary || !ctx.allow_missing(value)) + throw MissingPropertyValueException(object_schema.name, prop.name); + } + if (v) + object.set_property_value_impl(ctx, prop, *v, policy, is_default); } - if (v) - object.set_property_value_impl(ctx, prop, *v, policy, is_default); } if (object_schema.table_type == ObjectSchema::ObjectType::TopLevelAsymmetric) { return Object{}; diff --git a/src/realm/object-store/shared_realm.hpp b/src/realm/object-store/shared_realm.hpp index 5e98c96149f..23b79b1ea9a 100644 --- a/src/realm/object-store/shared_realm.hpp +++ b/src/realm/object-store/shared_realm.hpp @@ -104,6 +104,7 @@ struct RealmConfig { std::string fifo_files_fallback_path; bool in_memory = false; + bool flexible_schema = false; SchemaMode schema_mode = SchemaMode::Automatic; SchemaSubsetMode schema_subset_mode = SchemaSubsetMode::Strict; diff --git a/src/realm/query_conditions.hpp b/src/realm/query_conditions.hpp index 87f1f702407..2522b09b00e 100644 --- a/src/realm/query_conditions.hpp +++ b/src/realm/query_conditions.hpp @@ -284,6 +284,10 @@ struct Equal { { return (v == 0 && ubound == 0 && lbound == 0); } + bool operator()(int64_t v1, int64_t v2) const + { + return v1 == v2; + } static std::string description() { @@ -326,6 +330,10 @@ struct NotEqual { { return (v > ubound || v < lbound); } + bool operator()(int64_t v1, int64_t v2) const + { + return v1 != v2; + } template bool operator()(A, B, C, D) const = delete; @@ -798,6 +806,10 @@ struct Greater { static_cast(ubound); return lbound > v; } + bool operator()(int64_t v1, int64_t v2) const + { + return v1 > v2; + } static std::string description() { @@ -872,7 +884,6 @@ struct NotNull { } }; - struct Less { static const int avx = 0x11; // _CMP_LT_OQ template @@ -889,6 +900,11 @@ struct Less { return Mixed::types_are_comparable(m1, m2) && (m1 < m2); } + bool operator()(int64_t v1, int64_t v2) const + { + return v1 < v2; + } + template bool operator()(A, B, C, D) const { @@ -896,14 +912,12 @@ struct Less { return false; } static const int condition = cond_Less; - bool can_match(int64_t v, int64_t lbound, int64_t ubound) + bool can_match(int64_t v, int64_t lbound, int64_t) { - static_cast(ubound); return lbound < v; } - bool will_match(int64_t v, int64_t lbound, int64_t ubound) + bool will_match(int64_t v, int64_t, int64_t ubound) { - static_cast(lbound); return ubound < v; } static std::string description() @@ -934,6 +948,10 @@ struct LessEqual { { return (m1.is_null() && m2.is_null()) || (Mixed::types_are_comparable(m1, m2) && (m1 <= m2)); } + bool operator()(int64_t v1, int64_t v2) const + { + return v1 <= v2; + } template bool operator()(A, B, C, D) const @@ -970,6 +988,10 @@ struct GreaterEqual { { return (m1.is_null() && m2.is_null()) || (Mixed::types_are_comparable(m1, m2) && (m1 >= m2)); } + bool operator()(int64_t v1, int64_t v2) const + { + return v1 >= v2; + } template bool operator()(A, B, C, D) const @@ -984,6 +1006,155 @@ struct GreaterEqual { static const int condition = -1; }; +/* Unsigned LT. + + This can be determined by trial subtaction. However, some care must be exercised + since simply subtracting one vector from another will allow carries from one + bitfield to flow into the next one. To avoid this, we isolate bitfields by clamping + the MSBs to 1 in A and 0 in B before subtraction. After the subtraction the MSBs in + the result indicate borrows from the MSB. We then compute overflow (borrow OUT of MSB) + using boolean logic as described below. + + Unsigned LT is also used to find all zero fields or all non-zero fields, so it is + the backbone of all comparisons returning vectors. + */ + +// compute the overflows in unsigned trial subtraction A-B. The overflows +// will be marked by 1 in the sign bit of each field in the result. Other +// bits in the result are zero. +// Overflow are detected for each field pair where A is less than B. +inline uint64_t unsigned_LT_vector(uint64_t MSBs, uint64_t A, uint64_t B) +{ + // 1. compute borrow from most significant bit + // Isolate bitfields inside A and B before subtraction (prevent carries from spilling over) + // do this by clamping most significant bit in A to 1, and msb in B to 0 + auto A_isolated = A | MSBs; // 1 op + auto B_isolated = B & ~MSBs; // 2 ops + auto borrows_into_sign_bit = ~(A_isolated - B_isolated); // 2 ops (total latency 4) + + // 2. determine what subtraction against most significant bit would give: + // A B borrow-in: (A-B-borrow-in) + // 0 0 0 (0-0-0) = 0 + // 0 0 1 (0-0-1) = 1 + borrow-out + // 0 1 0 (0-1-0) = 1 + borrow-out + // 0 1 1 (0-1-1) = 0 + borrow-out + // 1 0 0 (1-0-0) = 1 + // 1 0 1 (1-0-1) = 0 + // 1 1 0 (1-1-0) = 0 + // 1 1 1 (1-1-1) = 1 + borrow-out + // borrow-out = (~A & B) | (~A & borrow-in) | (A & B & borrow-in) + // The overflows are simply the borrow-out, now encoded into the sign bits of each field. + auto overflows = (~A & B) | (~A & borrows_into_sign_bit) | (A & B & borrows_into_sign_bit); + // ^ 6 ops, total latency 6 (4+2) + return overflows & MSBs; // 1 op, total latency 7 + // total of 12 ops and a latency of 7. On a beefy CPU 3-4 of those can run in parallel + // and still reach a combined latency of 10 or less. +} + +template +uint64_t find_all_fields_unsigned(uint64_t MSBs, uint64_t A, uint64_t B); + +template +uint64_t find_all_fields(uint64_t MSBs, uint64_t A, uint64_t B); + +template <> +inline uint64_t find_all_fields(uint64_t MSBs, uint64_t A, uint64_t B) +{ + // 0 != A^B, same as asking 0 - (A^B) overflows. + return unsigned_LT_vector(MSBs, 0, A ^ B); +} + +template <> +inline uint64_t find_all_fields(uint64_t MSBs, uint64_t A, uint64_t B) +{ + // get the fields which are EQ and negate the result + auto all_fields_NE = find_all_fields(MSBs, A, B); + auto all_fields_NE_negated = ~all_fields_NE; + // must filter the negated vector so only MSB are left. + return MSBs & all_fields_NE_negated; +} + +template <> +inline uint64_t find_all_fields_unsigned(uint64_t MSBs, uint64_t A, uint64_t B) +{ + return find_all_fields(MSBs, A, B); +} + +template <> +inline uint64_t find_all_fields_unsigned(uint64_t MSBs, uint64_t A, uint64_t B) +{ + return find_all_fields(MSBs, A, B); +} + +template <> +inline uint64_t find_all_fields_unsigned(uint64_t MSBs, uint64_t A, uint64_t B) +{ + return unsigned_LT_vector(MSBs, A, B); +} + +template <> +inline uint64_t find_all_fields_unsigned(uint64_t MSBs, uint64_t A, uint64_t B) +{ + // Now A <= B is the same as !(A > B) so... + // reverse A and B to turn (A>B) --> (B +inline uint64_t find_all_fields_unsigned(uint64_t MSBs, uint64_t A, uint64_t B) +{ + return find_all_fields_unsigned(MSBs, B, A); +} + +template <> +inline uint64_t find_all_fields_unsigned(uint64_t MSBs, uint64_t A, uint64_t B) +{ + return find_all_fields_unsigned(MSBs, B, A); +} + +/* + Handling signed values + + Trial subtraction only works as is for unsigned. We simply transform signed into unsigned + by pusing all values up by 1<<(field_width-1). This makes all negative values positive and positive + values remain positive, although larger. Any overflow during the push can be ignored. + After that transformation Trial subtraction should correctly detect the LT condition. + + */ + + +template <> +inline uint64_t find_all_fields(uint64_t MSBs, uint64_t A, uint64_t B) +{ + auto sign_bits = MSBs; + return unsigned_LT_vector(MSBs, A ^ sign_bits, B ^ sign_bits); +} + +template <> +inline uint64_t find_all_fields(uint64_t MSBs, uint64_t A, uint64_t B) +{ + auto sign_bits = MSBs; + return find_all_fields_unsigned(MSBs, A ^ sign_bits, B ^ sign_bits); +} + +template <> +inline uint64_t find_all_fields(uint64_t MSBs, uint64_t A, uint64_t B) +{ + // A > B is the same as B < A + return find_all_fields(MSBs, B, A); +} + +template <> +inline uint64_t find_all_fields(uint64_t MSBs, uint64_t A, uint64_t B) +{ + // A >= B is the same as B <= A + return find_all_fields(MSBs, B, A); +} + } // namespace realm #endif // REALM_QUERY_CONDITIONS_HPP diff --git a/src/realm/query_engine.hpp b/src/realm/query_engine.hpp index 39d39b43a5a..94cc9612a48 100644 --- a/src/realm/query_engine.hpp +++ b/src/realm/query_engine.hpp @@ -510,6 +510,7 @@ static size_t find_first_haystack(LeafType& leaf, NeedleContainer& needles, size { // for a small number of conditions, it is faster to do a linear search than to compute the hash // the exact thresholds were found experimentally + if (needles.size() < linear_search_threshold) { for (size_t i = start; i < end; ++i) { auto element = leaf.get(i); diff --git a/src/realm/query_state.hpp b/src/realm/query_state.hpp index b2812276539..ac0480d7166 100644 --- a/src/realm/query_state.hpp +++ b/src/realm/query_state.hpp @@ -22,8 +22,6 @@ #include // size_t #include // unint8_t etc -#include - namespace realm { enum Action { act_ReturnFirst, act_Sum, act_Max, act_Min, act_Count, act_FindAll, act_Average }; @@ -34,6 +32,7 @@ enum { cond_Equal, cond_NotEqual, cond_Greater, cond_Less, cond_VTABLE_FINDER_CO class ArrayUnsigned; class Mixed; +class ArrayPayload; class QueryStateBase { public: diff --git a/src/realm/sync/instruction_applier.cpp b/src/realm/sync/instruction_applier.cpp index 60dfa4ae0c9..701503abdf9 100644 --- a/src/realm/sync/instruction_applier.cpp +++ b/src/realm/sync/instruction_applier.cpp @@ -1495,8 +1495,13 @@ InstructionApplier::PathResolver::Status InstructionApplier::PathResolver::resol InstructionApplier::PathResolver::Status InstructionApplier::PathResolver::resolve_field(Obj& obj, InternString field) { auto field_name = get_string(field); - ColKey col = obj.get_table()->get_column_key(field_name); + auto table = obj.get_table().unchecked_ptr(); + ColKey col = table->get_column_key(field_name); if (!col) { + if (auto ck = table->get_additional_prop_col()) { + auto dict = obj.get_dictionary(ck); + return resolve_dictionary_element(dict, field); + } on_error(util::format("%1: No such field: '%2' in class '%3'", m_instr_name, field_name, obj.get_table()->get_name())); return Status::DidNotResolve; diff --git a/src/realm/sync/instruction_replication.cpp b/src/realm/sync/instruction_replication.cpp index c845bd9c0bf..190a4c92dd3 100644 --- a/src/realm/sync/instruction_replication.cpp +++ b/src/realm/sync/instruction_replication.cpp @@ -611,51 +611,62 @@ void SyncReplication::set_clear(const CollectionBase& set) } } -void SyncReplication::dictionary_update(const CollectionBase& dict, const Mixed& key, const Mixed& value) +void SyncReplication::dictionary_update(const CollectionBase& dict, const Mixed& key, const Mixed* value) { // If link is unresolved, it should not be communicated. - if (value.is_unresolved_link()) { + if (value && value->is_unresolved_link()) { return; } - if (select_collection(dict)) { - Instruction::Update instr; - REALM_ASSERT(key.get_type() == type_String); - populate_path_instr(instr, dict); + Instruction::Update instr; + REALM_ASSERT(key.get_type() == type_String); + + const Table* source_table = dict.get_table().unchecked_ptr(); + auto col = dict.get_col_key(); + ObjKey obj_key = dict.get_owner_key(); + if (source_table->is_additional_props_col(col)) { + // Here we have to fake it and pretend we are setting/erasing a property on the object + if (!select_table(*source_table)) { + return; + } + + populate_path_instr(instr, *source_table, obj_key, {key.get_string()}); + } + else { + if (!select_collection(dict)) { + return; + } + + populate_path_instr(instr, *source_table, obj_key, dict.get_short_path()); StringData key_value = key.get_string(); instr.path.push_back(m_encoder.intern_string(key_value)); - instr.value = as_payload(dict, value); - instr.is_default = false; - emit(instr); } + if (value) { + instr.value = as_payload(*source_table, col, *value); + } + else { + instr.value = Instruction::Payload::Erased{}; + } + instr.is_default = false; + emit(instr); } void SyncReplication::dictionary_insert(const CollectionBase& dict, size_t ndx, Mixed key, Mixed value) { Replication::dictionary_insert(dict, ndx, key, value); - dictionary_update(dict, key, value); + dictionary_update(dict, key, &value); } void SyncReplication::dictionary_set(const CollectionBase& dict, size_t ndx, Mixed key, Mixed value) { Replication::dictionary_set(dict, ndx, key, value); - dictionary_update(dict, key, value); + dictionary_update(dict, key, &value); } void SyncReplication::dictionary_erase(const CollectionBase& dict, size_t ndx, Mixed key) { Replication::dictionary_erase(dict, ndx, key); - - if (select_collection(dict)) { - Instruction::Update instr; - REALM_ASSERT(key.get_type() == type_String); - populate_path_instr(instr, dict); - StringData key_value = key.get_string(); - instr.path.push_back(m_encoder.intern_string(key_value)); - instr.value = Instruction::Payload::Erased{}; - instr.is_default = false; - emit(instr); - } + dictionary_update(dict, key, nullptr); } void SyncReplication::dictionary_clear(const CollectionBase& dict) @@ -750,7 +761,25 @@ void SyncReplication::populate_path_instr(Instruction::PathInstruction& instr, c { REALM_ASSERT(key); // The first path entry will be the column key - REALM_ASSERT(path[0].is_col_key()); + std::string field_name; + if (path[0].is_col_key()) { + auto ck = path[0].get_col_key(); + if (table.is_additional_props_col(ck)) { + // We are modifying a collection nested in an additional property + REALM_ASSERT(path.size() > 1); + field_name = path[1].get_key(); + // Erase the "__additional" part of the path + path.erase(path.begin()); + } + else { + field_name = table.get_column_name(ck); + } + } + else { + // In the case of an additional property directly on an object, + // the first element is a string. + field_name = path[0].get_key(); + } if (table.is_embedded()) { // For embedded objects, Obj::traverse_path() yields the top object @@ -760,7 +789,7 @@ void SyncReplication::populate_path_instr(Instruction::PathInstruction& instr, c // Populate top object in the normal way. auto top_table = table.get_parent_group()->get_table(full_path.top_table); - full_path.path_from_top.emplace_back(table.get_column_name(path[0].get_col_key())); + full_path.path_from_top.emplace_back(field_name); for (auto it = path.begin() + 1; it != path.end(); ++it) { full_path.path_from_top.emplace_back(std::move(*it)); @@ -782,8 +811,6 @@ void SyncReplication::populate_path_instr(Instruction::PathInstruction& instr, c m_last_primary_key = instr.object; } - StringData field_name = table.get_column_name(path[0].get_col_key()); - if (m_last_field_name == field_name) { instr.field = m_last_interned_field_name; } diff --git a/src/realm/sync/instruction_replication.hpp b/src/realm/sync/instruction_replication.hpp index ff7fbd62de4..46f683dcec6 100644 --- a/src/realm/sync/instruction_replication.hpp +++ b/src/realm/sync/instruction_replication.hpp @@ -132,13 +132,13 @@ class SyncReplication : public Replication { void populate_path_instr(Instruction::PathInstruction&, const CollectionBase&); void populate_path_instr(Instruction::PathInstruction&, const CollectionBase&, uint32_t ndx); - void dictionary_update(const CollectionBase&, const Mixed& key, const Mixed& val); + void dictionary_update(const CollectionBase&, const Mixed& key, const Mixed* val); // Cache information for the purpose of avoiding excessive string comparisons / interning // lookups. const Table* m_last_table = nullptr; ObjKey m_last_object; - StringData m_last_field_name; + std::string m_last_field_name; InternString m_last_class_name; util::Optional m_last_primary_key; InternString m_last_interned_field_name; diff --git a/src/realm/sync/noinst/server/server_file_access_cache.hpp b/src/realm/sync/noinst/server/server_file_access_cache.hpp index 24af8a4adc4..51186f5b2ba 100644 --- a/src/realm/sync/noinst/server/server_file_access_cache.hpp +++ b/src/realm/sync/noinst/server/server_file_access_cache.hpp @@ -229,6 +229,7 @@ inline DBOptions ServerFileAccessCache::Slot::make_shared_group_options() const options.encryption_key = m_cache.m_encryption_key->data(); if (m_disable_sync_to_disk) options.durability = DBOptions::Durability::Unsafe; + options.allow_flexible_schema = true; return options; } diff --git a/src/realm/table.cpp b/src/realm/table.cpp index f41ec412ad5..fb910594ec5 100644 --- a/src/realm/table.cpp +++ b/src/realm/table.cpp @@ -262,6 +262,7 @@ using namespace realm; using namespace realm::util; Replication* Table::g_dummy_replication = nullptr; +static const StringData additional_properties_colum_name{"__additional"}; bool TableVersions::operator==(const TableVersions& other) const { @@ -634,12 +635,18 @@ void Table::init(ref_type top_ref, ArrayParent* parent, size_t ndx_in_parent, bo auto rot_pk_key = m_top.get_as_ref_or_tagged(top_position_for_pk_col); m_primary_key_col = rot_pk_key.is_tagged() ? ColKey(rot_pk_key.get_as_int()) : ColKey(); + m_additional_prop_col = ColKey(); if (m_top.size() <= top_position_for_flags) { m_table_type = Type::TopLevel; } else { uint64_t flags = m_top.get_as_ref_or_tagged(top_position_for_flags).get_as_int(); m_table_type = Type(flags & table_type_mask); + if (flags & additional_prop_mask) { + // If we have an additional properties column, it will always be first + REALM_ASSERT(m_spec.m_names.size() > 0 && m_spec.m_names.get(0) == additional_properties_colum_name); + m_additional_prop_col = ColKey(m_spec.m_keys.get(0)); + } } m_has_any_embedded_objects.reset(); @@ -2952,6 +2959,23 @@ void Table::do_set_primary_key_column(ColKey col_key) m_primary_key_col = col_key; } +void Table::do_add_additional_prop_column() +{ + ColumnAttrMask attr; + attr.set(col_attr_Dictionary); + attr.set(col_attr_Nullable); + ColKey col_key = generate_col_key(col_type_Mixed, attr); + + uint64_t flags = m_top.get_as_ref_or_tagged(top_position_for_flags).get_as_int(); + flags |= additional_prop_mask; + m_top.set(top_position_for_flags, RefOrTagged::make_tagged(flags)); + + m_additional_prop_col = + do_insert_root_column(col_key, col_type_Mixed, additional_properties_colum_name, type_String); + // Be sure that it will always be first + REALM_ASSERT(m_additional_prop_col.get_index().val == 0); +} + bool Table::contains_unique_values(ColKey col) const { if (search_index_type(col) == IndexType::General) { @@ -3350,3 +3374,34 @@ ColKey Table::find_opposite_column(ColKey col_key) const } return ColKey(); } + +ref_type Table::typed_write(ref_type ref, _impl::ArrayWriterBase& out) const +{ + REALM_ASSERT(ref == m_top.get_mem().get_ref()); + if (out.only_modified && m_alloc.is_read_only(ref)) + return ref; + out.table = this; + // ignore ref from here, just use Tables own accessors + TempArray dest(m_top.size()); + for (unsigned j = 0; j < m_top.size(); ++j) { + RefOrTagged rot = m_top.get_as_ref_or_tagged(j); + if (rot.is_tagged() || (rot.is_ref() && rot.get_as_ref() == 0)) { + dest.set(j, rot); + } + else { + ref_type new_ref; + if (j == 2) { + // only do type driven write for clustertree + new_ref = m_clusters.typed_write(rot.get_as_ref(), out); + } + else { + // rest is handled using untyped approach + Array a(m_alloc); + a.init_from_ref(rot.get_as_ref()); + new_ref = a.write(out, true, out.only_modified, false); + } + dest.set_as_ref(j, new_ref); + } + } + return dest.write(out); +} diff --git a/src/realm/table.hpp b/src/realm/table.hpp index 80496b12fb9..278f756041c 100644 --- a/src/realm/table.hpp +++ b/src/realm/table.hpp @@ -93,6 +93,7 @@ class Table { /// . enum class Type : uint8_t { TopLevel = 0, Embedded = 0x1, TopLevelAsymmetric = 0x2 }; constexpr static uint8_t table_type_mask = 0x3; + constexpr static uint8_t additional_prop_mask = 0x4; /// Construct a new freestanding top-level table with static /// lifetime. For debugging only. @@ -146,6 +147,15 @@ class Table { DataType get_dictionary_key_type(ColKey column_key) const noexcept; ColKey get_column_key(StringData name) const noexcept; ColKey get_column_key(StableIndex) const noexcept; + bool is_additional_props_col(ColKey ck) const + { + return ck == m_additional_prop_col; + } + ColKey get_additional_prop_col() const + { + return m_additional_prop_col; + } + ColKeys get_column_keys() const; typedef util::Optional> BacklinkOrigin; BacklinkOrigin find_backlink_origin(StringData origin_table_name, StringData origin_col_name) const noexcept; @@ -544,6 +554,9 @@ class Table { return false; } + ref_type typed_write(ref_type ref, _impl::ArrayWriterBase& out, bool deep, bool only_modified, + bool compress) const; + private: template TableView find_all(ColKey col_key, T value); @@ -688,6 +701,8 @@ class Table { Replication* const* m_repl; }; + ref_type typed_write(ref_type ref, _impl::ArrayWriterBase& out) const; + private: enum LifeCycleCookie { cookie_created = 0x1234, @@ -733,6 +748,7 @@ class Table { Array m_opposite_column; // 8th slot in m_top std::vector> m_index_accessors; ColKey m_primary_key_col; + ColKey m_additional_prop_col; Replication* const* m_repl; static Replication* g_dummy_replication; bool m_is_frozen = false; @@ -788,6 +804,7 @@ class Table { ColKey find_backlink_column(ColKey origin_col_key, TableKey origin_table) const; ColKey find_or_add_backlink_column(ColKey origin_col_key, TableKey origin_table); void do_set_primary_key_column(ColKey col_key); + void do_add_additional_prop_column(); void validate_column_is_unique(ColKey col_key) const; ObjKey get_next_valid_key(); @@ -949,6 +966,7 @@ class ColKeys { public: ColKeys(ConstTableRef&& t) : m_table(std::move(t)) + , m_offset(m_table->get_additional_prop_col() ? 1 : 0) { } @@ -959,7 +977,7 @@ class ColKeys { size_t size() const { - return m_table->get_column_count(); + return m_table->get_column_count() - m_offset; } bool empty() const { @@ -967,19 +985,20 @@ class ColKeys { } ColKey operator[](size_t p) const { - return ColKeyIterator(m_table, p).operator*(); + return ColKeyIterator(m_table, p + m_offset).operator*(); } ColKeyIterator begin() const { - return ColKeyIterator(m_table, 0); + return ColKeyIterator(m_table, m_offset); } ColKeyIterator end() const { - return ColKeyIterator(m_table, size()); + return ColKeyIterator(m_table, m_table->get_column_count()); } private: ConstTableRef m_table; + unsigned m_offset = 0; }; // Class used to collect a chain of links when building up a Query following links. diff --git a/src/realm/transaction.cpp b/src/realm/transaction.cpp index 9e93875923f..7deb3451184 100644 --- a/src/realm/transaction.cpp +++ b/src/realm/transaction.cpp @@ -170,6 +170,7 @@ Transaction::Transaction(DBRef _db, SlabAlloc* alloc, DB::ReadLockInfo& rli, DB: { bool writable = stage == DB::transact_Writing; m_transact_stage = DB::transact_Ready; + m_allow_additional_properties = db->m_allow_flexible_schema; set_transact_stage(stage); attach_shared(m_read_lock.m_top_ref, m_read_lock.m_file_size, writable, VersionID{rli.m_version, rli.m_reader_idx}); @@ -533,7 +534,7 @@ void Transaction::upgrade_file_format(int target_file_format_version) // Be sure to revisit the following upgrade logic when a new file format // version is introduced. The following assert attempt to help you not // forget it. - REALM_ASSERT_EX(target_file_format_version == 24, target_file_format_version); + REALM_ASSERT_EX(target_file_format_version == 25, target_file_format_version); // DB::do_open() must ensure that only supported version are allowed. // It does that by asking backup if the current file format version is @@ -584,6 +585,11 @@ void Transaction::upgrade_file_format(int target_file_format_version) t->migrate_col_keys(); } } + if (current_file_format_version < 25) { + for (auto k : table_keys) { + get_table(k); + } + } // NOTE: Additional future upgrade steps go here. } diff --git a/src/realm/transaction.hpp b/src/realm/transaction.hpp index abd95d7d3ff..4da316c0d2e 100644 --- a/src/realm/transaction.hpp +++ b/src/realm/transaction.hpp @@ -411,7 +411,7 @@ inline bool Transaction::promote_to_write(O* observer, bool nonblocking) db->m_logger->log(util::LogCategory::transaction, util::Logger::Level::trace, "Tr %1: Promote to write: %2 -> %3", m_log_id, old_version, m_read_lock.m_version); } - + update_allocator_wrappers(true); set_transact_stage(DB::transact_Writing); return true; } diff --git a/src/realm/util/config.h.in b/src/realm/util/config.h.in index 508dc064a91..3bb6cd9df36 100644 --- a/src/realm/util/config.h.in +++ b/src/realm/util/config.h.in @@ -21,3 +21,4 @@ #cmakedefine01 REALM_VALGRIND #cmakedefine01 REALM_ASAN #cmakedefine01 REALM_TSAN +#cmakedefine01 REALM_COMPRESS diff --git a/test/benchmark-common-tasks/main.cpp b/test/benchmark-common-tasks/main.cpp index 1218d509e89..b8e52fc5d70 100644 --- a/test/benchmark-common-tasks/main.cpp +++ b/test/benchmark-common-tasks/main.cpp @@ -1413,7 +1413,6 @@ struct BenchmarkQueryChainedOrIntsIndexed : BenchmarkQueryChainedOrInts { } }; - struct BenchmarkQueryIntEquality : BenchmarkQueryChainedOrInts { const char* name() const { diff --git a/test/object-store/c_api/c_api.cpp b/test/object-store/c_api/c_api.cpp index 3b80efa75c6..4e124f21c0e 100644 --- a/test/object-store/c_api/c_api.cpp +++ b/test/object-store/c_api/c_api.cpp @@ -2630,6 +2630,12 @@ TEST_CASE("C API - properties", "[c_api]") { CHECK(strings.get() != list2.get()); } + SECTION("get_by_name") { + auto list2 = cptr_checked(realm_get_list_by_name(obj2.get(), "strings")); + CHECK(realm_equals(strings.get(), list2.get())); + CHECK(strings.get() != list2.get()); + } + SECTION("insert, then get") { write([&]() { CHECK(checked(realm_list_insert(strings.get(), 0, a))); @@ -3739,6 +3745,12 @@ TEST_CASE("C API - properties", "[c_api]") { CHECK(strings.get() != dict2.get()); } + SECTION("get by name") { + auto dict2 = cptr_checked(realm_get_dictionary_by_name(obj1.get(), "nullable_string_dict")); + CHECK(realm_equals(strings.get(), dict2.get())); + CHECK(strings.get() != dict2.get()); + } + SECTION("insert, then get, then erase") { write([&]() { bool inserted = false; @@ -5841,6 +5853,94 @@ TEST_CASE("C API: convert", "[c_api]") { realm_release(realm); } +TEST_CASE("C API: flexible schema", "[c_api]") { + TestFile test_file; + ObjectSchema object_schema = {"Foo", {{"_id", PropertyType::Int, Property::IsPrimary{true}}}}; + + auto config = make_config(test_file.path.c_str(), false); + config->schema = Schema{object_schema}; + config->schema_version = 0; + config->flexible_schema = 1; + auto realm = realm_open(config.get()); + realm_class_info_t class_foo; + bool found = false; + CHECK(checked(realm_find_class(realm, "Foo", &found, &class_foo))); + REQUIRE(found); + + SECTION("Simple set/get/delete") { + checked(realm_begin_write(realm)); + + realm_value_t pk = rlm_int_val(42); + auto obj1 = cptr_checked(realm_object_create_with_primary_key(realm, class_foo.key, pk)); + checked(realm_set_value_by_name(obj1.get(), "age", rlm_int_val(23))); + const char* prop_names[10]; + size_t actual; + realm_get_additional_properties(obj1.get(), prop_names, 10, &actual); + REQUIRE(actual == 1); + CHECK(prop_names[0] == std::string_view("age")); + realm_has_property(obj1.get(), "age", &found); + REQUIRE(found); + realm_has_property(obj1.get(), "_id", &found); + REQUIRE(found); + realm_has_property(obj1.get(), "weight", &found); + REQUIRE(!found); + + realm_value_t value; + CHECK(checked(realm_get_value_by_name(obj1.get(), "age", &value))); + CHECK(value.type == RLM_TYPE_INT); + CHECK(value.integer == 23); + + checked(realm_erase_additional_property(obj1.get(), "age")); + realm_get_additional_properties(obj1.get(), nullptr, 0, &actual); + REQUIRE(actual == 0); + realm_commit(realm); + } + + SECTION("Set/get nested list") { + checked(realm_begin_write(realm)); + + realm_value_t pk = rlm_int_val(42); + auto obj1 = cptr_checked(realm_object_create_with_primary_key(realm, class_foo.key, pk)); + auto list = cptr_checked(realm_set_list_by_name(obj1.get(), "scores")); + REQUIRE(list); + realm_has_property(obj1.get(), "scores", &found); + REQUIRE(found); + + realm_value_t value; + CHECK(checked(realm_get_value_by_name(obj1.get(), "scores", &value))); + CHECK(value.type == RLM_TYPE_LIST); + + auto list1 = cptr_checked(realm_get_list_by_name(obj1.get(), "scores")); + REQUIRE(list1); + + realm_commit(realm); + } + + SECTION("Set/get nested dictionary") { + checked(realm_begin_write(realm)); + + realm_value_t pk = rlm_int_val(42); + auto obj1 = cptr_checked(realm_object_create_with_primary_key(realm, class_foo.key, pk)); + auto dict = cptr_checked(realm_set_dictionary_by_name(obj1.get(), "properties")); + REQUIRE(dict); + realm_has_property(obj1.get(), "properties", &found); + REQUIRE(found); + + realm_value_t value; + CHECK(checked(realm_get_value_by_name(obj1.get(), "properties", &value))); + CHECK(value.type == RLM_TYPE_DICTIONARY); + + auto dict1 = cptr_checked(realm_get_dictionary_by_name(obj1.get(), "properties")); + REQUIRE(dict1); + + realm_commit(realm); + } + + realm_close(realm); + REQUIRE(realm_is_closed(realm)); + realm_release(realm); +} + struct Userdata { std::atomic called{false}; bool has_error; diff --git a/test/object-store/object.cpp b/test/object-store/object.cpp index 37ebc08ec72..6b105f0ecd0 100644 --- a/test/object-store/object.cpp +++ b/test/object-store/object.cpp @@ -155,6 +155,100 @@ class CreatePolicyRecordingContext { mutable CreatePolicy last_create_policy; }; +TEST_CASE("object with flexible schema") { + using namespace std::string_literals; + _impl::RealmCoordinator::assert_no_open_realms(); + + InMemoryTestFile config; + config.automatic_change_notifications = false; + config.schema_mode = SchemaMode::AdditiveExplicit; + config.flexible_schema = true; + config.schema = Schema{{ + "table", + { + {"_id", PropertyType::Int, Property::IsPrimary{true}}, + }, + }}; + + config.schema_version = 0; + auto r = Realm::get_shared_realm(config); + + TestContext d(r); + auto create = [&](std::any&& value, CreatePolicy policy = CreatePolicy::ForceCreate) { + r->begin_transaction(); + auto obj = Object::create(d, r, *r->schema().find("table"), value, policy); + r->commit_transaction(); + return obj; + }; + + SECTION("create object") { + auto object = create(AnyDict{ + {"_id", INT64_C(1)}, + {"bool", true}, + {"int", INT64_C(5)}, + {"float", 2.2f}, + {"double", 3.3}, + {"string", "hello"s}, + {"date", Timestamp(10, 20)}, + {"object id", ObjectId("000000000000000000000001")}, + {"decimal", Decimal128("1.23e45")}, + {"uuid", UUID("3b241101-abba-baba-caca-4136c566a962")}, + {"mixed", "mixed"s}, + + {"bool array", AnyVec{true, false}}, + {"int array", AnyVec{INT64_C(5), INT64_C(6)}}, + {"float array", AnyVec{1.1f, 2.2f}}, + {"double array", AnyVec{3.3, 4.4}}, + {"string array", AnyVec{"a"s, "b"s, "c"s}}, + {"date array", AnyVec{Timestamp(10, 20), Timestamp(30, 40)}}, + {"object id array", AnyVec{ObjectId("AAAAAAAAAAAAAAAAAAAAAAAA"), ObjectId("BBBBBBBBBBBBBBBBBBBBBBBB")}}, + {"decimal array", AnyVec{Decimal128("1.23e45"), Decimal128("6.78e9")}}, + {"uuid array", AnyVec{UUID(), UUID("3b241101-e2bb-4255-8caf-4136c566a962")}}, + {"mixed array", + AnyVec{25, "b"s, 1.45, util::none, Timestamp(30, 40), Decimal128("1.23e45"), + ObjectId("AAAAAAAAAAAAAAAAAAAAAAAA"), UUID("3b241101-e2bb-4255-8caf-4136c566a962")}}, + {"dictionary", AnyDict{{"key", "value"s}}}, + }); + + Obj obj = object.get_obj(); + REQUIRE(obj.get("_id") == 1); + REQUIRE(obj.get("bool") == true); + REQUIRE(obj.get("int") == 5); + REQUIRE(obj.get("float") == 2.2f); + REQUIRE(obj.get("double") == 3.3); + REQUIRE(obj.get("string") == "hello"); + REQUIRE(obj.get("date") == Timestamp(10, 20)); + REQUIRE(obj.get("object id") == ObjectId("000000000000000000000001")); + REQUIRE(obj.get("decimal") == Decimal128("1.23e45")); + REQUIRE(obj.get("uuid") == UUID("3b241101-abba-baba-caca-4136c566a962")); + REQUIRE(obj.get("mixed") == Mixed("mixed")); + + auto check_array = [&](StringData prop, auto... values) { + auto vec = get_vector({values...}); + using U = typename decltype(vec)::value_type; + auto list = obj.get_list_ptr(prop); + size_t i = 0; + for (auto value : vec) { + CAPTURE(i); + REQUIRE(i < list->size()); + REQUIRE(value == list->get(i).get()); + ++i; + } + }; + check_array("bool array", true, false); + check_array("int array", INT64_C(5), INT64_C(6)); + check_array("float array", 1.1f, 2.2f); + check_array("double array", 3.3, 4.4); + check_array("string array", StringData("a"), StringData("b"), StringData("c")); + check_array("date array", Timestamp(10, 20), Timestamp(30, 40)); + check_array("object id array", ObjectId("AAAAAAAAAAAAAAAAAAAAAAAA"), ObjectId("BBBBBBBBBBBBBBBBBBBBBBBB")); + check_array("decimal array", Decimal128("1.23e45"), Decimal128("6.78e9")); + check_array("uuid array", UUID(), UUID("3b241101-e2bb-4255-8caf-4136c566a962")); + + REQUIRE(obj.get_dictionary_ptr("dictionary")->get("key") == Mixed("value")); + } +} + TEST_CASE("object") { using namespace std::string_literals; _impl::RealmCoordinator::assert_no_open_realms(); @@ -281,6 +375,7 @@ TEST_CASE("object") { {"person", { {"_id", PropertyType::String, Property::IsPrimary{true}}, + {"name", PropertyType::String}, {"age", PropertyType::Int}, {"scores", PropertyType::Array | PropertyType::Int}, {"assistant", PropertyType::Object | PropertyType::Nullable, "person"}, @@ -1180,7 +1275,6 @@ TEST_CASE("object") { {"data", "olleh"s}, {"date", Timestamp(10, 20)}, {"object", AnyDict{{"_id", INT64_C(10)}, {"value", INT64_C(10)}}}, - {"array", AnyVector{AnyDict{{"value", INT64_C(20)}}}}, {"object id", ObjectId("000000000000000000000001")}, {"decimal", Decimal128("1.23e45")}, {"uuid", UUID("3b241101-0000-0000-0000-4136c566a962")}, @@ -1532,7 +1626,6 @@ TEST_CASE("object") { {"string array", AnyVec{"a"s, "b"s, "c"s}}, {"data array", AnyVec{"d"s, "e"s, "f"s}}, {"date array", AnyVec{}}, - {"object array", AnyVec{AnyDict{{"_id", INT64_C(20)}, {"value", INT64_C(20)}}}}, {"object id array", AnyVec{ObjectId("000000000000000000000001")}}, {"decimal array", AnyVec{Decimal128("1.23e45")}}, {"uuid array", AnyVec{UUID("3b241101-1111-bbbb-cccc-4136c566a962")}}, @@ -1672,7 +1765,6 @@ TEST_CASE("object") { {"data", "olleh"s}, {"date", Timestamp(10, 20)}, {"object", AnyDict{{"_id", INT64_C(10)}, {"value", INT64_C(10)}}}, - {"array", AnyVector{AnyDict{{"value", INT64_C(20)}}}}, {"object id", ObjectId("000000000000000000000001")}, {"decimal", Decimal128("1.23e45")}, {"uuid", UUID("3b241101-aaaa-bbbb-cccc-4136c566a962")}, @@ -1688,7 +1780,6 @@ TEST_CASE("object") { {"data", "olleh"s}, {"date", Timestamp(10, 20)}, {"object", AnyDict{{"_id", INT64_C(10)}, {"value", INT64_C(10)}}}, - {"array", AnyVector{AnyDict{{"value", INT64_C(20)}}}}, {"object id", ObjectId("000000000000000000000001")}, {"decimal", Decimal128("1.23e45")}, {"uuid", UUID("3b241101-aaaa-bbbb-cccc-4136c566a962")}, @@ -1720,9 +1811,9 @@ TEST_CASE("object") { obj = create(AnyDict{{"_id", d.null_value()}}, "nullable string pk"); REQUIRE(obj.get_obj().is_null(col_pk_str)); - obj = create(AnyDict{{}}, "nullable int pk"); + obj = create(AnyDict{}, "nullable int pk"); REQUIRE(obj.get_obj().get>(col_pk_int) == 10); - obj = create(AnyDict{{}}, "nullable string pk"); + obj = create(AnyDict{}, "nullable string pk"); REQUIRE(obj.get_obj().get(col_pk_str) == "value"); } diff --git a/test/object-store/results.cpp b/test/object-store/results.cpp index 1814d06cd87..03d684ef1e7 100644 --- a/test/object-store/results.cpp +++ b/test/object-store/results.cpp @@ -103,7 +103,6 @@ struct TestContext : CppContext { } }; - TEST_CASE("notifications: async delivery", "[notifications]") { _impl::RealmCoordinator::assert_no_open_realms(); TestFile config; diff --git a/test/test_alloc.cpp b/test/test_alloc.cpp index 334389d5fe3..f030e6f073e 100644 --- a/test/test_alloc.cpp +++ b/test/test_alloc.cpp @@ -70,17 +70,21 @@ using namespace realm::util; namespace { + void set_capacity(char* header, size_t value) { + NodeHeader::set_wtype_in_header(NodeHeader::wtype_Ignore, header); typedef unsigned char uchar; uchar* h = reinterpret_cast(header); h[0] = uchar((value >> 19) & 0x000000FF); h[1] = uchar((value >> 11) & 0x000000FF); h[2] = uchar((value >> 3) & 0x000000FF); + REALM_ASSERT(NodeHeader::get_capacity_from_header(header) == value); } size_t get_capacity(const char* header) { + REALM_ASSERT(NodeHeader::get_wtype_from_header(header) == NodeHeader::wtype_Ignore); typedef unsigned char uchar; const uchar* h = reinterpret_cast(header); return (size_t(h[0]) << 19) + (size_t(h[1]) << 11) + (h[2] << 3); @@ -281,8 +285,11 @@ TEST(Alloc_Fuzzy) refs.push_back(r); set_capacity(r.get_addr(), siz); - // write some data to the allcoated area so that we can verify it later - memset(r.get_addr() + 3, static_cast(reinterpret_cast(r.get_addr())), siz - 3); + // write some data to the allcoated area so that we can verify it later. + // We must keep byte 4 in the header unharmed, since it is needed later by the allocator + // to determine the encoding, and hence the size of any released object. + // We simply skip the header. + memset(r.get_addr() + 8, static_cast(reinterpret_cast(r.get_addr())), siz - 8); } else if (refs.size() > 0) { // free random entry @@ -298,7 +305,7 @@ TEST(Alloc_Fuzzy) size_t siz = get_capacity(r.get_addr()); // verify that all the data we wrote during allocation is intact - for (size_t c = 3; c < siz; c++) { + for (size_t c = 8; c < siz; c++) { if (r.get_addr()[c] != static_cast(reinterpret_cast(r.get_addr()))) { // faster than using 'CHECK' for each character, which is slow CHECK(false); @@ -376,7 +383,7 @@ NONCONCURRENT_TEST_IF(Alloc_MapFailureRecovery, _impl::SimulatedFailure::is_enab { // Extending the first mapping const auto initial_baseline = alloc.get_baseline(); const auto initial_version = alloc.get_mapping_version(); - const char* initial_translated = alloc.translate(1000); + const char* initial_translated = alloc.translate_in_slab(1000); _impl::SimulatedFailure::prime_mmap([](size_t) { return true; @@ -387,7 +394,7 @@ NONCONCURRENT_TEST_IF(Alloc_MapFailureRecovery, _impl::SimulatedFailure::is_enab CHECK_THROW(alloc.update_reader_view(page_size * 2), std::bad_alloc); CHECK_EQUAL(initial_baseline, alloc.get_baseline()); CHECK_EQUAL(initial_version, alloc.get_mapping_version()); - CHECK_EQUAL(initial_translated, alloc.translate(1000)); + CHECK_EQUAL(initial_translated, alloc.translate_in_slab(1000)); _impl::SimulatedFailure::prime_mmap(nullptr); alloc.get_file().resize(page_size * 2); @@ -411,7 +418,7 @@ NONCONCURRENT_TEST_IF(Alloc_MapFailureRecovery, _impl::SimulatedFailure::is_enab { // Add a new complete section after a complete section const auto initial_baseline = alloc.get_baseline(); const auto initial_version = alloc.get_mapping_version(); - const char* initial_translated = alloc.translate(1000); + const char* initial_translated = alloc.translate_in_slab(1000); _impl::SimulatedFailure::prime_mmap([](size_t) { return true; @@ -420,14 +427,14 @@ NONCONCURRENT_TEST_IF(Alloc_MapFailureRecovery, _impl::SimulatedFailure::is_enab CHECK_THROW(alloc.update_reader_view(section_size * 2), std::bad_alloc); CHECK_EQUAL(initial_baseline, alloc.get_baseline()); CHECK_EQUAL(initial_version, alloc.get_mapping_version()); - CHECK_EQUAL(initial_translated, alloc.translate(1000)); + CHECK_EQUAL(initial_translated, alloc.translate_in_slab(1000)); _impl::SimulatedFailure::prime_mmap(nullptr); alloc.update_reader_view(section_size * 2); CHECK_EQUAL(alloc.get_baseline(), section_size * 2); - CHECK_EQUAL(initial_version, alloc.get_mapping_version()); // did not alter an existing mapping - CHECK_EQUAL(initial_translated, alloc.translate(1000)); // first section was not remapped - CHECK_EQUAL(0, *alloc.translate(section_size * 2 - page_size)); + CHECK_EQUAL(initial_version, alloc.get_mapping_version()); // did not alter an existing mapping + CHECK_EQUAL(initial_translated, alloc.translate_in_slab(1000)); // first section was not remapped + CHECK_EQUAL(0, *alloc.translate_in_slab(section_size * 2 - page_size)); alloc.purge_old_mappings(4, 4); } @@ -437,8 +444,8 @@ NONCONCURRENT_TEST_IF(Alloc_MapFailureRecovery, _impl::SimulatedFailure::is_enab { // Add complete section and a a partial section after that const auto initial_baseline = alloc.get_baseline(); const auto initial_version = alloc.get_mapping_version(); - const char* initial_translated_1 = alloc.translate(1000); - const char* initial_translated_2 = alloc.translate(section_size + 1000); + const char* initial_translated_1 = alloc.translate_in_slab(1000); + const char* initial_translated_2 = alloc.translate_in_slab(section_size + 1000); _impl::SimulatedFailure::prime_mmap([](size_t size) { // Let the first allocation succeed and only the second one fail @@ -448,21 +455,22 @@ NONCONCURRENT_TEST_IF(Alloc_MapFailureRecovery, _impl::SimulatedFailure::is_enab CHECK_THROW(alloc.update_reader_view(section_size * 3 + page_size), std::bad_alloc); CHECK_EQUAL(initial_baseline, alloc.get_baseline()); CHECK_EQUAL(initial_version, alloc.get_mapping_version()); - CHECK_EQUAL(initial_translated_1, alloc.translate(1000)); - CHECK_EQUAL(initial_translated_2, alloc.translate(section_size + 1000)); + CHECK_EQUAL(initial_translated_1, alloc.translate_in_slab(1000)); + CHECK_EQUAL(initial_translated_2, alloc.translate_in_slab(section_size + 1000)); _impl::SimulatedFailure::prime_mmap(nullptr); alloc.update_reader_view(section_size * 3 + page_size); CHECK_EQUAL(alloc.get_baseline(), section_size * 3 + page_size); CHECK_EQUAL(initial_version, alloc.get_mapping_version()); // did not alter an existing mapping - CHECK_EQUAL(initial_translated_1, alloc.translate(1000)); - CHECK_EQUAL(initial_translated_2, alloc.translate(section_size + 1000)); - CHECK_EQUAL(0, *alloc.translate(section_size * 2 + 1000)); + CHECK_EQUAL(initial_translated_1, alloc.translate_in_slab(1000)); + CHECK_EQUAL(initial_translated_2, alloc.translate_in_slab(section_size + 1000)); + CHECK_EQUAL(0, *alloc.translate_in_slab(section_size * 2 + 1000)); alloc.purge_old_mappings(5, 5); } } + // This test reproduces the sporadic issue that was seen for large refs (addresses) // on 32-bit iPhone 5 Simulator runs on certain host machines. TEST(Alloc_ToAndFromRef) diff --git a/test/test_array.cpp b/test/test_array.cpp index 637a394670c..a77c698b7fa 100644 --- a/test/test_array.cpp +++ b/test/test_array.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include "test.hpp" @@ -95,6 +96,27 @@ void has_zero_byte(TestContext& test_context, int64_t value, size_t reps) } // anonymous namespace +TEST(Array_Bits) +{ + CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(0), 0); + CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(1), 1); + CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(2), 2); + CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(3), 2); + CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(4), 3); + CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(5), 3); + CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(7), 3); + CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(8), 4); + CHECK_EQUAL(NodeHeader::signed_to_num_bits(0), 1); + CHECK_EQUAL(NodeHeader::signed_to_num_bits(1), 2); + CHECK_EQUAL(NodeHeader::signed_to_num_bits(-1), 1); + CHECK_EQUAL(NodeHeader::signed_to_num_bits(-2), 2); + CHECK_EQUAL(NodeHeader::signed_to_num_bits(-3), 3); + CHECK_EQUAL(NodeHeader::signed_to_num_bits(-4), 3); + CHECK_EQUAL(NodeHeader::signed_to_num_bits(3), 3); + CHECK_EQUAL(NodeHeader::signed_to_num_bits(4), 4); + CHECK_EQUAL(NodeHeader::signed_to_num_bits(7), 4); +} + TEST(Array_General) { Array c(Allocator::get_default()); @@ -1559,4 +1581,555 @@ NONCONCURRENT_TEST(Array_count) c.destroy(); } +TEST(DirectBitFields) +{ + uint64_t a[2]; + a[0] = a[1] = 0; + { + BfIterator it(a, 0, 7, 7, 8); + REALM_ASSERT(*it == 0); + auto it2(it); + ++it2; + it2.set_value(127 + 128); + REALM_ASSERT(*it == 0); + ++it; + REALM_ASSERT(*it == 127); + ++it; + REALM_ASSERT(*it == 0); + } + // reverse polarity + a[0] = a[1] = -1ULL; + { + BfIterator it(a, 0, 7, 7, 8); + REALM_ASSERT(*it == 127); + auto it2(it); + ++it2; + it2.set_value(42 + 128); + REALM_ASSERT(*it == 127); + ++it; + REALM_ASSERT(*it == 42); + ++it; + REALM_ASSERT(*it == 127); + } +} + +TEST(Extended_Array_encoding) +{ + using Encoding = NodeHeader::Encoding; + Array array(Allocator::get_default()); + auto mem = array.get_alloc().alloc(10); + init_header(mem.get_addr(), Encoding::Flex, 7, 1, 1, 1, 1); + array.init_from_mem(mem); + auto array_header = array.get_header(); + auto encoding = array.get_encoding(array_header); + CHECK(encoding == Encoding::Flex); + + Array another_array(Allocator::get_default()); + another_array.init_from_ref(array.get_ref()); + auto another_header = another_array.get_header(); + auto another_encoding = another_array.get_encoding(another_header); + CHECK(encoding == another_encoding); + + array.get_alloc().free_(mem); +} + +TEST(Array_cares_about) +{ + std::vector expected{ + 0xffffffffffffffff, 0xffffffffffffffff, 0xffffffffffffffff, 0x7fffffffffffffff, 0xffffffffffffffff, + 0xfffffffffffffff, 0xfffffffffffffff, 0x7fffffffffffffff, 0xffffffffffffffff, 0x7fffffffffffffff, + 0xfffffffffffffff, 0x7fffffffffffff, 0xfffffffffffffff, 0xfffffffffffff, 0xffffffffffffff, + 0xfffffffffffffff, 0xffffffffffffffff, 0x7ffffffffffff, 0x3fffffffffffff, 0x1ffffffffffffff, + 0xfffffffffffffff, 0x7fffffffffffffff, 0xfffffffffff, 0x3fffffffffff, 0xffffffffffff, + 0x3ffffffffffff, 0xfffffffffffff, 0x3fffffffffffff, 0xffffffffffffff, 0x3ffffffffffffff, + 0xfffffffffffffff, 0x3fffffffffffffff, 0xffffffffffffffff, 0x1ffffffff, 0x3ffffffff, + 0x7ffffffff, 0xfffffffff, 0x1fffffffff, 0x3fffffffff, 0x7fffffffff, + 0xffffffffff, 0x1ffffffffff, 0x3ffffffffff, 0x7ffffffffff, 0xfffffffffff, + 0x1fffffffffff, 0x3fffffffffff, 0x7fffffffffff, 0xffffffffffff, 0x1ffffffffffff, + 0x3ffffffffffff, 0x7ffffffffffff, 0xfffffffffffff, 0x1fffffffffffff, 0x3fffffffffffff, + 0x7fffffffffffff, 0xffffffffffffff, 0x1ffffffffffffff, 0x3ffffffffffffff, 0x7ffffffffffffff, + 0xfffffffffffffff, 0x1fffffffffffffff, 0x3fffffffffffffff, 0x7fffffffffffffff, 0xffffffffffffffff}; + std::vector res; + for (uint8_t i = 0; i <= 64; i++) { + res.push_back(cares_about(i)); + } + CHECK_EQUAL(res, expected); +} + +TEST(AlignDirectBitFields) +{ + uint64_t a[2]; + a[0] = a[1] = 0; + { + BfIterator it(a, 0, 7, 7, 8); + REALM_ASSERT(*it == 0); + auto it2(it); + ++it2; + it2.set_value(127 + 128); + REALM_ASSERT(*it == 0); + ++it; + REALM_ASSERT(*it == 127); + ++it; + REALM_ASSERT(*it == 0); + } + // reverse polarity + a[0] = a[1] = -1ULL; + { + BfIterator it(a, 0, 7, 7, 8); + REALM_ASSERT(*it == 127); + auto it2(it); + ++it2; + it2.set_value(42 + 128); + REALM_ASSERT(*it == 127); + ++it; + REALM_ASSERT(*it == 42); + ++it; + REALM_ASSERT(*it == 127); + } +} + +TEST(TestSignValuesStoredIterator) +{ + { + // positive values are easy. + uint64_t a[2]; + BfIterator it(a, 0, 8, 8, 0); + for (size_t i = 0; i < 16; ++i) { + it.set_value(i); + ++it; + } + it.move(0); + for (size_t i = 0; i < 16; ++i) { + auto v = *it; + CHECK_EQUAL(v, i); + ++it; + } + } + { + // negative values require a bit more work + uint64_t a[2]; + BfIterator it(a, 0, 8, 8, 0); + for (size_t i = 0; i < 16; ++i) { + it.set_value(-i); + ++it; + } + it.move(0); + for (int64_t i = 0; i < 16; ++i) { + const auto sv = sign_extend_value(8, *it); + CHECK_EQUAL(sv, -i); + ++it; + } + it.move(0); + const auto sign_mask = 1ULL << (7); + for (int64_t i = 0; i < 16; ++i) { + const auto sv = sign_extend_field_by_mask(sign_mask, *it); + CHECK_EQUAL(sv, -i); + ++it; + } + } +} + +TEST(VerifyIterationAcrossWords) +{ + uint64_t a[4]{0, 0, 0, 0}; // 4 64 bit words, let's store N elements of 5bits each + BfIterator it(a, 0, 5, 5, 0); + // 51 is the max amount of values we can fit in 4 words. Writting beyond this point is likely going + // to crash. Writing beyond the 4 words is not possible in practice because the Array has boundery checks + // and enough memory is reserved during compression. + srand((unsigned)time(0)); // no need to use complex stuff + std::vector values; + for (size_t i = 0; i < 51; i++) { + int64_t randomNumber = rand() % 16; // max value that can fit in 5 bits (4 bit for the value + 1 sign) + values.push_back(randomNumber); + it.set_value(randomNumber); + ++it; + } + + { + // normal bf iterator + it.move(0); // reset to first value. + // go through the memory, 5 bits by 5 bits. + // every 12 values, we will read the value across + // 2 words. + for (size_t i = 0; i < 51; ++i) { + const auto v = sign_extend_value(5, *it); + CHECK_EQUAL(v, values[i]); + ++it; + } + } + + { + // unaligned iterator + UnalignedWordIter u_it(a, 0); + for (size_t i = 0; i < 51; ++i) { + const auto v = sign_extend_value(5, u_it.consume(5) & 0x1F); + CHECK_EQUAL(v, values[i]); + } + } +} + +TEST(LowerBoundCorrectness) +{ + constexpr auto size = 16; + int64_t a[size]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; + + std::vector expected_default_lb; + for (const auto v : a) { + const auto pos = lower_bound<64>((const char*)(&a), size, v); + expected_default_lb.push_back(pos); + } + + // now simulate the compression of a in less bits + uint64_t buff[2] = {0, 0}; + BfIterator it(buff, 0, 5, 5, 0); // 5 bits because 4 bits for the values + 1 bit for the sign + for (size_t i = 0; i < 16; ++i) { + it.set_value(i); + CHECK_EQUAL(*it, a[i]); + ++it; + } + struct MyClass { + uint64_t* _data; + int64_t get(size_t ndx) const + { + BfIterator it(_data, 0, 5, 5, ndx); + return sign_extend_value(5, *it); + } + }; + // a bit of set up here. + MyClass my_class; + my_class._data = buff; + using Fetcher = impl::CompressedDataFetcher; + Fetcher my_fetcher; + my_fetcher.ptr = &my_class; + + // verify that the fetcher returns the same values + for (size_t i = 0; i < size; ++i) { + CHECK_EQUAL(my_fetcher.ptr->get(i), a[i]); + } + + std::vector diffent_width_lb; + for (const auto v : a) { + const auto pos = lower_bound((const char*)buff, size, v, my_fetcher); + diffent_width_lb.push_back(pos); + } + + CHECK_EQUAL(expected_default_lb, diffent_width_lb); +} + +TEST(UpperBoundCorrectness) +{ + constexpr auto size = 16; + int64_t a[size]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; + std::vector expected_default_ub; + for (const auto v : a) { + const auto pos = upper_bound<64>((const char*)(&a), size, v); + expected_default_ub.push_back(pos); + } + + // now simulate the compression of a in less bits + uint64_t buff[2] = {0, 0}; + BfIterator it(buff, 0, 5, 5, 0); // 5 bits because 4 bits for the values + 1 bit for the sign + for (size_t i = 0; i < size; ++i) { + it.set_value(i); + CHECK_EQUAL(*it, a[i]); + ++it; + } + struct MyClass { + uint64_t* _data; + int64_t get(size_t ndx) const + { + BfIterator it(_data, 0, 5, 5, ndx); + return sign_extend_value(5, *it); + } + }; + // a bit of set up here. + MyClass my_class; + my_class._data = buff; + using Fetcher = impl::CompressedDataFetcher; + Fetcher my_fetcher; + my_fetcher.ptr = &my_class; + + // verify that the fetcher returns the same values + for (size_t i = 0; i < size; ++i) { + CHECK_EQUAL(my_fetcher.ptr->get(i), a[i]); + } + + std::vector diffent_width_ub; + for (const auto v : a) { + const auto pos = upper_bound((const char*)buff, size, v, my_fetcher); + diffent_width_ub.push_back(pos); + } + + CHECK_EQUAL(expected_default_ub, diffent_width_ub); +} + +TEST(ParallelSearchEqualMatch) +{ + std::mt19937_64 gen64; + constexpr size_t buflen = 4; + uint64_t buff[buflen]; + std::vector values; + for (uint8_t width = 1; width <= 64; width++) { + const size_t size = (buflen * 64) / width; + const uint64_t bit_mask = 0xFFFFFFFFFFFFFFFFULL >> (64 - width); // (1ULL << width) - 1; + + values.resize(size); + BfIterator it(buff, 0, width, width, 0); + for (size_t i = 0; i < size; ++i) { + uint64_t u_val = gen64() & bit_mask; + int64_t val = sign_extend_value(width, u_val); + values[i] = val; + it.set_value(val); + ++it; + } + it.move(0); + for (size_t i = 0; i < size; ++i) { + auto v = sign_extend_value(width, *it); + CHECK_EQUAL(v, values[i]); + ++it; + } + + std::sort(values.begin(), values.end()); + auto last = std::unique(values.begin(), values.end()); + for (auto val = values.begin(); val != last; val++) { + const auto mask = 1ULL << (width - 1); + const auto msb = populate(width, mask); + const auto search_vector = populate(width, *val); + + // perform the check with a normal iteration + size_t start = 0; + const auto end = size; + std::vector linear_scan_result; + while (start < end) { + it.move(start); + const auto sv = sign_extend_value(width, *it); + if (sv == *val) + linear_scan_result.push_back(start); + ++start; + } + + // Now use the optimized version + static auto vector_compare_eq = [](auto msb, auto a, auto b) { + return find_all_fields(msb, a, b); + }; + + start = 0; + std::vector parallel_result; + while (start < end) { + start = + parallel_subword_find(vector_compare_eq, buff, size_t{0}, width, msb, search_vector, start, end); + if (start != end) + parallel_result.push_back(start); + start += 1; + } + + CHECK(!parallel_result.empty()); + CHECK(!linear_scan_result.empty()); + CHECK_EQUAL(linear_scan_result, parallel_result); + } + } +} + +TEST(ParallelSearchEqualNoMatch) +{ + uint64_t buff[2] = {0, 0}; + constexpr size_t width = 2; + constexpr size_t size = 64; + constexpr int64_t key = 2; + BfIterator it(buff, 0, width, width, 0); + for (size_t i = 0; i < size; ++i) { + it.set_value(1); + ++it; + } + it.move(0); + for (size_t i = 0; i < size; ++i) { + auto v = sign_extend_value(width, *it); + CHECK_EQUAL(v, 1); + ++it; + } + const auto mask = 1ULL << (width - 1); + const auto msb = populate(width, mask); + const auto search_vector = populate(width, key); + + static auto vector_compare_eq = [](auto msb, auto a, auto b) { + return find_all_fields(msb, a, b); + }; + + size_t start = 0; + const auto end = size; + std::vector parallel_result; + while (start < end) { + start = parallel_subword_find(vector_compare_eq, buff, size_t{0}, width, msb, search_vector, start, end); + if (start != end) + parallel_result.push_back(start); + start += 1; + } + + // perform the same check but with a normal iteration + start = 0; + std::vector linear_scan_result; + while (start < end) { + it.move(start); + const auto sv = sign_extend_value(width, *it); + if (sv == key) + linear_scan_result.push_back(start); + ++start; + } + + CHECK(parallel_result.empty()); + CHECK(linear_scan_result.empty()); +} + +TEST(ParallelSearchNotEqual) +{ + uint64_t buff[2] = {0, 0}; + constexpr size_t width = 2; + constexpr size_t size = 64; + constexpr int64_t key = 2; + BfIterator it(buff, 0, width, width, 0); + for (size_t i = 0; i < size; ++i) { + it.set_value(1); + ++it; + } + it.move(0); + for (size_t i = 0; i < size; ++i) { + auto v = sign_extend_value(width, *it); + CHECK_EQUAL(v, 1); + ++it; + } + const auto mask = 1ULL << (width - 1); + const auto msb = populate(width, mask); + const auto search_vector = populate(width, key); + + static auto vector_compare_neq = [](auto msb, auto a, auto b) { + return find_all_fields(msb, a, b); + }; + + size_t start = 0; + const auto end = size; + std::vector parallel_result; + while (start < end) { + start = parallel_subword_find(vector_compare_neq, buff, size_t{0}, width, msb, search_vector, start, end); + if (start != end) + parallel_result.push_back(start); + start += 1; + } + + // perform the same check but with a normal iteration + start = 0; + std::vector linear_scan_result; + while (start < end) { + it.move(start); + const auto sv = sign_extend_value(width, *it); + if (sv != key) + linear_scan_result.push_back(start); + ++start; + } + + CHECK(!parallel_result.empty()); + CHECK(!linear_scan_result.empty()); + CHECK_EQUAL(parallel_result, linear_scan_result); +} + +TEST(ParallelSearchLessThan) +{ + uint64_t buff[2] = {0, 0}; + constexpr size_t width = 4; + constexpr size_t size = 32; + constexpr int64_t key = 3; + BfIterator it(buff, 0, width, width, 0); + for (size_t i = 0; i < size; ++i) { + it.set_value(2); + ++it; + } + it.move(0); + for (size_t i = 0; i < size; ++i) { + auto v = sign_extend_value(width, *it); + CHECK_EQUAL(v, 2); + ++it; + } + const auto mask = 1ULL << (width - 1); + const auto msb = populate(width, mask); + const auto search_vector = populate(width, key); + + static auto vector_compare_lt = [](auto msb, auto a, auto b) { + return find_all_fields(msb, a, b); + }; + + size_t start = 0; + const auto end = size; + std::vector parallel_result; + while (start < end) { + start = parallel_subword_find(vector_compare_lt, buff, size_t{0}, width, msb, search_vector, start, end); + if (start != end) + parallel_result.push_back(start); + start += 1; + } + + // perform the same check but with a normal iteration + start = 0; + std::vector linear_scan_result; + while (start < end) { + it.move(start); + const auto sv = sign_extend_value(width, *it); + if (sv < key) + linear_scan_result.push_back(start); + ++start; + } + CHECK(!parallel_result.empty()); + CHECK(!linear_scan_result.empty()); + CHECK_EQUAL(parallel_result, linear_scan_result); +} + +TEST(ParallelSearchGreaterThan) +{ + uint64_t buff[2] = {0, 0}; + constexpr size_t width = 4; + constexpr size_t size = 32; + constexpr int64_t key = 2; + BfIterator it(buff, 0, width, width, 0); + for (size_t i = 0; i < size; ++i) { + it.set_value(3); + ++it; + } + it.move(0); + for (size_t i = 0; i < size; ++i) { + auto v = sign_extend_value(width, *it); + CHECK_EQUAL(v, 3); + ++it; + } + const auto mask = 1ULL << (width - 1); + const auto msb = populate(width, mask); + const auto search_vector = populate(width, key); + + static auto vector_compare_gt = [](auto msb, auto a, auto b) { + return find_all_fields(msb, a, b); + }; + + size_t start = 0; + const auto end = size; + std::vector parallel_result; + while (start < end) { + start = parallel_subword_find(vector_compare_gt, buff, size_t{0}, width, msb, search_vector, start, end); + if (start != end) + parallel_result.push_back(start); + start += 1; + } + + // perform the same check but with a normal iteration + start = 0; + std::vector linear_scan_result; + while (start < end) { + it.move(start); + const auto sv = sign_extend_value(width, *it); + if (sv > key) + linear_scan_result.push_back(start); + ++start; + } + CHECK(!parallel_result.empty()); + CHECK(!linear_scan_result.empty()); + CHECK_EQUAL(parallel_result, linear_scan_result); +} + + #endif // TEST_ARRAY diff --git a/test/test_array_integer.cpp b/test/test_array_integer.cpp index a26cecf52b2..9ccdf25653e 100644 --- a/test/test_array_integer.cpp +++ b/test/test_array_integer.cpp @@ -19,6 +19,7 @@ #include "testsettings.hpp" #include +#include #include #include @@ -31,6 +32,1575 @@ using namespace realm; using namespace realm::test_util; +// #define ARRAY_PERFORMANCE_TESTING +#if !defined(REALM_DEBUG) && defined(ARRAY_PERFORMANCE_TESTING) +NONCONCURRENT_TEST(perf_array_encode_get_vs_array_get_less_32bit) +{ + using namespace std; + using namespace std::chrono; + size_t n_values = 1000; + size_t n_runs = 100; + std::cout << " < 32 bit values " << std::endl; + std::cout << " N values = " << n_values << std::endl; + std::cout << " N runs = " << n_runs << std::endl; + + std::vector input_array; + ArrayInteger a(Allocator::get_default()); + ArrayInteger compressed_a(Allocator::get_default()); + a.create(); + + for (size_t i = 0; i < n_values; i++) + input_array.push_back(i); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(input_array.begin(), input_array.end(), g); + for (const auto& v : input_array) + a.add(v); + + auto t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) + REALM_ASSERT(a.get(i) == input_array[i]); + } + auto t2 = high_resolution_clock::now(); + + std::cout << " Positive values - Array::get(): " << duration_cast(t2 - t1).count() << " ns" + << std::endl; + std::cout << " Positive values - Array::get(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + t1 = high_resolution_clock::now(); + + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + REALM_ASSERT(compressed_a.get(i) == a.get(i)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Positive values - ArrayCompress::get(): " << duration_cast(t2 - t1).count() << " ns" + << std::endl; + std::cout << " Positive values - ArrayCompress::get(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.destroy(); + compressed_a.destroy(); + a.create(); + input_array.clear(); + for (size_t i = 0; i < n_values; i++) + input_array.push_back(-i); + std::random_device rd1; + std::mt19937 g1(rd1()); + std::shuffle(input_array.begin(), input_array.end(), g1); + for (const auto& v : input_array) + a.add(v); + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) + REALM_ASSERT(a.get(i) == input_array[i]); + } + t2 = high_resolution_clock::now(); + + std::cout << std::endl; + + std::cout << " Negative values - Array::get(): " << duration_cast(t2 - t1).count() << " ns" + << std::endl; + std::cout << " Negative values - Array::get(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + REALM_ASSERT(compressed_a.get(i) == a.get(i)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Negative values - ArrayCompress::get(): " << duration_cast(t2 - t1).count() << " ns" + << std::endl; + std::cout << " Negative values - ArrayCompress::get(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.destroy(); + compressed_a.destroy(); +} + + +NONCONCURRENT_TEST(Test_basic_find_EQ_less_32bit) +{ + using namespace std; + using namespace std::chrono; + size_t n_values = 1000; + size_t n_runs = 100; + std::cout << " Value with bitwidth < 32 " << std::endl; + std::cout << " N values = " << n_values << std::endl; + std::cout << " N runs = " << n_runs << std::endl; + + std::vector input_array; + ArrayInteger a(Allocator::get_default()); + ArrayInteger compressed_a(Allocator::get_default()); + a.create(); + + for (size_t i = 0; i < n_values; i++) + input_array.push_back(i); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(input_array.begin(), input_array.end(), g); + for (const auto& v : input_array) + a.add(v); + + auto t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + auto ndx = a.find_first(input_array[i]); + REALM_ASSERT(ndx != realm::not_found); + REALM_ASSERT(a.get(ndx) == input_array[ndx]); + } + } + auto t2 = high_resolution_clock::now(); + + std::cout << " Positive values - Array::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Positive values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + auto v = a.find_first(input_array[i]); + auto v1 = compressed_a.find_first(input_array[i]); + REALM_ASSERT(v == v1); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + auto ndx = compressed_a.find_first(input_array[i]); + REALM_ASSERT(ndx != realm::not_found); + REALM_ASSERT(compressed_a.get(ndx) == input_array[ndx]); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Positive values - ArrayCompress::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Positive values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + std::cout << std::endl; + + a.destroy(); + compressed_a.destroy(); + a.create(); + input_array.clear(); + for (size_t i = 0; i < n_values; i++) + input_array.push_back(-i); + std::random_device rd1; + std::mt19937 g1(rd1()); + std::shuffle(input_array.begin(), input_array.end(), g1); + for (const auto& v : input_array) + a.add(v); + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + auto v = a.find_first(input_array[i]); + auto v1 = compressed_a.find_first(input_array[i]); + REALM_ASSERT(v == v1); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + auto ndx = a.find_first(input_array[i]); + REALM_ASSERT(ndx != realm::not_found); + REALM_ASSERT(a.get(ndx) == input_array[ndx]); + } + } + t2 = high_resolution_clock::now(); + + std::cout << " Negative values - Array::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Negative values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + auto ndx = compressed_a.find_first(input_array[i]); + REALM_ASSERT(ndx != realm::not_found); + REALM_ASSERT(compressed_a.get(ndx) == a.get(ndx)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Negative values - ArrayCompress::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Negative values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.destroy(); + compressed_a.destroy(); +} + +NONCONCURRENT_TEST(Test_basic_find_NEQ_value_less_32bit) +{ + using namespace std; + using namespace std::chrono; + size_t n_values = 1000; + size_t n_runs = 100; + std::cout << " Value with bitwidth < 32 " << std::endl; + std::cout << " N values = " << n_values << std::endl; + std::cout << " N runs = " << n_runs << std::endl; + + std::vector input_array; + ArrayInteger a(Allocator::get_default()); + ArrayInteger compressed_a(Allocator::get_default()); + a.create(); + + for (size_t i = 0; i < n_values; i++) + input_array.push_back(i); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(input_array.begin(), input_array.end(), g); + for (const auto& v : input_array) + a.add(v); + + QueryStateFindFirst state1; + QueryStateFindFirst state2; + auto t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(i, 0, a.size(), &state1); + REALM_ASSERT(state1.m_state != realm::not_found); + REALM_ASSERT(a.get(state1.m_state) == input_array[state1.m_state]); + } + } + auto t2 = high_resolution_clock::now(); + + std::cout << " Positive values - Array::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Positive values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(i, 0, a.size(), &state1); + compressed_a.find(i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state1.m_state == state2.m_state); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + compressed_a.find(i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state2.m_state != realm::not_found); + REALM_ASSERT(compressed_a.get(state2.m_state) == a.get(state2.m_state)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Positive values - ArrayCompress::find(): " + << duration_cast(t2 - t1).count() << " ms" << std::endl; + std::cout << " Positive values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + std::cout << std::endl; + + a.destroy(); + compressed_a.destroy(); + a.create(); + input_array.clear(); + for (size_t i = 0; i < n_values; i++) + input_array.push_back(-i); + std::random_device rd1; + std::mt19937 g1(rd1()); + std::shuffle(input_array.begin(), input_array.end(), g1); + for (const auto& v : input_array) + a.add(v); + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // NEQ for signed integers is not working. TODO: investigate this. + // verify that both find the same thing + + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(-i, 0, a.size(), &state1); + compressed_a.find(-i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state1.m_state == state2.m_state); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(-i, 0, a.size(), &state1); + REALM_ASSERT(state1.m_state != realm::not_found); + REALM_ASSERT(a.get(state1.m_state) == input_array[state1.m_state]); + } + } + t2 = high_resolution_clock::now(); + + std::cout << " Negative values - Array::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Negative values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + compressed_a.find(-i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state2.m_state != realm::not_found); + REALM_ASSERT(compressed_a.get(state2.m_state) == a.get(state2.m_state)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Negative values - ArrayCompress::find(): " + << duration_cast(t2 - t1).count() << " ms" << std::endl; + std::cout << " Negative values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.destroy(); + compressed_a.destroy(); +} + +NONCONCURRENT_TEST(Test_basic_find_LT_value_less_32bit) +{ + using namespace std; + using namespace std::chrono; + size_t n_values = 1000; + size_t n_runs = 100; + std::cout << " Value with bitwidth < 32 " << std::endl; + std::cout << " N values = " << n_values << std::endl; + std::cout << " N runs = " << n_runs << std::endl; + + std::vector input_array; + ArrayInteger a(Allocator::get_default()); + ArrayInteger compressed_a(Allocator::get_default()); + a.create(); + + for (size_t i = 0; i < n_values; i++) + input_array.push_back(i); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(input_array.begin(), input_array.end(), g); + for (const auto& v : input_array) + a.add(v); + + QueryStateFindFirst state1{}; + QueryStateFindFirst state2{}; + auto t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 1; i < n_values; ++i) { // there is nothing less than 0 + a.find(i, 0, a.size(), &state1); + REALM_ASSERT(state1.m_state != realm::not_found); + REALM_ASSERT(a.get(state1.m_state) == input_array[state1.m_state]); + } + } + auto t2 = high_resolution_clock::now(); + + std::cout << " Positive values - Array::find(): " << duration_cast(t2 - t1).count() << " ms" + << std::endl; + std::cout << " Positive values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + state1 = {}; + state2 = {}; + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(i, 0, a.size(), &state1); + compressed_a.find(i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state1.m_state == state2.m_state); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 1; i < n_values; ++i) { // there is nothing less than 0 + compressed_a.find(i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state2.m_state != realm::not_found); + REALM_ASSERT(compressed_a.get(state2.m_state) == a.get(state2.m_state)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Positive values - ArrayCompress::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Positive values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + std::cout << std::endl; + + a.destroy(); + compressed_a.destroy(); + a.create(); + input_array.clear(); + for (size_t i = 0; i < n_values; i++) + input_array.push_back(-i); + std::random_device rd1; + std::mt19937 g1(rd1()); + std::shuffle(input_array.begin(), input_array.end(), g1); + for (const auto& v : input_array) + a.add(v); + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + state1 = {}; + state2 = {}; + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(-i, 0, a.size(), &state1); + compressed_a.find(-i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state1.m_state == state2.m_state); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values - 1; ++i) { // nothing less than the biggest negative number + a.find(-i, 0, a.size(), &state1); + REALM_ASSERT(state1.m_state != realm::not_found); + REALM_ASSERT(a.get(state1.m_state) == input_array[state1.m_state]); + } + } + t2 = high_resolution_clock::now(); + + std::cout << " Negative values - Array::find(): " << duration_cast(t2 - t1).count() << " ms" + << std::endl; + std::cout << " Negative values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values - 1; ++i) { // nothing less than the biggest negative number + compressed_a.find(-i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state2.m_state != realm::not_found); + REALM_ASSERT(compressed_a.get(state2.m_state) == a.get(state2.m_state)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Negative values - ArrayCompress::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Negative values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.destroy(); + compressed_a.destroy(); +} + +NONCONCURRENT_TEST(Test_basic_find_GT_value_less_32bit) +{ + // GT subword parallel search is not working... TODO : investigate + using namespace std; + using namespace std::chrono; + size_t n_values = 1000; + size_t n_runs = 100; + std::cout << " Value with bitwidth < 32 " << std::endl; + std::cout << " N values = " << n_values << std::endl; + std::cout << " N runs = " << n_runs << std::endl; + + std::vector input_array; + ArrayInteger a(Allocator::get_default()); + ArrayInteger compressed_a(Allocator::get_default()); + a.create(); + + for (size_t i = 0; i < n_values; i++) + input_array.push_back(i); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(input_array.begin(), input_array.end(), g); + for (const auto& v : input_array) + a.add(v); + + QueryStateFindFirst state1; + QueryStateFindFirst state2; + auto t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values - 1; ++i) { // nothing greatest than the last number + a.find(i, 0, a.size(), &state1); + REALM_ASSERT(state1.m_state != realm::not_found); + REALM_ASSERT(a.get(state1.m_state) == input_array[state1.m_state]); + } + } + auto t2 = high_resolution_clock::now(); + + std::cout << " Positive values - Array::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Positive values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + state1 = {}; + state2 = {}; + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(i, 0, a.size(), &state1); + compressed_a.find(i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state1.m_state == state2.m_state); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values - 1; ++i) { // nothing bigger than the last val + compressed_a.find(i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state2.m_state != realm::not_found); + REALM_ASSERT(compressed_a.get(state2.m_state) == a.get(state2.m_state)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Positive values - ArrayCompress::find(): " + << duration_cast(t2 - t1).count() << " ms" << std::endl; + std::cout << " Positive values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + std::cout << std::endl; + + a.destroy(); + compressed_a.destroy(); + a.create(); + input_array.clear(); + for (size_t i = 0; i < n_values; i++) + input_array.push_back(-i); + std::random_device rd1; + std::mt19937 g1(rd1()); + std::shuffle(input_array.begin(), input_array.end(), g1); + for (const auto& v : input_array) + a.add(v); + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + state1 = {}; + state2 = {}; + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(-i, 0, a.size(), &state1); + compressed_a.find(-i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state1.m_state == state2.m_state); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 1; i < n_values; ++i) { // nothing bigger than 0 + a.find(-i, 0, a.size(), &state1); + REALM_ASSERT(state1.m_state != realm::not_found); + REALM_ASSERT(a.get(state1.m_state) == input_array[state1.m_state]); + } + } + t2 = high_resolution_clock::now(); + + std::cout << " Negative values - Array::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Negative values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 1; i < n_values; ++i) { // nothing bigger than 0 + compressed_a.find(-i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state2.m_state != realm::not_found); + REALM_ASSERT(compressed_a.get(state2.m_state) == a.get(state2.m_state)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Negative values - ArrayCompress::find(): " + << duration_cast(t2 - t1).count() << " ms" << std::endl; + std::cout << " Negative values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.destroy(); + compressed_a.destroy(); +} + +NONCONCURRENT_TEST(perf_array_encode_get_vs_array_get_greater_32bit) +{ + using namespace std; + using namespace std::chrono; + size_t start_value = 0x0000000100000000; // 32 bit val + size_t n_values = 1000; + size_t n_runs = 100; + std::cout << " >= 32 bit values " << std::endl; + std::cout << " N values = " << n_values << std::endl; + std::cout << " N runs = " << n_runs << std::endl; + + std::vector input_array; + ArrayInteger a(Allocator::get_default()); + ArrayInteger compressed_a(Allocator::get_default()); + a.create(); + + for (size_t i = 0; i < n_values; i++) + input_array.push_back(start_value + i); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(input_array.begin(), input_array.end(), g); + for (const auto& v : input_array) + a.add(v); + + auto t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) + REALM_ASSERT(a.get(i) == input_array[i]); + } + auto t2 = high_resolution_clock::now(); + + std::cout << " Positive values - Array::get(): " << duration_cast(t2 - t1).count() << " ns" + << std::endl; + std::cout << " Positive values - Array::get(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + t1 = high_resolution_clock::now(); + + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + REALM_ASSERT(compressed_a.get(i) == a.get(i)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Positive values - ArrayCompress::get(): " << duration_cast(t2 - t1).count() << " ns" + << std::endl; + std::cout << " Positive values - ArrayCompress::get(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.destroy(); + compressed_a.destroy(); + a.create(); + input_array.clear(); + for (size_t i = 0; i < n_values; i++) + input_array.push_back(-i); + std::random_device rd1; + std::mt19937 g1(rd1()); + std::shuffle(input_array.begin(), input_array.end(), g1); + for (const auto& v : input_array) + a.add(v); + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) + REALM_ASSERT(a.get(i) == input_array[i]); + } + t2 = high_resolution_clock::now(); + + std::cout << std::endl; + + std::cout << " Negative values - Array::get(): " << duration_cast(t2 - t1).count() << " ns" + << std::endl; + std::cout << " Negative values - Array::get(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + REALM_ASSERT(compressed_a.get(i) == a.get(i)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Negative values - ArrayCompress::get(): " << duration_cast(t2 - t1).count() << " ns" + << std::endl; + std::cout << " Negative values - ArrayCompress::get(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.destroy(); + compressed_a.destroy(); +} + +NONCONCURRENT_TEST(Test_basic_find_EQ_greater_32bit) +{ + using namespace std; + using namespace std::chrono; + size_t start_value = 0x000001000000000; // 32 bit val + size_t n_values = 1000; + size_t n_runs = 100; + std::cout << " Value with bitwidth >= 32 " << std::endl; + std::cout << " N values = " << n_values << std::endl; + std::cout << " N runs = " << n_runs << std::endl; + + std::vector input_array; + ArrayInteger a(Allocator::get_default()); + ArrayInteger compressed_a(Allocator::get_default()); + a.create(); + + for (size_t i = 0; i < n_values; i++) + input_array.push_back(start_value + i); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(input_array.begin(), input_array.end(), g); + for (const auto& v : input_array) + a.add(v); + + auto t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + auto ndx = a.find_first(start_value + i); + REALM_ASSERT(ndx != realm::not_found); + REALM_ASSERT(a.get(ndx) == input_array[ndx]); + } + } + auto t2 = high_resolution_clock::now(); + + std::cout << " Positive values - Array::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Positive values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + REALM_ASSERT(a.find_first(start_value + i) == compressed_a.find_first(start_value + i)); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + auto ndx = compressed_a.find_first(start_value + i); + REALM_ASSERT(ndx != realm::not_found); + REALM_ASSERT(compressed_a.get(ndx) == a.get(ndx)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Positive values - ArrayCompress::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Positive values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + std::cout << std::endl; + + a.destroy(); + compressed_a.destroy(); + a.create(); + input_array.clear(); + for (size_t i = 0; i < n_values; i++) + input_array.push_back(-(start_value + i)); + std::random_device rd1; + std::mt19937 g1(rd1()); + std::shuffle(input_array.begin(), input_array.end(), g1); + for (const auto& v : input_array) + a.add(v); + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + const auto k = -(start_value + i); + const auto v1 = a.find_first(k); + const auto v2 = compressed_a.find_first(k); + REALM_ASSERT(v1 == v2); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + auto ndx = a.find_first(-(start_value + i)); + REALM_ASSERT(ndx != realm::not_found); + REALM_ASSERT(a.get(ndx) == input_array[ndx]); + } + } + t2 = high_resolution_clock::now(); + + std::cout << " Negative values - Array::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Negative values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + auto ndx = compressed_a.find_first(-(start_value + i)); + REALM_ASSERT(ndx != realm::not_found); + REALM_ASSERT(compressed_a.get(ndx) == a.get(ndx)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Negative values - ArrayCompress::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Negative values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.destroy(); + compressed_a.destroy(); +} + +NONCONCURRENT_TEST(Test_basic_find_NEQ_value_greater_32bit) +{ + using namespace std; + using namespace std::chrono; + size_t start_value = 0x0000000100000000; // 32 bit val + size_t n_values = 1000; + size_t n_runs = 100; + std::cout << " Value with bitwidth >= 32 " << std::endl; + std::cout << " N values = " << n_values << std::endl; + std::cout << " N runs = " << n_runs << std::endl; + + std::vector input_array; + ArrayInteger a(Allocator::get_default()); + ArrayInteger compressed_a(Allocator::get_default()); + a.create(); + + for (size_t i = 0; i < n_values; i++) + input_array.push_back(start_value + i); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(input_array.begin(), input_array.end(), g); + for (const auto& v : input_array) + a.add(v); + + QueryStateFindFirst state1; + QueryStateFindFirst state2; + auto t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(start_value + i, 0, a.size(), &state1); + REALM_ASSERT(state1.m_state != realm::not_found); + REALM_ASSERT(a.get(state1.m_state) == input_array[state1.m_state]); + } + } + auto t2 = high_resolution_clock::now(); + + std::cout << " Positive values - Array::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Positive values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(start_value + i, 0, a.size(), &state1); + compressed_a.find(start_value + i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state1.m_state == state2.m_state); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + compressed_a.find(start_value + i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state2.m_state != realm::not_found); + REALM_ASSERT(compressed_a.get(state2.m_state) == a.get(state2.m_state)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Positive values - ArrayCompress::find(): " + << duration_cast(t2 - t1).count() << " ms" << std::endl; + std::cout << " Positive values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + std::cout << std::endl; + + a.destroy(); + compressed_a.destroy(); + a.create(); + input_array.clear(); + for (size_t i = 0; i < n_values; i++) + input_array.push_back(-(start_value + i)); + std::random_device rd1; + std::mt19937 g1(rd1()); + std::shuffle(input_array.begin(), input_array.end(), g1); + for (const auto& v : input_array) + a.add(v); + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(-(start_value + i), 0, a.size(), &state1); + compressed_a.find(-(start_value + i), 0, compressed_a.size(), &state2); + REALM_ASSERT(state1.m_state == state2.m_state); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(-(start_value + i), 0, a.size(), &state1); + REALM_ASSERT(state1.m_state != realm::not_found); + REALM_ASSERT(a.get(state1.m_state) == input_array[state1.m_state]); + } + } + t2 = high_resolution_clock::now(); + + std::cout << " Negative values - Array::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Negative values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + compressed_a.find(-(start_value + i), 0, compressed_a.size(), &state2); + REALM_ASSERT(state2.m_state != realm::not_found); + REALM_ASSERT(compressed_a.get(state2.m_state) == a.get(state2.m_state)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Negative values - ArrayCompress::find(): " + << duration_cast(t2 - t1).count() << " ms" << std::endl; + std::cout << " Negative values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.destroy(); + compressed_a.destroy(); +} + +NONCONCURRENT_TEST(Test_basic_find_LT_value_greater_32bit) +{ + using namespace std; + using namespace std::chrono; + size_t start_value = 0x0000000100000000; // 32 bit val + size_t n_values = 1000; + size_t n_runs = 100; + std::cout << " Value with bitwidth >= 32 " << std::endl; + std::cout << " N values = " << n_values << std::endl; + std::cout << " N runs = " << n_runs << std::endl; + + std::vector input_array; + ArrayInteger a(Allocator::get_default()); + ArrayInteger compressed_a(Allocator::get_default()); + a.create(); + + for (size_t i = 0; i < n_values; i++) + input_array.push_back(start_value + i); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(input_array.begin(), input_array.end(), g); + for (const auto& v : input_array) + a.add(v); + + QueryStateFindFirst state1; + QueryStateFindFirst state2; + auto t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 1; i < n_values; ++i) { + a.find(start_value + i, 0, a.size(), &state1); + REALM_ASSERT(state1.m_state != realm::not_found); + REALM_ASSERT(a.get(state1.m_state) == input_array[state1.m_state]); + } + } + auto t2 = high_resolution_clock::now(); + + std::cout << " Positive values - Array::find(): " << duration_cast(t2 - t1).count() << " ms" + << std::endl; + std::cout << " Positive values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + state1 = {}; + state2 = {}; + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(start_value + i, 0, a.size(), &state1); + compressed_a.find(start_value + i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state1.m_state == state2.m_state); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 1; i < n_values; ++i) { + compressed_a.find(start_value + i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state2.m_state != realm::not_found); + REALM_ASSERT(compressed_a.get(state2.m_state) == a.get(state2.m_state)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Positive values - ArrayCompress::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Positive values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + std::cout << std::endl; + + a.destroy(); + compressed_a.destroy(); + a.create(); + input_array.clear(); + for (size_t i = 0; i < n_values; i++) + input_array.push_back(-(start_value + i)); + std::random_device rd1; + std::mt19937 g1(rd1()); + std::shuffle(input_array.begin(), input_array.end(), g1); + for (const auto& v : input_array) + a.add(v); + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(-(start_value + i), 0, a.size(), &state1); + compressed_a.find(-(start_value + i), 0, compressed_a.size(), &state2); + REALM_ASSERT(state1.m_state == state2.m_state); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(-(start_value + i), 0, a.size(), &state1); + REALM_ASSERT(state1.m_state != realm::not_found); + REALM_ASSERT(a.get(state1.m_state) == input_array[state1.m_state]); + } + } + t2 = high_resolution_clock::now(); + + std::cout << " Negative values - Array::find(): " << duration_cast(t2 - t1).count() << " ms" + << std::endl; + std::cout << " Negative values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + compressed_a.find(-(start_value + i), 0, compressed_a.size(), &state2); + REALM_ASSERT(state2.m_state != realm::not_found); + REALM_ASSERT(compressed_a.get(state2.m_state) == a.get(state2.m_state)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Negative values - ArrayCompress::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Negative values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.destroy(); + compressed_a.destroy(); +} + +NONCONCURRENT_TEST(Test_basic_find_GT_value_greater_32bit) +{ + using namespace std; + using namespace std::chrono; + size_t start_value = 0x0000100000000; // 32 bit val + size_t n_values = 1000; + size_t n_runs = 100; + std::cout << " Value with bitwidth >= 32 " << std::endl; + std::cout << " N values = " << n_values << std::endl; + std::cout << " N runs = " << n_runs << std::endl; + + std::vector input_array; + ArrayInteger a(Allocator::get_default()); + ArrayInteger compressed_a(Allocator::get_default()); + a.create(); + + for (size_t i = 0; i < n_values; i++) + input_array.push_back(start_value + i); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(input_array.begin(), input_array.end(), g); + for (const auto& v : input_array) + a.add(v); + + QueryStateFindFirst state1; + QueryStateFindFirst state2; + auto t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values - 1; ++i) { + a.find(start_value + i, 0, a.size(), &state1); + REALM_ASSERT(state1.m_state != realm::not_found); + REALM_ASSERT(a.get(state1.m_state) == input_array[state1.m_state]); + } + } + auto t2 = high_resolution_clock::now(); + + std::cout << " Positive values - Array::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Positive values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + state1 = {}; + state2 = {}; + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + const auto k = start_value + i; + a.find(k, 0, a.size(), &state1); + compressed_a.find(k, 0, compressed_a.size(), &state2); + REALM_ASSERT(state1.m_state == state2.m_state); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values - 1; ++i) { + compressed_a.find(start_value + i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state2.m_state != realm::not_found); + REALM_ASSERT(compressed_a.get(state2.m_state) == a.get(state2.m_state)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Positive values - ArrayCompress::find(): " + << duration_cast(t2 - t1).count() << " ms" << std::endl; + std::cout << " Positive values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + std::cout << std::endl; + + a.destroy(); + compressed_a.destroy(); + a.create(); + input_array.clear(); + for (size_t i = 0; i < n_values; i++) + input_array.push_back(-(start_value + i)); + std::random_device rd1; + std::mt19937 g1(rd1()); + std::shuffle(input_array.begin(), input_array.end(), g1); + for (const auto& v : input_array) + a.add(v); + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 1; i < n_values; ++i) { + a.find(-(start_value + i), 0, a.size(), &state1); + compressed_a.find(-(start_value + i), 0, compressed_a.size(), &state2); + REALM_ASSERT(state1.m_state == state2.m_state); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(-(start_value + i), 0, a.size(), &state1); + REALM_ASSERT(state1.m_state != realm::not_found); + REALM_ASSERT(a.get(state1.m_state) == input_array[state1.m_state]); + } + } + t2 = high_resolution_clock::now(); + + std::cout << " Negative values - Array::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Negative values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 1; i < n_values; ++i) { + compressed_a.find(-(start_value + i), 0, compressed_a.size(), &state2); + REALM_ASSERT(state2.m_state != realm::not_found); + REALM_ASSERT(compressed_a.get(state2.m_state) == a.get(state2.m_state)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Negative values - ArrayCompress::find(): " + << duration_cast(t2 - t1).count() << " ms" << std::endl; + std::cout << " Negative values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.destroy(); + compressed_a.destroy(); +} + +#endif + +// disable this test if forcing compression to Packed. +#if !REALM_COMPRESS +TEST(Test_ArrayInt_no_compress) +{ + ArrayInteger a(Allocator::get_default()); + ArrayInteger a1(Allocator::get_default()); + a.create(); + a.add(10); + a.add(11); + a.add(12); + // the original array is never encoded. a1 is the array to write down to disk + // in this case compression is not needed + CHECK_NOT(a.try_compress(a1)); + CHECK_NOT(a.is_compressed()); + CHECK(a.get(0) == 10); + CHECK(a.get(1) == 11); + CHECK(a.get(2) == 12); + a.destroy(); + a1.destroy(); +} + +TEST(Test_ArrayInt_compress_decompress_needed) +{ + ArrayInteger a(Allocator::get_default()); + ArrayInteger a1(Allocator::get_default()); + a.create(); + a.add(10); + a.add(5); + a.add(5); + // uncompressed requires 3 x 4 bits, compressed takes 2 x 5 bits + 3 x 2 bits + // with 8 byte alignment this is both 16 bytes. + CHECK_NOT(a.try_compress(a1)); + CHECK_NOT(a.is_compressed()); + a.add(10); + a.add(15); + // uncompressed is 5x4 bits, compressed is 3x5 bits + 5x2 bits + // with 8 byte alignment this is both 16 bytes + CHECK_NOT(a.try_compress(a1)); + CHECK_NOT(a.is_compressed()); + a.add(10); + a.add(15); + a.add(10); + a.add(15); + // uncompressed is 9x4 bits, compressed is 3x5 bits + 9x2 bits + // with 8 byte alignment this is both 16 bytes + CHECK_NOT(a.try_compress(a1)); + CHECK_NOT(a.is_compressed()); + a.add(-1); + // the addition of -1 forces the array from unsigned to signed form + // changing from 4 bits per element to 8 bits. + // (1,2,4 bit elements are unsigned, larger elements are signed) + // uncompressed is 10x8 bits, compressed is 3x5 bits + 10x2 bits + // with alignment, this is 24 bytes uncompressed and 16 bytes compressed + CHECK(a.try_compress(a1)); + CHECK_NOT(a.is_compressed()); + CHECK(a.get(0) == 10); + CHECK(a.get(1) == 5); + CHECK(a.get(2) == 5); + CHECK(a.get(3) == 10); + CHECK(a.get(4) == 15); + CHECK(a1.is_compressed()); + auto v = a1.get(0); + CHECK(v == a.get(0)); + CHECK(a1.get(1) == a.get(1)); + CHECK(a1.get(2) == a.get(2)); + CHECK(a1.get(3) == a.get(3)); + CHECK(a1.get(4) == a.get(4)); + a.destroy(); + a1.destroy(); +} +#endif + +TEST(Test_ArrayInt_get_all) +{ + std::vector vs = {3656152302, 2814021986, 4195757081, 3272933168, 3466127978, 2777289082, + 4247467684, 3825361855, 2496524560, 4052938301, 3765455798, 2527633011, + 3448934593, 3699340964, 4057735040, 3294068800}; + ArrayInteger a(Allocator::get_default()); + ArrayInteger a1(Allocator::get_default()); + a.create(); + for (const auto i : vs) + a.add(i); + CHECK(a.try_compress(a1)); + CHECK(a1.is_compressed()); + auto res = a1.get_all(0, a1.size()); + CHECK(res == vs); + a.destroy(); + a1.destroy(); +} + +TEST(Test_array_same_size_less_bits) +{ + ArrayInteger a(Allocator::get_default()); + ArrayInteger a1(Allocator::get_default()); + a.create(); + a.add(1000000); + a.add(1000000); + a.add(1000000); + CHECK(a.try_compress(a1)); + CHECK_NOT(a.is_compressed()); + CHECK(a.get_any(0) == 1000000); + CHECK(a.get_any(1) == 1000000); + CHECK(a.get_any(2) == 1000000); + CHECK(a1.is_compressed()); + CHECK(a1.get_any(0) == 1000000); + CHECK(a1.get_any(1) == 1000000); + CHECK(a1.get_any(2) == 1000000); + a.destroy(); + a1.destroy(); +} + +TEST(Test_ArrayInt_negative_nums) +{ + ArrayInteger a(Allocator::get_default()); + ArrayInteger a1(Allocator::get_default()); + a.create(); + a.add(-1000000); + a.add(0); + a.add(1000000); + CHECK_NOT(a.is_compressed()); + CHECK(a.try_compress(a1)); + a1.destroy(); + CHECK(a.get(0) == -1000000); + CHECK(a.get(1) == 0); + CHECK(a.get(2) == 1000000); + a.add(-1000000); + a.add(-1000000); + CHECK(a.try_compress(a1)); + CHECK_NOT(a.is_compressed()); + CHECK(a.get(0) == -1000000); + CHECK(a.get(1) == 0); + CHECK(a.get(2) == 1000000); + CHECK(a.get(3) == -1000000); + CHECK(a.get(4) == -1000000); + a.add(0); + a1.destroy(); + CHECK(a.try_compress(a1)); + CHECK_NOT(a.is_compressed()); + CHECK(a1.is_compressed()); + + CHECK(a1.get(0) == a.get(0)); + CHECK(a1.get(1) == a.get(1)); + CHECK(a1.get(2) == a.get(2)); + CHECK(a1.get(3) == a.get(3)); + CHECK(a1.get(4) == a.get(4)); + CHECK(a1.get(5) == a.get(5)); + + a.add(1000000); + a1.destroy(); // this decodes the array + CHECK(a.try_compress(a1)); + CHECK_NOT(a.is_compressed()); + CHECK(a1.is_compressed()); + CHECK(a1.get(0) == a.get(0)); + CHECK(a1.get(1) == a.get(1)); + CHECK(a1.get(2) == a.get(2)); + CHECK(a1.try_decompress()); + a.add(-1000000); + a1.destroy(); + CHECK(a.try_compress(a1)); + CHECK_NOT(a.is_compressed()); + CHECK(a1.is_compressed()); + CHECK(a1.get(0) == a.get(0)); + CHECK(a1.get(1) == a.get(1)); + CHECK(a1.get(2) == a.get(2)); + a.add(0); + a1.destroy(); + CHECK(a.try_compress(a1)); + CHECK_NOT(a.is_compressed()); + CHECK(a1.is_compressed()); + CHECK(a1.get(0) == a.get(0)); + CHECK(a1.get(1) == a.get(1)); + CHECK(a1.get(2) == a.get(2)); + a.add(1000000); + a1.destroy(); + CHECK(a.try_compress(a1)); + CHECK_NOT(a.is_compressed()); + CHECK(a1.is_compressed()); + CHECK(a.size() == 10); + CHECK(a.size() == a1.size()); + CHECK(a1.is_compressed()); + CHECK(a1.get(0) == a.get(0)); + CHECK(a1.get(1) == a.get(1)); + CHECK(a1.get(2) == a.get(2)); + CHECK(a1.get(3) == a.get(3)); + CHECK(a1.get(4) == a.get(4)); + CHECK(a1.get(5) == a.get(5)); + CHECK(a1.get(6) == a.get(6)); + CHECK(a1.get(7) == a.get(7)); + CHECK(a1.get(8) == a.get(8)); + a.destroy(); + a1.destroy(); +} + +TEST(Test_ArrayInt_compress_data) +{ + ArrayInteger a(Allocator::get_default()); + ArrayInteger a1(Allocator::get_default()); + + a.create(); + a.add(-4427957085475570907); + a.add(-4427957085475570907); + a.add(-4427957085475570907); + a.add(-4427957085475570907); + a.add(4); + a.add(5); + a.add(6); + a.add(7); + a.add(8); + a.add(4); + a.try_compress(a1); + bool ok = a1.is_compressed(); + CHECK(ok); + CHECK(a1.is_compressed()); + CHECK(a1.is_attached()); + CHECK(a.is_attached()); + for (size_t i = 0; i < a.size(); ++i) { + auto v0 = a1.get(i); + auto v1 = a.get(i); + CHECK(v0 == v1); + } + a.destroy(); + a1.destroy(); + + a.create(); + a.add(-4427957085475570907); + a.add(-4427957085475570907); + a.add(-4427957085475570907); + a.add(-4427957085475570907); + a.try_compress(a1); + for (size_t i = 0; i < a.size(); ++i) + CHECK(a1.get(i) == a.get(i)); + a.destroy(); + a1.destroy(); + + a.create(); + + a.add(16388); + a.add(409); + a.add(16388); + a.add(16388); + a.add(409); + a.add(16388); + CHECK(a.size() == 6); + // Current: [16388:16, 409:16, 16388:16, 16388:16, 409:16, 16388:16], space needed: 6*16 bits = 96 bits + + // header + // compress the array is a good option. + CHECK(a.try_compress(a1)); + CHECK(a1.is_compressed()); + // Compressed: [409:16, 16388:16][1:1,0:1,1:1,1:1,0:1,1:1], space needed: 2*16 bits + 6 * 1 bit = 38 bits + + // header + CHECK(a1.size() == a.size()); + CHECK(a1.get(0) == a.get(0)); + CHECK(a1.get(1) == a.get(1)); + CHECK(a1.get(2) == a.get(2)); + CHECK(a1.get(3) == a.get(3)); + CHECK(a1.get(4) == a.get(4)); + CHECK(a1.get(5) == a.get(5)); + // decompress + CHECK(a1.try_decompress()); + a.add(20); + // compress again, it should be a viable option + a1.destroy(); + CHECK(a.try_compress(a1)); + CHECK(a1.is_compressed()); + CHECK(a1.size() == 7); + CHECK(a1.get(0) == a.get(0)); + CHECK(a1.get(1) == a.get(1)); + CHECK(a1.get(2) == a.get(2)); + CHECK(a1.get(3) == a.get(3)); + CHECK(a1.get(4) == a.get(4)); + CHECK(a1.get(5) == a.get(5)); + CHECK(a1.get(6) == a.get(6)); + CHECK(a1.try_decompress()); + CHECK_NOT(a1.is_compressed()); + CHECK(a1.get(0) == a.get(0)); + CHECK(a1.get(1) == a.get(1)); + CHECK(a1.get(2) == a.get(2)); + CHECK(a1.get(3) == a.get(3)); + CHECK(a1.get(4) == a.get(4)); + CHECK(a1.get(5) == a.get(5)); + CHECK(a1.get(6) == a.get(6)); + a.destroy(); + a1.destroy(); +} + +TEST(Test_ArrayInt_compress_data_init_from_mem) +{ + ArrayInteger a(Allocator::get_default()); + ArrayInteger a1(Allocator::get_default()); + a.create(); + a.add(16388); + a.add(409); + a.add(16388); + a.add(16388); + a.add(409); + a.add(16388); + const auto sz = a.size(); + CHECK(sz == 6); + // Current: [16388:16, 409:16, 16388:16, 16388:16, 409:16, 16388:16], + // space needed: 6*16 bits = 96 bits + header + // compress the array is a good option (it should already be compressed). + CHECK(a.try_compress(a1)); + CHECK(a1.is_compressed()); + // Array should be in compressed form now + auto mem = a1.get_mem(); + ArrayInteger a2(Allocator::get_default()); + a2.init_from_mem(mem); // initialise a1 with a + // check a2 + CHECK(a2.is_compressed()); + const auto sz2 = a2.size(); + CHECK(sz2 == 6); + CHECK(a2.get(0) == 16388); + CHECK(a2.get(1) == 409); + CHECK(a2.get(2) == 16388); + CHECK(a2.get(3) == 16388); + CHECK(a2.get(4) == 409); + CHECK(a2.get(5) == 16388); + // decompress a2 and compresses again + CHECK(a2.is_compressed()); + CHECK(a2.try_decompress()); + CHECK_NOT(a2.is_compressed()); + a2.add(20); + CHECK(a2.try_compress(a1)); + CHECK(a1.is_compressed()); + CHECK(a1.size() == 7); + CHECK(a1.get(0) == 16388); + CHECK(a1.get(1) == 409); + CHECK(a1.get(2) == 16388); + CHECK(a1.get(3) == 16388); + CHECK(a1.get(4) == 409); + CHECK(a1.get(5) == 16388); + CHECK(a1.get(6) == 20); + CHECK(a1.try_decompress()); + a.destroy(); + a1.destroy(); + a2.destroy(); + CHECK_NOT(a.is_attached()); + CHECK_NOT(a1.is_attached()); + CHECK_NOT(a2.is_attached()); +} TEST(ArrayIntNull_SetNull) { @@ -244,3 +1814,114 @@ TEST(ArrayRef_Basic) a.destroy(); } + +TEST_TYPES(ArrayInt_comparison, Equal, NotEqual, Less, Greater) +{ + using Cond = TEST_TYPE; + ArrayInteger a(Allocator::get_default()); + ArrayInteger a1(Allocator::get_default()); + a.create(); + + // check first positive values < 32 bits + constexpr auto N = 300; + constexpr auto M = 3; + for (size_t i = 0; i < N; i++) + for (size_t j = 0; j < M; ++j) + a.add(i); + + auto sz = a.size(); + CHECK(sz == M * N); + + CHECK(a.try_compress(a1)); + CHECK(a1.is_compressed()); + + // Array should be in compressed form now and values should match + for (size_t i = 0; i < sz; ++i) + CHECK(a.get(i) == a1.get(i)); + + for (int i = (int)(sz)-1; i >= 0; --i) { + QueryStateFindFirst m_first1, m_first2; + CHECK(a.find(i, 0, sz, &m_first1) == a1.find(i, 0, sz, &m_first2)); + CHECK(m_first1.m_state == m_first2.m_state); + } + + IntegerColumn accu1(Allocator::get_default()); + IntegerColumn accu2(Allocator::get_default()); + accu1.create(); + accu2.create(); + for (int i = (int)(sz)-1; i >= 0; --i) { + QueryStateFindAll m1{accu1}, m2{accu2}; + CHECK(a.find(i, 0, sz, &m1) == a1.find(i, 0, sz, &m2)); + CHECK(m1.match_count() == m2.match_count()); + } + + // check negative numbers now. + a1.destroy(); + a.clear(); + + for (size_t i = 0; i < N; i++) + for (size_t j = 0; j < M; ++j) + a.add(-i); + + sz = a.size(); + CHECK(sz == M * N); + + CHECK(a.try_compress(a1)); + CHECK(a1.is_compressed()); + + // Array should be in compressed form now and values should match + for (size_t i = 0; i < sz; ++i) + CHECK(a.get(i) == a1.get(i)); + + for (int64_t i = (int64_t)(sz)-1; i >= 0; --i) { + QueryStateFindFirst m_first1, m_first2; + CHECK(a.find(-i, 0, sz, &m_first1) == a1.find(-i, 0, sz, &m_first2)); + CHECK(m_first1.m_state == m_first2.m_state); + } + + accu1.clear(); + accu2.clear(); + for (int i = (int)(sz)-1; i >= 0; --i) { + QueryStateFindAll m1{accu1}, m2{accu2}; + CHECK(a.find(-i, 0, sz, &m1) == a1.find(-i, 0, sz, &m2)); + CHECK(m1.match_count() == m2.match_count()); + } + + accu1.destroy(); + accu2.destroy(); + a.destroy(); + a1.destroy(); + +#if REALM_COMPRESS + a.create(); + std::random_device dev; + std::mt19937 rng(dev()); + const auto min_range_t = (size_t)std::numeric_limits::min(); + const auto max_range_t = (size_t)std::numeric_limits::max(); + std::uniform_int_distribution dist(min_range_t, max_range_t); + sz = 100; + for (size_t i = 0; i < sz; ++i) { + auto v = (int)dist(rng); + a.add(v); + } + a.try_compress(a1); + + for (size_t i = 0; i < sz; ++i) + CHECK(a.get(i) == a1.get(i)); + + CHECK(a1.is_compressed()); + for (size_t i = 0; i < sz; ++i) { + QueryStateFindFirst m_first1, m_first2; + CHECK(a.find(a.get(i), 0, sz, &m_first1) == a1.find(a1.get(i), 0, sz, &m_first2)); + CHECK(m_first1.m_state == m_first2.m_state); + if (m_first1.m_state != realm::not_found) + CHECK(a.get(m_first1.m_state) == a1.get(m_first2.m_state)); + } + + a.destroy(); + a1.destroy(); +#endif + + CHECK_NOT(a.is_attached()); + CHECK_NOT(a1.is_attached()); +} diff --git a/test/test_group.cpp b/test/test_group.cpp index c6d39637a4b..c3c11c16ed5 100644 --- a/test/test_group.cpp +++ b/test/test_group.cpp @@ -2315,4 +2315,198 @@ TEST(Group_UniqueColumnKeys) CHECK_NOT_EQUAL(col_foo, col_bar); } +TEST(Group_ArrayCompression_Correctness) +{ + GROUP_TEST_PATH(path); + + // Create group with one list which maps to array_integer + Group to_disk; + TableRef table = to_disk.add_table("test"); + auto col_key = table->add_column_list(type_Int, "lint"); + auto obj = table->create_object(); + auto array = obj.get_list(col_key); + array.add(16388); + array.add(409); + array.add(16388); + array.add(16388); + array.add(409); + array.add(16388); + CHECK_EQUAL(array.size(), 6); + CHECK_EQUAL(array.get_any(0).get_int(), 16388); + CHECK_EQUAL(array.get_any(1).get_int(), 409); + CHECK_EQUAL(array.get_any(2).get_int(), 16388); + CHECK_EQUAL(array.get_any(3).get_int(), 16388); + CHECK_EQUAL(array.get_any(4).get_int(), 409); + CHECK_EQUAL(array.get_any(5).get_int(), 16388); + + // Serialize to disk (compression should happen when the proper leaf array is serialized to disk) + to_disk.write(path, crypt_key()); + +#ifdef REALM_DEBUG + to_disk.verify(); +#endif + + // Load the tables + Group from_disk(path, crypt_key()); + TableRef read_table = from_disk.get_table("test"); + auto col_key1 = read_table->get_column_key("lint"); + auto obj1 = read_table->get_object(0); + auto l1 = obj1.get_list(col_key1); + CHECK(l1.size() == array.size()); + CHECK(*read_table == *table); + for (size_t i = 0; i < l1.size(); ++i) { + CHECK_EQUAL(l1.get_any(i), array.get_any(i)); + } + +#ifdef REALM_DEBUG + from_disk.verify(); +#endif +} + +TEST(Group_ArrayCompression_Correctness_Negative) +{ + GROUP_TEST_PATH(path); + + // Create group with one list which maps to array_integer + Group to_disk; + TableRef table = to_disk.add_table("test"); + auto col_key = table->add_column_list(type_Int, "lint"); + auto obj = table->create_object(); + auto array = obj.get_list(col_key); + + array.add(-1); + array.add(-1); + array.add(-1); + array.add(-1); + array.add(std::numeric_limits::max()); + array.add(std::numeric_limits::max()); + + CHECK_EQUAL(array.size(), 6); + CHECK_EQUAL(array.get_any(0).get_int(), -1); + CHECK_EQUAL(array.get_any(1).get_int(), -1); + CHECK_EQUAL(array.get_any(2).get_int(), -1); + CHECK_EQUAL(array.get_any(3).get_int(), -1); + CHECK_EQUAL(array.get_any(4).get_int(), std::numeric_limits::max()); + CHECK_EQUAL(array.get_any(5).get_int(), std::numeric_limits::max()); + + // Serialize to disk (compression should happen when the proper leaf array is serialized to disk) + to_disk.write(path, crypt_key()); + +#ifdef REALM_DEBUG + to_disk.verify(); +#endif + + // Load the tables + Group from_disk(path, crypt_key()); + TableRef read_table = from_disk.get_table("test"); + auto col_key1 = read_table->get_column_key("lint"); + auto obj1 = read_table->get_object(0); + auto l1 = obj1.get_list(col_key1); + CHECK(l1.size() == array.size()); + CHECK(*read_table == *table); + for (size_t i = 0; i < l1.size(); ++i) { + CHECK_EQUAL(l1.get_any(i), array.get_any(i)); + } + +#ifdef REALM_DEBUG + from_disk.verify(); +#endif +} + +TEST(Group_ArrayCompression_Correctness_Funny_Values) +{ + GROUP_TEST_PATH(path); + + // Create group with one list which maps to array_integer + Group to_disk; + TableRef table = to_disk.add_table("test"); + auto col_key = table->add_column_list(type_Int, "lint"); + auto obj = table->create_object(); + auto array = obj.get_list(col_key); + + std::vector vs = {3656152302, 2814021986, 4195757081, 3272933168, 3466127978, 2777289082, + 4247467684, 3825361855, 2496524560, 4052938301, 3765455798, 2527633011, + 3448934593, 3699340964, 4057735040, 3294068800}; + + size_t ndx = 0; + for (const auto v : vs) { + array.add(v); + CHECK_EQUAL(v, array.get(ndx++)); + } + CHECK_EQUAL(array.size(), vs.size()); + + // Serialize to disk (compression should happen when the proper leaf array is serialized to disk) + to_disk.write(path, crypt_key()); + +#ifdef REALM_DEBUG + to_disk.verify(); +#endif + + // Load the tables + Group from_disk(path, crypt_key()); + TableRef read_table = from_disk.get_table("test"); + auto col_key1 = read_table->get_column_key("lint"); + auto obj1 = read_table->get_object(0); + auto l1 = obj1.get_list(col_key1); + CHECK(l1.size() == array.size()); + CHECK(*read_table == *table); + for (size_t i = 0; i < l1.size(); ++i) { + CHECK_EQUAL(l1.get_any(i), array.get_any(i)); + } + +#ifdef REALM_DEBUG + from_disk.verify(); +#endif +} + + +TEST(Group_ArrayCompression_Correctness_Random_Input) +{ + GROUP_TEST_PATH(path); + + // Create group with one list which maps to array_integer + Group to_disk; + TableRef table = to_disk.add_table("test"); + auto col_key = table->add_column_list(type_Int, "lint"); + auto obj = table->create_object(); + auto array = obj.get_list(col_key); + + std::random_device dev; + std::mt19937 rng(dev()); + constexpr auto min = std::numeric_limits::min(); + constexpr auto max = std::numeric_limits::max(); + std::uniform_int_distribution dist6(static_cast(min), + static_cast(max)); + for (size_t i = 0; i < 1000; ++i) { + const auto v = dist6(rng); + array.add(v); + const auto stored_v = array.get_any(i).get_int(); + CHECK_EQUAL(stored_v, v); + } + + // Serialize to disk (compression should happen when the proper leaf array is serialized to disk) + to_disk.write(path, crypt_key()); + +#ifdef REALM_DEBUG + to_disk.verify(); +#endif + + // Load the tables + Group from_disk(path, crypt_key()); + TableRef read_table = from_disk.get_table("test"); + auto col_key1 = read_table->get_column_key("lint"); + auto obj1 = read_table->get_object(0); + auto l1 = obj1.get_list(col_key1); + CHECK(l1.size() == array.size()); + CHECK(*read_table == *table); + for (size_t i = 0; i < l1.size(); ++i) { + CHECK_EQUAL(l1.get_any(i), array.get_any(i)); + } + +#ifdef REALM_DEBUG + from_disk.verify(); +#endif +} + + #endif // TEST_GROUP diff --git a/test/test_links.cpp b/test/test_links.cpp index be08d2c7392..7561364089b 100644 --- a/test/test_links.cpp +++ b/test/test_links.cpp @@ -1167,11 +1167,13 @@ TEST(Links_FormerMemLeakCase) auto col = origin->add_column(*target, "link"); origin->create_object().set(col, k); origin->create_object().set(col, k); + wt.get_group().verify(); wt.commit(); } { WriteTransaction wt(sg_w); TableRef target = wt.get_table("target"); + wt.get_group().verify(); target->begin()->remove(); wt.get_group().verify(); wt.commit(); diff --git a/test/test_list.cpp b/test/test_list.cpp index 1aef9760882..d8e3f1fc1de 100644 --- a/test/test_list.cpp +++ b/test/test_list.cpp @@ -633,6 +633,41 @@ TEST(List_AggOps) test_lists_numeric_agg(test_context, sg, type_Decimal, Decimal128(realm::null()), true); } +TEST(Test_Write_List_Nested_In_Mixed) +{ + SHARED_GROUP_TEST_PATH(path); + std::string message; + DBOptions options; + options.logger = test_context.logger; + DBRef db = DB::create(make_in_realm_history(), path, options); + auto tr = db->start_write(); + auto table = tr->add_table("table"); + auto col_any = table->add_column(type_Mixed, "something"); + + Obj obj = table->create_object(); + obj.set_any(col_any, Mixed{20}); + tr->verify(); + tr->commit_and_continue_writing(); // commit simple mixed + tr->verify(); + + obj.set_collection(col_any, CollectionType::List); + auto list = obj.get_list_ptr(col_any); + list->add(Mixed{10}); + list->add(Mixed{11}); + tr->verify(); + tr->commit_and_continue_writing(); // commit nested list in mixed + tr->verify(); + + // spicy it up a little bit... + list->insert_collection(2, CollectionType::List); + list->insert_collection(3, CollectionType::List); + list->get_list(2)->add(Mixed{20}); + list->get_list(3)->add(Mixed{21}); + tr->commit_and_continue_writing(); + tr->verify(); + tr->close(); +} + TEST(List_Nested_InMixed) { SHARED_GROUP_TEST_PATH(path); @@ -824,6 +859,10 @@ TEST(List_Nested_InMixed) "[{\"Seven\":7, \"Six\":6}, \"Hello\", {\"Points\": [1.25, 4.5, 6.75], \"Hello\": \"World\"}]"); CHECK_EQUAL(obj.get_list_ptr(col_any)->size(), 3); // tr->to_json(std::cout); + tr->commit(); + db->compact(); + tr = db->start_write(); + tr->verify(); } diff --git a/test/test_query.cpp b/test/test_query.cpp index 77229367012..1c380e409cb 100644 --- a/test/test_query.cpp +++ b/test/test_query.cpp @@ -5795,4 +5795,38 @@ TEST(Query_NestedLinkCount) CHECK_EQUAL(q.count(), 3); } +TEST_TYPES(Query_IntCompressed, Equal, NotEqual, Less, LessEqual, Greater, GreaterEqual) +{ + TEST_TYPE c; + SHARED_GROUP_TEST_PATH(path); + int ints[] = {-120, -111, -70, -61, -55, -45, -22, -15, -3, 2, 7, 18, 25, 33, 55, 56, 66, 78, 104, 125}; + std::vector values; + for (int j = 1; j < 21; j++) { + for (int i = 0; i < j; i++) { + values.push_back(ints[i]); + } + } + + auto db = DB::create(path); + auto wt = db->start_write(); + auto t = wt->add_table("table"); + auto col = t->add_column(type_Int, "id"); + for (auto val : values) { + t->create_object().set(col, val); + } + wt->commit_and_continue_as_read(); + + for (int val : {-1000, -125, 2, 3, 6, 126, 1000}) { + size_t num_matches = 0; + for (auto i : values) { + if (c(i, val)) + num_matches++; + } + + char query_str[20]; + snprintf(query_str, 20, "id %s %d", c.description().c_str(), val); + CHECK_EQUAL(t->query(query_str).count(), num_matches); + } +} + #endif // TEST_QUERY diff --git a/test/test_shared.cpp b/test/test_shared.cpp index 73ba955251b..df9ae2f778d 100644 --- a/test/test_shared.cpp +++ b/test/test_shared.cpp @@ -95,34 +95,32 @@ using unit_test::TestContext; // `experiments/testcase.cpp` and then run `sh build.sh // check-testcase` (or one of its friends) from the command line. -#if 0 + // Sorting benchmark -ONLY(Query_QuickSort2) +TEST(Query_QuickSort2) { Random random(random_int()); // Seed from slow global generator // Triggers QuickSort because range > len Table ttt; - auto ints = ttt.add_column(type_Int, "1"); + // auto ints = ttt.add_column(type_Int, "1"); auto strings = ttt.add_column(type_String, "2"); for (size_t t = 0; t < 10000; t++) { Obj o = ttt.create_object(); - // o.set(ints, random.draw_int_mod(1100)); + // o.set(ints, random.draw_int_mod(1100)); o.set(strings, "a"); } Query q = ttt.where(); - std::cerr << "GO"; - for (size_t t = 0; t < 1000; t++) { TableView tv = q.find_all(); tv.sort(strings); - // tv.ints(strings); + // tv.ints(strings); } } -#endif + #if REALM_WINDOWS namespace { diff --git a/test/test_sync.cpp b/test/test_sync.cpp index 933e3516f50..57dd1d11c67 100644 --- a/test/test_sync.cpp +++ b/test/test_sync.cpp @@ -6223,6 +6223,70 @@ TEST(Sync_DeleteCollectionInCollection) } } +TEST(Sync_AdditionalProperties) +{ + DBOptions options; + options.allow_flexible_schema = true; + SHARED_GROUP_TEST_PATH(db_1_path); + SHARED_GROUP_TEST_PATH(db_2_path); + auto db_1 = DB::create(make_client_replication(), db_1_path, options); + auto db_2 = DB::create(make_client_replication(), db_2_path, options); + + TEST_DIR(dir); + fixtures::ClientServerFixture fixture{dir, test_context}; + fixture.start(); + + Session session_1 = fixture.make_session(db_1, "/test"); + Session session_2 = fixture.make_session(db_2, "/test"); + + write_transaction(db_1, [&](WriteTransaction& tr) { + auto& g = tr.get_group(); + auto table = g.add_table_with_primary_key("class_Table", type_Int, "id"); + auto col_any = table->add_column(type_Mixed, "any"); + auto foo = table->create_object_with_primary_key(123); + foo.set_any(col_any, "FooBar"); + foo.set("age", 10); + foo.set_collection("scores", CollectionType::List); + auto list = foo.get_list_ptr("scores"); + list->add(4.6); + }); + + session_1.wait_for_upload_complete_or_client_stopped(); + session_2.wait_for_download_complete_or_client_stopped(); + + write_transaction(db_2, [&](WriteTransaction& tr) { + auto table = tr.get_table("class_Table"); + CHECK_EQUAL(table->size(), 1); + + auto obj = table->get_object_with_primary_key(123); + auto col_keys = table->get_column_keys(); + CHECK_EQUAL(col_keys.size(), 2); + CHECK_EQUAL(table->get_column_name(col_keys[0]), "id"); + CHECK_EQUAL(table->get_column_name(col_keys[1]), "any"); + auto props = obj.get_additional_properties(); + CHECK_EQUAL(props.size(), 2); + CHECK_EQUAL(obj.get("age"), 10); + CHECK_EQUAL(obj.get_any("any"), Mixed("FooBar")); + auto list = obj.get_list_ptr("scores"); + CHECK_EQUAL(list->get(0), Mixed(4.6)); + CHECK_THROW_ANY(obj.get_any("some")); + CHECK_THROW_ANY(obj.erase_additional_prop("any")); + obj.erase_additional_prop("age"); + }); + + session_2.wait_for_upload_complete_or_client_stopped(); + session_1.wait_for_download_complete_or_client_stopped(); + + write_transaction(db_1, [&](WriteTransaction& tr) { + auto table = tr.get_table("class_Table"); + CHECK_EQUAL(table->size(), 1); + + auto obj = table->get_object_with_primary_key(123); + auto props = obj.get_additional_properties(); + CHECK_EQUAL(props.size(), 1); + }); +} + TEST(Sync_NestedCollectionClear) { TEST_CLIENT_DB(db_1); diff --git a/test/test_table.cpp b/test/test_table.cpp index 80df42e1824..52e06fb2659 100644 --- a/test/test_table.cpp +++ b/test/test_table.cpp @@ -46,7 +46,7 @@ using namespace std::chrono; #include "test_types_helper.hpp" // #include -// #define PERFORMACE_TESTING +// #define PERFORMANCE_TESTING using namespace realm; using namespace realm::util; @@ -2954,9 +2954,122 @@ NONCONCURRENT_TEST(Table_QuickSort2) std::cout << " time: " << duration_cast(t2 - t1).count() / nb_reps << " ns/rep" << std::endl; } +NONCONCURRENT_TEST(Table_object_timestamp) +{ +#if !defined(REALM_DEBUG) && defined(PERFORMANCE_TESTING) + int nb_rows = 10'000'000; + int num_runs = 100; +#else + int nb_rows = 100'000; + int num_runs = 1; +#endif + SHARED_GROUP_TEST_PATH(path); + std::unique_ptr hist(make_in_realm_history()); + DBRef sg = DB::create(*hist, path, DBOptions(crypt_key())); + ColKey c0; + + CALLGRIND_START_INSTRUMENTATION; + + std::cout << nb_rows << " rows - timestamps" << std::endl; + + { + WriteTransaction wt(sg); + auto table = wt.add_table("test"); + + c0 = table->add_column(type_Timestamp, "ts"); + + + auto t1 = steady_clock::now(); + + for (int i = 0; i < nb_rows; i++) { + Timestamp t(i, i); + table->create_object(ObjKey(i)).set_all(t); + } + + auto t2 = steady_clock::now(); + std::cout << " insertion time: " << duration_cast(t2 - t1).count() / nb_rows << " ns/key" + << std::endl; + + CHECK_EQUAL(table->size(), nb_rows); + wt.commit(); + } + { + ReadTransaction rt(sg); + auto table = rt.get_table("test"); + + auto t1 = steady_clock::now(); + Timestamp t(nb_rows / 2, nb_rows / 2); + for (int j = 0; j < num_runs; ++j) { + auto result = table->where().equal(c0, t).find_all(); + } + + auto t2 = steady_clock::now(); + + std::cout << " find all : " << duration_cast(t2 - t1).count() / num_runs << " ms" + << std::endl; + } +} + +NONCONCURRENT_TEST(Table_object_search) +{ +#if !defined(REALM_DEBUG) && defined(PERFORMANCE_TESTING) + int nb_rows = 10'000'000; + int num_runs = 100; +#else + int nb_rows = 100'000; + int num_runs = 1; +#endif + SHARED_GROUP_TEST_PATH(path); + std::unique_ptr hist(make_in_realm_history()); + DBRef sg = DB::create(*hist, path, DBOptions(crypt_key())); + ColKey c0; + ColKey c1; + + CALLGRIND_START_INSTRUMENTATION; + + std::cout << nb_rows << " rows - sequential" << std::endl; + + { + WriteTransaction wt(sg); + auto table = wt.add_table("test"); + + c0 = table->add_column(type_Int, "int1"); + c1 = table->add_column(type_Int, "int2", true); + + + auto t1 = steady_clock::now(); + + for (int i = 0; i < nb_rows; i++) { + table->create_object(ObjKey(i)).set_all(i << 1, i << 2); + } + + auto t2 = steady_clock::now(); + std::cout << " insertion time: " << duration_cast(t2 - t1).count() / nb_rows << " ns/key" + << std::endl; + + CHECK_EQUAL(table->size(), nb_rows); + wt.commit(); + } + { + ReadTransaction rt(sg); + auto table = rt.get_table("test"); + + auto t1 = steady_clock::now(); + + for (int j = 0; j < num_runs; ++j) { + auto result = table->find_all_int(c0, nb_rows / 2); + } + + auto t2 = steady_clock::now(); + + std::cout << " find all : " << duration_cast(t2 - t1).count() / num_runs << " ms" + << std::endl; + } +} + NONCONCURRENT_TEST(Table_object_sequential) { -#ifdef PERFORMACE_TESTING +#if !defined(REALM_DEBUG) && defined(PERFORMANCE_TESTING) int nb_rows = 10'000'000; int num_runs = 1; #else @@ -3106,7 +3219,7 @@ NONCONCURRENT_TEST(Table_object_sequential) NONCONCURRENT_TEST(Table_object_seq_rnd) { -#ifdef PERFORMACE_TESTING +#if !defined(REALM_DEBUG) && defined(PERFORMANCE_TESTING) size_t rows = 1'000'000; int runs = 100; // runs for building scenario #else @@ -3149,7 +3262,7 @@ NONCONCURRENT_TEST(Table_object_seq_rnd) } // scenario established! int nb_rows = int(key_values.size()); -#ifdef PERFORMACE_TESTING +#if !defined(REALM_DEBUG) && defined(PERFORMANCE_TESTING) int num_runs = 10; // runs for timing access #else int num_runs = 1; // runs for timing access @@ -3221,7 +3334,7 @@ NONCONCURRENT_TEST(Table_object_seq_rnd) NONCONCURRENT_TEST(Table_object_random) { -#ifdef PERFORMACE_TESTING +#if !defined(REALM_DEBUG) && defined(PERFORMANCE_TESTING) int nb_rows = 1'000'000; int num_runs = 10; #else diff --git a/test/test_unresolved_links.cpp b/test/test_unresolved_links.cpp index adaf6981130..60f50ee3488 100644 --- a/test/test_unresolved_links.cpp +++ b/test/test_unresolved_links.cpp @@ -837,35 +837,6 @@ TEST(Links_ManyObjects) tr->commit(); } -TEST(Unresolved_PerformanceLinks) -{ - constexpr int nb_objects = 1000; - using namespace std::chrono; - - SHARED_GROUP_TEST_PATH(path); - auto hist = make_in_realm_history(); - DBRef db = DB::create(*hist, path); - - auto tr = db->start_write(); - auto table = tr->add_table_with_primary_key("table", type_Int, "id"); - auto origin = tr->add_table("origin"); - auto col = origin->add_column(*table, "link"); - auto key = table->get_objkey_from_primary_key(1); - for (int i = 0; i < nb_objects; i++) { - origin->create_object().set(col, key); - } - tr->commit_and_continue_as_read(); - tr->promote_to_write(); - auto t1 = steady_clock::now(); - table->create_object_with_primary_key(1); - auto t2 = steady_clock::now(); - tr->commit_and_continue_as_read(); - CHECK(t2 > t1); - // std::cout << "Time: " << duration_cast(t2 - t1).count() << " us" << std::endl; - tr->promote_to_write(); - tr->verify(); -} - TEST(Unresolved_PerformanceLinkList) { constexpr int nb_objects = 1000; @@ -889,6 +860,7 @@ TEST(Unresolved_PerformanceLinkList) ll.add(key3); } tr->commit_and_continue_as_read(); + // compresses tr->promote_to_write(); auto t1 = steady_clock::now(); table->create_object_with_primary_key(1); @@ -897,7 +869,6 @@ TEST(Unresolved_PerformanceLinkList) auto t2 = steady_clock::now(); tr->commit_and_continue_as_read(); CHECK(t2 > t1); - // std::cout << "Time: " << duration_cast(t2 - t1).count() << " us" << std::endl; tr->promote_to_write(); tr->verify(); } diff --git a/test/test_upgrade_database.cpp b/test/test_upgrade_database.cpp index ae95d1a02da..dad101ce10d 100644 --- a/test/test_upgrade_database.cpp +++ b/test/test_upgrade_database.cpp @@ -36,6 +36,7 @@ #include #include "test.hpp" #include "test_table_helper.hpp" +#include "util/compare_groups.hpp" #include @@ -273,8 +274,8 @@ TEST(Upgrade_Database_11) TableRef foo = g.add_table_with_primary_key("foo", type_Int, "id", false); TableRef bar = g.add_table_with_primary_key("bar", type_String, "name", false); TableRef o = g.add_table("origin"); - auto col1 = o->add_column_link(type_Link, "link1", *foo); - auto col2 = o->add_column_link(type_Link, "link2", *bar); + auto col1 = o->add_column(*foo, "link1"); + auto col2 = o->add_column(*bar, "link2"); for (auto id : ids) { auto obj = foo->create_object_with_primary_key(id); @@ -884,4 +885,107 @@ TEST_IF(Upgrade_Database_23, REALM_MAX_BPNODE_SIZE == 4 || REALM_MAX_BPNODE_SIZE g.write(path); #endif // TEST_READ_UPGRADE_MODE } + +TEST_IF(Upgrade_Database_24, REALM_MAX_BPNODE_SIZE == 4 || REALM_MAX_BPNODE_SIZE == 1000) +{ + std::string path = test_util::get_test_resource_path() + "test_upgrade_database_" + + util::to_string(REALM_MAX_BPNODE_SIZE) + "_24.realm"; + +#if TEST_READ_UPGRADE_MODE + CHECK_OR_RETURN(File::exists(path)); + + SHARED_GROUP_TEST_PATH(temp_copy); + + + // Make a copy of the database so that we keep the original file intact and unmodified + File::copy(path, temp_copy); + auto hist = make_in_realm_history(); + DBOptions options; + options.logger = test_context.logger; + auto sg = DB::create(*hist, temp_copy, options); + auto wt = sg->start_write(); + // rt->to_json(std::cout); + wt->verify(); + + SHARED_GROUP_TEST_PATH(path2); + wt->write(path2); + auto db2 = DB::create(path2); + auto wt2 = db2->start_write(); + CHECK(test_util::compare_groups(*wt, *wt2)); +#else + // NOTE: This code must be executed from an old file-format-version 24 + // core in order to create a file-format-version 25 test file! + + const size_t cnt = 10 * REALM_MAX_BPNODE_SIZE; + + std::vector string_values{ + "white", "yellow", "red", "orange", "green", "blue", "grey", "violet", "purple", "black", + }; + + StringData long_string(R"(1. Jeg ved en lærkerede, +jeg siger ikke mer; +den findes på en hede, +et sted som ingen ser. + +2. I reden er der unger, +og ungerne har dun. +De pipper de har tunger, +og reden er så lun. + +3. Og de to gamle lærker, +de flyver tæt omkring. +Jeg tænker nok de mærker, +jeg gør dem ingenting. + +4. Jeg lurer bag en slåen. +Der står jeg ganske nær. +Jeg rækker mig på tåen +og holder på mit vejr. + +5. For ræven han vil bide +og drengen samle bær. +men ingen skal få vide, +hvor lærkereden er. +)"); + StringData dict_value( + R"({"Seven":7, "Six":6, "Points": [1.25, 4.5, 6.75], "Attributes": {"Height": 202, "Weight": 92}})"); + + Timestamp now(std::chrono::system_clock::now()); + auto now_seconds = now.get_seconds(); + + Group g; + + TableRef t = g.add_table_with_primary_key("table", type_ObjectId, "_id", false); + auto col_int = t->add_column(type_Int, "int"); + auto col_optint = t->add_column(type_Int, "optint", true); + /* auto col_bool = */ t->add_column(type_Bool, "bool"); + auto col_string = t->add_column(type_String, "string"); + /* auto col_binary = */ t->add_column(type_Binary, "binary"); + auto col_mixed = t->add_column(type_Mixed, "any"); + auto col_date = t->add_column(type_Timestamp, "date"); + /* auto col_float = */ t->add_column(type_Float, "float"); + /* auto col_double = */ t->add_column(type_Double, "double"); + /* auto col_decimal = */ t->add_column(type_Decimal, "decimal"); + /* auto col_uuid = */ t->add_column(type_UUID, "uuid"); + + TableRef target = g.add_table_with_primary_key("target", type_Int, "_id", false); + auto col_link = t->add_column(*target, "link"); + + auto target_key = target->create_object_with_primary_key(1).get_key(); + for (size_t i = 0; i < cnt; i++) { + + auto o = t->create_object_with_primary_key(ObjectId::gen()); + o.set(col_int, uint64_t(0xffff) + i); + o.set(col_optint, uint64_t(0xffff) + (i % 10)); + o.set(col_string, string_values[i % string_values.size()]); + o.set(col_date, Timestamp(now_seconds + i, 0)); + o.set(col_link, target_key); + } + auto obj = t->create_object_with_primary_key(ObjectId::gen()); + obj.set_json(col_mixed, dict_value); + obj.set(col_string, long_string); + g.write(path); +#endif // TEST_READ_UPGRADE_MODE +} + #endif // TEST_GROUP diff --git a/test/test_upgrade_database_1000_24.realm b/test/test_upgrade_database_1000_24.realm new file mode 100644 index 00000000000..58e86453c3a Binary files /dev/null and b/test/test_upgrade_database_1000_24.realm differ diff --git a/test/test_upgrade_database_4_24.realm b/test/test_upgrade_database_4_24.realm new file mode 100644 index 00000000000..7d8280b0f55 Binary files /dev/null and b/test/test_upgrade_database_4_24.realm differ