diff --git a/include/geode/basic/database.h b/include/geode/basic/database.h new file mode 100644 index 000000000..3c920f555 --- /dev/null +++ b/include/geode/basic/database.h @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2019 - 2022 Geode-solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ + +#pragma once + +#include +#include +#include + +#include +#include + +namespace geode +{ + class Identifier; + struct uuid; +} // namespace geode + +namespace geode +{ + /*! + * Stores any classes inherited from Identifier. It owns every data + * registered. Data is also stored on disk and offload from the memory after + * some unused time to save memory and performances. + */ + class opengeode_basic_api Database + { + OPENGEODE_DISABLE_COPY( Database ); + struct Storage; + + public: + using serializer_function = std::function< void( PContext& ) >; + + /*! + * Class holding a const reference of data. + * @warning Do not destroy this Data class if the const reference + * obtained using its get() method is still in used. + */ + class opengeode_basic_api Data + { + public: + Data( std::shared_ptr< Storage > storage ); + ~Data(); + Data( Data&& other ); + + template < typename DataType > + const DataType& get() + { + const auto* typed_data = + dynamic_cast< const DataType* >( &data() ); + OPENGEODE_EXCEPTION( + typed_data, "[Data::get] Cannot cast data into DataType" ); + return *typed_data; + } + + private: + const Identifier& data() const; + + private: + IMPLEMENTATION_MEMBER( impl_ ); + }; + + public: + Database( absl::string_view directory ); + ~Database(); + + index_t nb_data() const; + + template < typename DataType > + uuid register_new_data( DataType&& data ) + { + static_assert( std::is_base_of< Identifier, DataType >::value, + "[Database::register_data] Data is not a subclass of " + "Identifier" ); + uuid new_id; + register_unique_data( + new_id, absl::make_unique< DataType >( std::move( data ) ) ); + return new_id; + } + + template < typename DataType > + uuid register_new_data( std::unique_ptr< DataType >&& data ) + { + static_assert( std::is_base_of< Identifier, DataType >::value, + "[Database::register_data] Data is not a subclass of " + "Identifier" ); + uuid new_id; + register_unique_data( new_id, std::move( data ) ); + return new_id; + } + + template < typename DataType > + void update_data( const uuid& id, DataType&& data ) + { + static_assert( std::is_base_of< Identifier, DataType >::value, + "[Database::register_data] Data is not a subclass of " + "Identifier" ); + return register_unique_data( + id, absl::make_unique< DataType >( std::move( data ) ) ); + } + + template < typename DataType > + void update_data( const uuid& id, std::unique_ptr< DataType >&& data ) + { + static_assert( std::is_base_of< Identifier, DataType >::value, + "[Database::register_data] Data is not a subclass of " + "Identifier" ); + return register_unique_data( id, std::move( data ) ); + } + + void delete_data( const uuid& id ); + + /*! + * Retrieve a read only reference to the data corresponding to the given + * uuid. + */ + Data get_data( const uuid& id ) const; + + template < typename DataType > + std::unique_ptr< DataType > take_data( const uuid& id ) + { + get_data( id ).get< DataType >(); + auto* data = + dynamic_cast< DataType* >( steal_data( id ).release() ); + return std::unique_ptr< DataType >{ data }; + } + + /*! + * Use this method to register custom serializer functions to allow + * saving any custom Object on disk + */ + void register_serializer_functions( + serializer_function serializer, serializer_function deserializer ); + + private: + void register_unique_data( + const uuid& id, std::unique_ptr< Identifier >&& data ); + + std::unique_ptr< Identifier > steal_data( const uuid& id ); + + private: + IMPLEMENTATION_MEMBER( impl_ ); + }; +} // namespace geode \ No newline at end of file diff --git a/include/geode/basic/identifier.h b/include/geode/basic/identifier.h index 14d5ac2c8..fc00689a8 100644 --- a/include/geode/basic/identifier.h +++ b/include/geode/basic/identifier.h @@ -44,7 +44,7 @@ namespace geode static constexpr auto DEFAULT_NAME = "default_name"; Identifier( Identifier&& ); - ~Identifier(); + virtual ~Identifier(); const uuid& id() const; diff --git a/src/geode/basic/CMakeLists.txt b/src/geode/basic/CMakeLists.txt index 071bc9238..1d7650376 100644 --- a/src/geode/basic/CMakeLists.txt +++ b/src/geode/basic/CMakeLists.txt @@ -29,6 +29,7 @@ add_geode_library( "common.cpp" "console_logger_client.cpp" "console_progress_logger_client.cpp" + "database.cpp" "filename.cpp" "identifier.cpp" "identifier_builder.cpp" @@ -53,6 +54,7 @@ add_geode_library( "common.h" "console_logger_client.h" "console_progress_logger_client.h" + "database.h" "factory.h" "filename.h" "greyscale_color.h" diff --git a/src/geode/basic/database.cpp b/src/geode/basic/database.cpp new file mode 100644 index 000000000..0788b7a26 --- /dev/null +++ b/src/geode/basic/database.cpp @@ -0,0 +1,364 @@ +/* + * Copyright (c) 2019 - 2022 Geode-solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ +#include + +#include +#include +#include +#include + +#include + +#include + +#include + +#include +#include +#include + +namespace +{ + constexpr auto DATA_EXPIRATION = std::chrono::minutes( 3 ); +} // namespace + +namespace geode +{ + class Database::Storage + { + public: + Storage( std::unique_ptr< geode::Identifier >&& data ) + : data_{ std::move( data ) } + { + } + + ~Storage() + { + terminate_storage(); + while( !queue_.empty() ) + { + queue_.front().wait(); + queue_.pop(); + } + } + + bool expired() const + { + return !data_; + } + + bool unused() const + { + return counter_ == 0; + } + + void new_data_reference() + { + const std::lock_guard< std::mutex > locking{ lock_ }; + counter_++; + last_++; + } + + void delete_data_reference() + { + const std::lock_guard< std::mutex > locking{ lock_ }; + OPENGEODE_ASSERT( + counter_ > 0, "[Database::Storage] Cannot decrement" ); + counter_--; + if( unused() ) + { + clean_queue(); + wait_for_memory_release(); + } + } + + void set_data( std::unique_ptr< geode::Identifier >&& data ) + { + const std::lock_guard< std::mutex > locking{ lock_ }; + counter_ = 0; + data_ = std::move( data ); + } + + const std::unique_ptr< geode::Identifier >& data() const + { + return data_; + } + + geode::Identifier* steal_data() + { + return data_.release(); + } + + private: + void terminate_storage() + { + const std::lock_guard< std::mutex > locking{ lock_ }; + terminate_ = true; + condition_.notify_all(); + } + + void clean_queue() + { + while( !queue_.empty() ) + { + if( !queue_.front().ready() ) + { + return; + } + queue_.pop(); + } + } + + void wait_for_memory_release() + { + const auto last = last_; + queue_.emplace( async::spawn( [this, last] { + std::unique_lock< std::mutex > locking{ lock_ }; + if( !condition_.wait_for( + locking, DATA_EXPIRATION, [this, last] { + return terminate_; + } ) ) + { + if( last == last_ ) + { + data_.reset(); + } + } + } ) ); + } + + private: + std::unique_ptr< Identifier > data_; + bool terminate_{ false }; + index_t counter_{ 0 }; + std::mutex lock_; + std::condition_variable condition_; + index_t last_{ 0 }; + std::queue< async::task< void > > queue_; + }; + + class Database::Impl + { + public: + Impl( absl::string_view directory ) + : directory_{ to_string( directory ) } + { + if( !ghc::filesystem::exists( directory_ ) ) + { + ghc::filesystem::create_directory( directory_ ); + } + } + + index_t nb_data() const + { + return storage_.size(); + } + + void register_unique_data( + const uuid& id, std::unique_ptr< Identifier >&& data ) + { + save_data( id, data ); + register_data( id, std::move( data ) ); + } + + std::shared_ptr< Storage > data( const uuid& id ) const + { + auto& storage = storage_.at( id ); + if( !storage || storage->expired() ) + { + storage = std::make_shared< Storage >( load_data( id ) ); + } + return storage; + } + + std::unique_ptr< Identifier > steal_data( const uuid& id ) + { + auto& storage = storage_.at( id ); + if( storage && storage->unused() && !storage->expired() ) + { + auto* data = storage->steal_data(); + storage.reset(); + return std::unique_ptr< Identifier >{ data }; + } + return load_data( id ); + } + + void delete_data( const uuid& id ) + { + storage_.erase( id ); + } + + void register_serializer_functions( + serializer_function serializer, serializer_function deserializer ) + { + serializers_.push_back( serializer ); + deserializers_.push_back( deserializer ); + } + + private: + const std::unique_ptr< Identifier >& register_data( + const uuid& id, std::unique_ptr< Identifier >&& data ) + { + const auto it = storage_.find( id ); + if( it != storage_.end() ) + { + if( it->second->unused() ) + { + it->second->set_data( std::move( data ) ); + return it->second->data(); + } + delete_data( id ); + return do_register_data( id, std::move( data ) ); + } + return do_register_data( id, std::move( data ) ); + } + + const std::unique_ptr< Identifier >& do_register_data( + const uuid& id, std::unique_ptr< Identifier >&& data ) + { + auto new_storage = std::make_shared< Storage >( std::move( data ) ); + return storage_.emplace( id, std::move( new_storage ) ) + .first->second->data(); + } + + void save_data( + const uuid& id, const std::unique_ptr< Identifier >& data ) const + { + const auto filename = absl::StrCat( directory_, "/", id.string() ); + std::ofstream file{ filename, std::ofstream::binary }; + TContext context; + for( const auto& serializer : serializers_ ) + { + serializer( std::get< 0 >( context ) ); + } + Serializer archive{ context, file }; + archive.ext( data, bitsery::ext::StdSmartPtr{} ); + archive.adapter().flush(); + OPENGEODE_EXCEPTION( std::get< 1 >( context ).isValid(), + "[Database::save_data] Error while writing file: ", filename ); + } + + std::unique_ptr< Identifier > load_data( const uuid& id ) const + { + const auto filename = absl::StrCat( directory_, "/", id.string() ); + std::ifstream file{ filename, std::ifstream::binary }; + OPENGEODE_EXCEPTION( + file, "[Database::load_data] Failed to open file: ", filename ); + TContext context{}; + for( const auto& deserializer : deserializers_ ) + { + deserializer( std::get< 0 >( context ) ); + } + Deserializer archive{ context, file }; + std::unique_ptr< Identifier > data; + archive.ext( data, bitsery::ext::StdSmartPtr{} ); + const auto& adapter = archive.adapter(); + OPENGEODE_EXCEPTION( + adapter.error() == bitsery::ReaderError::NoError + && adapter.isCompletedSuccessfully() + && std::get< 1 >( context ).isValid(), + "[Database::load_data] Error while reading file: ", filename ); + return data; + } + + private: + std::string directory_; + mutable absl::flat_hash_map< uuid, std::shared_ptr< Storage > > + storage_; + std::vector< serializer_function > serializers_; + std::vector< serializer_function > deserializers_; + }; + + class Database::Data::Impl + { + public: + Impl( std::shared_ptr< Storage > storage ) + : storage_{ std::move( storage ) } + { + storage_->new_data_reference(); + } + + ~Impl() + { + storage_->delete_data_reference(); + } + + const Identifier& data() const + { + return *storage_->data(); + } + + private: + std::shared_ptr< Storage > storage_; + }; + + Database::Data::Data( std::shared_ptr< Storage > storage ) + : impl_{ std::move( storage ) } + { + } + + Database::Data::~Data() {} // NOLINT + + Database::Data::Data( Data&& other ) : impl_{ std::move( other.impl_ ) } {} + + const Identifier& Database::Data::data() const + { + return impl_->data(); + } + + Database::Database( absl::string_view directory ) : impl_{ directory } {} + + Database::~Database() {} // NOLINT + + index_t Database::nb_data() const + { + return impl_->nb_data(); + } + + void Database::register_unique_data( + const uuid& id, std::unique_ptr< Identifier >&& data ) + { + impl_->register_unique_data( id, std::move( data ) ); + } + + Database::Data Database::get_data( const uuid& id ) const + { + return { impl_->data( id ) }; + } + + std::unique_ptr< Identifier > Database::steal_data( const uuid& id ) + { + return impl_->steal_data( id ); + } + + void Database::delete_data( const uuid& id ) + { + impl_->delete_data( id ); + } + + void Database::register_serializer_functions( + serializer_function serializer, serializer_function deserializer ) + { + impl_->register_serializer_functions( serializer, deserializer ); + } +} // namespace geode \ No newline at end of file diff --git a/src/geode/mesh/core/bitsery_archive.cpp b/src/geode/mesh/core/bitsery_archive.cpp index f38830e34..7fb438ed3 100644 --- a/src/geode/mesh/core/bitsery_archive.cpp +++ b/src/geode/mesh/core/bitsery_archive.cpp @@ -48,6 +48,12 @@ namespace bitsery { namespace ext { + template <> + struct PolymorphicBaseClass< geode::Identifier > + : PolymorphicDerivedClasses< geode::VertexSet > + { + }; + template <> struct PolymorphicBaseClass< geode::VertexSet > : PolymorphicDerivedClasses< geode::Graph, diff --git a/tests/basic/CMakeLists.txt b/tests/basic/CMakeLists.txt index bd38487dc..7240a6038 100644 --- a/tests/basic/CMakeLists.txt +++ b/tests/basic/CMakeLists.txt @@ -44,6 +44,11 @@ add_geode_test( DEPENDENCIES ${PROJECT_NAME}::basic ) +add_geode_test( + SOURCE "test-database.cpp" + DEPENDENCIES + ${PROJECT_NAME}::basic +) add_geode_test( SOURCE "test-factory.cpp" DEPENDENCIES diff --git a/tests/basic/test-database.cpp b/tests/basic/test-database.cpp new file mode 100644 index 000000000..311316d66 --- /dev/null +++ b/tests/basic/test-database.cpp @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2019 - 2022 Geode-solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ + +#include +#include +#include +#include +#include + +#include + +struct Foo : public geode::Identifier +{ + Foo() = default; + Foo( double value ) : value_( value ) {} + + template < typename Archive > + void serialize( Archive& archive ) + { + archive.ext( *this, bitsery::ext::BaseClass< geode::Identifier >{} ); + archive.value8b( value_ ); + } + + double value_{ 0 }; +}; + +void register_foo_serializer( geode::PContext& context ) +{ + context + .registerSingleBaseBranch< geode::Serializer, geode::Identifier, Foo >( + "Foo" ); +} + +void register_foo_deserializer( geode::PContext& context ) +{ + context.registerSingleBaseBranch< geode::Deserializer, geode::Identifier, + Foo >( "Foo" ); +} + +void test_take_data( geode::Database& database, const geode::uuid& id ) +{ + auto stolen_foo = database.take_data< Foo >( id ); + OPENGEODE_EXCEPTION( + stolen_foo->value_ == 42, "[Test] Wrong value after take data" ); + auto foo_data = database.get_data( id ); + const auto& foo = foo_data.get< Foo >(); + OPENGEODE_EXCEPTION( + foo.value_ == 42, "[Test] Wrong value after register data" ); + OPENGEODE_EXCEPTION( stolen_foo.get() != &foo, + "[Test] Objects adresses should be different" ); +} + +void test_take_wrong_data( geode::Database& database, const geode::uuid& id ) +{ + try + { + auto stolen_foo = database.take_data< geode::uuid >( id ); + throw geode::OpenGeodeException{ + "[Test] Should not be able to cast data into uuid" + }; + } + catch( ... ) + { + return; + } +} + +geode::uuid test_register_data( geode::Database& database ) +{ + auto foo = database.register_new_data( Foo{ 42 } ); + auto foo_data = database.get_data( foo ); + OPENGEODE_EXCEPTION( foo_data.get< Foo >().value_ == 42, + "[Test] Wrong value after register data" ); + return foo; +} + +geode::uuid test_register_unique_data( geode::Database& database ) +{ + auto foo = database.register_new_data( absl::make_unique< Foo >( 22 ) ); + auto foo_data = database.get_data( foo ); + OPENGEODE_EXCEPTION( foo_data.get< Foo >().value_ == 22, + "[Test] Wrong value after register unique data" ); + return foo; +} + +void test_modify_data( geode::Database& database, const geode::uuid& id ) +{ + auto foo_data = database.get_data( id ); + const auto& foo = foo_data.get< Foo >(); + auto taken_foo = database.take_data< Foo >( id ); + taken_foo->value_ = 12; + OPENGEODE_EXCEPTION( + taken_foo->value_ == 12, "[Test] Wrong value after modify taken data" ); + OPENGEODE_EXCEPTION( + foo.value_ == 42, "[Test] Wrong value after register data" ); + database.update_data( id, std::move( taken_foo ) ); + auto foo_data2 = database.get_data( id ); + const auto& foo2 = foo_data2.get< Foo >(); + OPENGEODE_EXCEPTION( + foo2.value_ == 12, "[Test] Wrong value after register data" ); + OPENGEODE_EXCEPTION( + foo.value_ == 42, "[Test] Wrong value after register data" ); +} + +void test() +{ + geode::Database database( "temp" ); + database.register_serializer_functions( + register_foo_serializer, register_foo_deserializer ); + auto foo0 = test_register_data( database ); + test_register_unique_data( database ); + test_take_data( database, foo0 ); + test_modify_data( database, foo0 ); + test_take_wrong_data( database, foo0 ); + OPENGEODE_EXCEPTION( + database.nb_data() == 2, "[Test] Database incomplete" ); +} + +OPENGEODE_TEST( "filename" ) \ No newline at end of file