From fafc0958eed6f5064f3e181849e7b47ab91927a1 Mon Sep 17 00:00:00 2001
From: Konstantin Munichev <toobwn@gmail.com>
Date: Wed, 20 Mar 2019 21:54:14 +0300
Subject: [PATCH] Flat file block storage (#2180)
Signed-off-by: Konstantin Munichev <toobwn@gmail.com>
---
irohad/ametsuchi/CMakeLists.txt | 2 +
irohad/ametsuchi/block_storage_factory.hpp | 1 -
irohad/ametsuchi/impl/flat_file/flat_file.cpp | 88 ++++-----
irohad/ametsuchi/impl/flat_file/flat_file.hpp | 42 ++--
.../impl/flat_file_block_storage.cpp | 85 +++++++++
.../impl/flat_file_block_storage.hpp | 50 +++++
.../impl/flat_file_block_storage_factory.cpp | 31 +++
.../impl/flat_file_block_storage_factory.hpp | 34 ++++
irohad/main/application.cpp | 13 +-
.../block_json_deserializer.hpp | 2 +-
.../iroha_internal/block_json_serializer.hpp | 2 +-
test/module/irohad/ametsuchi/CMakeLists.txt | 6 +
.../flat_file_block_storage_test.cpp | 180 ++++++++++++++++++
.../irohad/ametsuchi/flat_file_test.cpp | 62 +++++-
.../in_memory_block_storage_test.cpp | 8 +-
test/module/shared_model/interface_mocks.hpp | 15 ++
16 files changed, 531 insertions(+), 90 deletions(-)
create mode 100644 irohad/ametsuchi/impl/flat_file_block_storage.cpp
create mode 100644 irohad/ametsuchi/impl/flat_file_block_storage.hpp
create mode 100644 irohad/ametsuchi/impl/flat_file_block_storage_factory.cpp
create mode 100644 irohad/ametsuchi/impl/flat_file_block_storage_factory.hpp
create mode 100644 test/module/irohad/ametsuchi/flat_file_block_storage_test.cpp
diff --git a/irohad/ametsuchi/CMakeLists.txt b/irohad/ametsuchi/CMakeLists.txt
index 158f6ec988..cd9860493f 100644
--- a/irohad/ametsuchi/CMakeLists.txt
+++ b/irohad/ametsuchi/CMakeLists.txt
@@ -20,6 +20,8 @@ add_library(ametsuchi
impl/tx_presence_cache_impl.cpp
impl/in_memory_block_storage.cpp
impl/in_memory_block_storage_factory.cpp
+ impl/flat_file_block_storage.cpp
+ impl/flat_file_block_storage_factory.cpp
)
target_link_libraries(ametsuchi
diff --git a/irohad/ametsuchi/block_storage_factory.hpp b/irohad/ametsuchi/block_storage_factory.hpp
index c0e7655407..e738813d12 100644
--- a/irohad/ametsuchi/block_storage_factory.hpp
+++ b/irohad/ametsuchi/block_storage_factory.hpp
@@ -12,7 +12,6 @@
namespace iroha {
namespace ametsuchi {
-
/**
* Creates a block storage
*/
diff --git a/irohad/ametsuchi/impl/flat_file/flat_file.cpp b/irohad/ametsuchi/impl/flat_file/flat_file.cpp
index b3b73643d2..5620205b0d 100644
--- a/irohad/ametsuchi/impl/flat_file/flat_file.cpp
+++ b/irohad/ametsuchi/impl/flat_file/flat_file.cpp
@@ -18,6 +18,7 @@
using namespace iroha::ametsuchi;
using Identifier = FlatFile::Identifier;
+using BlockIdCollectionType = FlatFile::BlockIdCollectionType;
// ----------| public API |----------
@@ -27,9 +28,20 @@ std::string FlatFile::id_to_name(Identifier id) {
return os.str();
}
+boost::optional<Identifier> FlatFile::name_to_id(const std::string &name) {
+ if (name.size() != FlatFile::DIGIT_CAPACITY) {
+ return boost::none;
+ }
+ try {
+ auto id = std::stoul(name);
+ return boost::make_optional<Identifier>(id);
+ } catch (const std::exception &e) {
+ return boost::none;
+ }
+}
+
boost::optional<std::unique_ptr<FlatFile>> FlatFile::create(
- const std::string &path,
- logger::LoggerPtr log) {
+ const std::string &path, logger::LoggerPtr log) {
boost::system::error_code err;
if (not boost::filesystem::is_directory(path, err)
and not boost::filesystem::create_directory(path, err)) {
@@ -37,19 +49,24 @@ boost::optional<std::unique_ptr<FlatFile>> FlatFile::create(
return boost::none;
}
- auto res = FlatFile::check_consistency(path, log);
- return std::make_unique<FlatFile>(*res, path, private_tag{}, std::move(log));
+ BlockIdCollectionType files_found;
+ for (auto it = boost::filesystem::directory_iterator{path};
+ it != boost::filesystem::directory_iterator{};
+ ++it) {
+ if (auto id = FlatFile::name_to_id(it->path().filename().string())) {
+ files_found.insert(*id);
+ } else {
+ boost::filesystem::remove(it->path());
+ }
+ }
+
+ return std::make_unique<FlatFile>(
+ path, std::move(files_found), private_tag{}, std::move(log));
}
bool FlatFile::add(Identifier id, const Bytes &block) {
// TODO(x3medima17): Change bool to generic Result return type
- if (id != current_id_ + 1) {
- log_->warn("Cannot append non-consecutive block");
- return false;
- }
-
- auto next_id = id;
const auto file_name = boost::filesystem::path{dump_dir_} / id_to_name(id);
// Write block to binary file
@@ -71,8 +88,7 @@ bool FlatFile::add(Identifier id, const Bytes &block) {
file.write(reinterpret_cast<const char *>(block.data()),
block.size() * val_size);
- // Update internals, release lock
- current_id_ = next_id;
+ available_blocks_.insert(id);
return true;
}
@@ -100,50 +116,24 @@ std::string FlatFile::directory() const {
}
Identifier FlatFile::last_id() const {
- return current_id_.load();
+ return (available_blocks_.empty()) ? 0 : *available_blocks_.rbegin();
}
void FlatFile::dropAll() {
iroha::remove_dir_contents(dump_dir_, log_);
- auto res = FlatFile::check_consistency(dump_dir_, log_);
- current_id_.store(*res);
+ available_blocks_.clear();
+}
+
+const BlockIdCollectionType &FlatFile::blockIdentifiers() const {
+ return available_blocks_;
}
// ----------| private API |----------
-FlatFile::FlatFile(Identifier current_id,
- const std::string &path,
+FlatFile::FlatFile(std::string path,
+ BlockIdCollectionType existing_files,
FlatFile::private_tag,
logger::LoggerPtr log)
- : dump_dir_(path), log_{std::move(log)} {
- current_id_.store(current_id);
-}
-
-boost::optional<Identifier> FlatFile::check_consistency(
- const std::string &dump_dir, logger::LoggerPtr log) {
- if (dump_dir.empty()) {
- log->error("check_consistency({}), not directory", dump_dir);
- return boost::none;
- }
-
- auto const files = [&dump_dir] {
- std::vector<boost::filesystem::path> ps;
- std::copy(boost::filesystem::directory_iterator{dump_dir},
- boost::filesystem::directory_iterator{},
- std::back_inserter(ps));
- std::sort(ps.begin(), ps.end(), std::less<boost::filesystem::path>());
- return ps;
- }();
-
- auto const missing = boost::range::find_if(
- files | boost::adaptors::indexed(1), [](const auto &it) {
- return FlatFile::id_to_name(it.index()) != it.value().filename();
- });
-
- std::for_each(
- missing.get(), files.cend(), [](const boost::filesystem::path &p) {
- boost::filesystem::remove(p);
- });
-
- return missing.get() - files.cbegin();
-}
+ : dump_dir_(std::move(path)),
+ available_blocks_(std::move(existing_files)),
+ log_{std::move(log)} {}
diff --git a/irohad/ametsuchi/impl/flat_file/flat_file.hpp b/irohad/ametsuchi/impl/flat_file/flat_file.hpp
index da20283075..59f7081848 100644
--- a/irohad/ametsuchi/impl/flat_file/flat_file.hpp
+++ b/irohad/ametsuchi/impl/flat_file/flat_file.hpp
@@ -8,8 +8,8 @@
#include "ametsuchi/key_value_storage.hpp"
-#include <atomic>
#include <memory>
+#include <set>
#include "logger/logger_fwd.hpp"
@@ -29,6 +29,8 @@ namespace iroha {
public:
// ----------| public API |----------
+ using BlockIdCollectionType = std::set<Identifier>;
+
static const uint32_t DIGIT_CAPACITY = 16;
/**
@@ -45,6 +47,13 @@ namespace iroha {
*/
static std::string id_to_name(Identifier id);
+ /**
+ * Converts aligned string (see above) to number.
+ * @param name - name to convert
+ * @return id or boost::none
+ */
+ static boost::optional<Identifier> name_to_id(const std::string &name);
+
/**
* Create storage in paths
* @param path - target path for creating
@@ -62,18 +71,12 @@ namespace iroha {
Identifier last_id() const override;
+ void dropAll() override;
+
/**
- * Checking consistency of storage for provided folder
- * If some block in the middle is missing all blocks following it are
- * deleted
- * @param dump_dir - folder of storage
- * @param log - log for local messages
- * @return - last available identifier
+ * @return collection of available block ids
*/
- static boost::optional<Identifier> check_consistency(
- const std::string &dump_dir, logger::LoggerPtr log);
-
- void dropAll() override;
+ const BlockIdCollectionType &blockIdentifiers() const;
// ----------| modify operations |----------
@@ -88,29 +91,24 @@ namespace iroha {
// ----------| private API |----------
/**
- * Create storage in path with respect to last key
- * @param last_id - maximal key written in storage
+ * Create storage in path
* @param path - folder of storage
+ * @param existing_files - collection of existing files names
* @param log to print progress
*/
- FlatFile(Identifier last_id,
- const std::string &path,
+ FlatFile(std::string path,
+ BlockIdCollectionType existing_files,
FlatFile::private_tag,
logger::LoggerPtr log);
private:
- // ----------| private fields |----------
-
- /**
- * Last written key
- */
- std::atomic<Identifier> current_id_;
-
/**
* Folder of storage
*/
const std::string dump_dir_;
+ BlockIdCollectionType available_blocks_;
+
logger::LoggerPtr log_;
public:
diff --git a/irohad/ametsuchi/impl/flat_file_block_storage.cpp b/irohad/ametsuchi/impl/flat_file_block_storage.cpp
new file mode 100644
index 0000000000..dba89a8a1e
--- /dev/null
+++ b/irohad/ametsuchi/impl/flat_file_block_storage.cpp
@@ -0,0 +1,85 @@
+/**
+ * Copyright Soramitsu Co., Ltd. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include "ametsuchi/impl/flat_file_block_storage.hpp"
+
+#include <boost/filesystem.hpp>
+
+#include "backend/protobuf/block.hpp"
+#include "common/byteutils.hpp"
+#include "logger/logger.hpp"
+
+using namespace iroha::ametsuchi;
+
+FlatFileBlockStorage::FlatFileBlockStorage(
+ std::unique_ptr<FlatFile> flat_file,
+ std::shared_ptr<shared_model::interface::BlockJsonConverter> json_converter,
+ logger::LoggerPtr log)
+ : flat_file_storage_(std::move(flat_file)),
+ json_converter_(std::move(json_converter)),
+ log_(std::move(log)) {}
+
+FlatFileBlockStorage::~FlatFileBlockStorage() {
+ log_->info("Remove {} temp directory", flat_file_storage_->directory());
+ boost::filesystem::remove_all(flat_file_storage_->directory());
+}
+
+bool FlatFileBlockStorage::insert(
+ std::shared_ptr<const shared_model::interface::Block> block) {
+ return json_converter_->serialize(*block).match(
+ [&](const expected::Value<std::string> &block_json) {
+ return flat_file_storage_->add(block->height(),
+ stringToBytes(block_json.value));
+ },
+ [this](const auto &error) {
+ log_->warn("Error while block serialization: {}", error.error);
+ return false;
+ });
+}
+
+bool FlatFileBlockStorage::insert(const shared_model::interface::Block &block) {
+ return insert(clone(block));
+}
+
+boost::optional<std::shared_ptr<const shared_model::interface::Block>>
+FlatFileBlockStorage::fetch(
+ shared_model::interface::types::HeightType height) const {
+ auto storage_block = flat_file_storage_->get(height);
+ if (not storage_block) {
+ return boost::none;
+ }
+
+ return json_converter_->deserialize(bytesToString(*storage_block))
+ .match(
+ [&](expected::Value<std::unique_ptr<shared_model::interface::Block>>
+ &block) {
+ return boost::make_optional<
+ std::shared_ptr<const shared_model::interface::Block>>(
+ std::move(block.value));
+ },
+ [&](expected::Error<std::string> &error)
+ -> boost::optional<
+ std::shared_ptr<const shared_model::interface::Block>> {
+ log_->warn("Error while block deserialization: {}", error.error);
+ return boost::none;
+ });
+}
+
+size_t FlatFileBlockStorage::size() const {
+ return flat_file_storage_->blockIdentifiers().size();
+}
+
+void FlatFileBlockStorage::clear() {
+ flat_file_storage_->dropAll();
+}
+
+void FlatFileBlockStorage::forEach(
+ iroha::ametsuchi::BlockStorage::FunctionType function) const {
+ for (auto block_id : flat_file_storage_->blockIdentifiers()) {
+ auto block = fetch(block_id);
+ BOOST_ASSERT(block);
+ function(*block);
+ }
+}
diff --git a/irohad/ametsuchi/impl/flat_file_block_storage.hpp b/irohad/ametsuchi/impl/flat_file_block_storage.hpp
new file mode 100644
index 0000000000..3cdb7be820
--- /dev/null
+++ b/irohad/ametsuchi/impl/flat_file_block_storage.hpp
@@ -0,0 +1,50 @@
+/**
+ * Copyright Soramitsu Co., Ltd. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#ifndef IROHA_FLAT_FILE_BLOCK_STORAGE_HPP
+#define IROHA_FLAT_FILE_BLOCK_STORAGE_HPP
+
+#include "ametsuchi/block_storage.hpp"
+
+#include "ametsuchi/impl/flat_file/flat_file.hpp"
+#include "interfaces/iroha_internal/block_json_converter.hpp"
+#include "logger/logger_fwd.hpp"
+
+namespace iroha {
+ namespace ametsuchi {
+ class FlatFileBlockStorage : public BlockStorage {
+ public:
+ FlatFileBlockStorage(
+ std::unique_ptr<FlatFile> flat_file,
+ std::shared_ptr<shared_model::interface::BlockJsonConverter>
+ json_converter,
+ logger::LoggerPtr log);
+
+ ~FlatFileBlockStorage() override;
+
+ bool insert(
+ std::shared_ptr<const shared_model::interface::Block> block) override;
+
+ bool insert(const shared_model::interface::Block &block) override;
+
+ boost::optional<std::shared_ptr<const shared_model::interface::Block>>
+ fetch(shared_model::interface::types::HeightType height) const override;
+
+ size_t size() const override;
+
+ void clear() override;
+
+ void forEach(FunctionType function) const override;
+
+ private:
+ std::unique_ptr<FlatFile> flat_file_storage_;
+ std::shared_ptr<shared_model::interface::BlockJsonConverter>
+ json_converter_;
+ logger::LoggerPtr log_;
+ };
+ } // namespace ametsuchi
+} // namespace iroha
+
+#endif // IROHA_FLAT_FILE_BLOCK_STORAGE_HPP
diff --git a/irohad/ametsuchi/impl/flat_file_block_storage_factory.cpp b/irohad/ametsuchi/impl/flat_file_block_storage_factory.cpp
new file mode 100644
index 0000000000..6b69e46f66
--- /dev/null
+++ b/irohad/ametsuchi/impl/flat_file_block_storage_factory.cpp
@@ -0,0 +1,31 @@
+/**
+ * Copyright Soramitsu Co., Ltd. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include "ametsuchi/impl/flat_file_block_storage_factory.hpp"
+
+#include "ametsuchi/impl/flat_file_block_storage.hpp"
+
+using namespace iroha::ametsuchi;
+
+FlatFileBlockStorageFactory::FlatFileBlockStorageFactory(
+ std::function<std::string()> path_provider,
+ std::shared_ptr<shared_model::interface::BlockJsonConverter>
+ json_block_converter,
+ logger::LoggerManagerTreePtr log_manager)
+ : path_provider_(std::move(path_provider)),
+ json_block_converter_(std::move(json_block_converter)),
+ log_manager_(std::move(log_manager)) {}
+
+std::unique_ptr<BlockStorage> FlatFileBlockStorageFactory::create() {
+ auto flat_file = FlatFile::create(
+ path_provider_(), log_manager_->getChild("FlatFile")->getLogger());
+ if (not flat_file) {
+ return nullptr;
+ }
+ return std::make_unique<FlatFileBlockStorage>(
+ std::move(flat_file.get()),
+ json_block_converter_,
+ log_manager_->getChild("FlatFileBlockFactory")->getLogger());
+}
diff --git a/irohad/ametsuchi/impl/flat_file_block_storage_factory.hpp b/irohad/ametsuchi/impl/flat_file_block_storage_factory.hpp
new file mode 100644
index 0000000000..30d9f288e5
--- /dev/null
+++ b/irohad/ametsuchi/impl/flat_file_block_storage_factory.hpp
@@ -0,0 +1,34 @@
+/**
+ * Copyright Soramitsu Co., Ltd. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#ifndef IROHA_FLAT_FILE_BLOCK_STORAGE_FACTORY_HPP
+#define IROHA_FLAT_FILE_BLOCK_STORAGE_FACTORY_HPP
+
+#include "ametsuchi/block_storage_factory.hpp"
+
+#include "interfaces/iroha_internal/block_json_converter.hpp"
+#include "logger/logger_manager.hpp"
+
+namespace iroha {
+ namespace ametsuchi {
+ class FlatFileBlockStorageFactory : public BlockStorageFactory {
+ public:
+ FlatFileBlockStorageFactory(
+ std::function<std::string()> path_provider,
+ std::shared_ptr<shared_model::interface::BlockJsonConverter>
+ json_block_converter,
+ logger::LoggerManagerTreePtr log_manager);
+ std::unique_ptr<BlockStorage> create() override;
+
+ private:
+ std::function<std::string()> path_provider_;
+ std::shared_ptr<shared_model::interface::BlockJsonConverter>
+ json_block_converter_;
+ logger::LoggerManagerTreePtr log_manager_;
+ };
+ } // namespace ametsuchi
+} // namespace iroha
+
+#endif // IROHA_FLAT_FILE_BLOCK_STORAGE_FACTORY_HPP
diff --git a/irohad/main/application.cpp b/irohad/main/application.cpp
index 751becedc9..dc2085236b 100644
--- a/irohad/main/application.cpp
+++ b/irohad/main/application.cpp
@@ -5,7 +5,9 @@
#include "main/application.hpp"
-#include "ametsuchi/impl/in_memory_block_storage_factory.hpp"
+#include <boost/filesystem.hpp>
+
+#include "ametsuchi/impl/flat_file_block_storage_factory.hpp"
#include "ametsuchi/impl/storage_impl.hpp"
#include "ametsuchi/impl/tx_presence_cache_impl.hpp"
#include "ametsuchi/impl/wsv_restorer_impl.hpp"
@@ -161,7 +163,14 @@ void Irohad::initStorage() {
std::make_shared<shared_model::proto::ProtoPermissionToString>();
auto block_converter =
std::make_shared<shared_model::proto::ProtoBlockJsonConverter>();
- auto block_storage_factory = std::make_unique<InMemoryBlockStorageFactory>();
+ auto block_storage_factory = std::make_unique<FlatFileBlockStorageFactory>(
+ []() {
+ return (boost::filesystem::temp_directory_path()
+ / boost::filesystem::unique_path())
+ .string();
+ },
+ block_converter,
+ log_manager_);
auto storageResult = StorageImpl::create(block_store_dir_,
pg_conn_,
common_objects_factory_,
diff --git a/shared_model/interfaces/iroha_internal/block_json_deserializer.hpp b/shared_model/interfaces/iroha_internal/block_json_deserializer.hpp
index 444bfaf5b4..d57273f71a 100644
--- a/shared_model/interfaces/iroha_internal/block_json_deserializer.hpp
+++ b/shared_model/interfaces/iroha_internal/block_json_deserializer.hpp
@@ -26,7 +26,7 @@ namespace shared_model {
* @return pointer to a block if json was valid or an error
*/
virtual iroha::expected::Result<std::unique_ptr<Block>, std::string>
- deserialize(const types::JsonType &json) const noexcept = 0;
+ deserialize(const types::JsonType &json) const = 0;
virtual ~BlockJsonDeserializer() = default;
};
diff --git a/shared_model/interfaces/iroha_internal/block_json_serializer.hpp b/shared_model/interfaces/iroha_internal/block_json_serializer.hpp
index f505e5321d..238528cb67 100644
--- a/shared_model/interfaces/iroha_internal/block_json_serializer.hpp
+++ b/shared_model/interfaces/iroha_internal/block_json_serializer.hpp
@@ -26,7 +26,7 @@ namespace shared_model {
* @return json string or an error
*/
virtual iroha::expected::Result<types::JsonType, std::string>
- serialize(const Block &block) const noexcept = 0;
+ serialize(const Block &block) const = 0;
virtual ~BlockJsonSerializer() = default;
};
diff --git a/test/module/irohad/ametsuchi/CMakeLists.txt b/test/module/irohad/ametsuchi/CMakeLists.txt
index 09bed3e35d..63e12c49fc 100644
--- a/test/module/irohad/ametsuchi/CMakeLists.txt
+++ b/test/module/irohad/ametsuchi/CMakeLists.txt
@@ -74,6 +74,12 @@ target_link_libraries(in_memory_block_storage_test
ametsuchi
)
+addtest(flat_file_block_storage_test flat_file_block_storage_test.cpp)
+target_link_libraries(flat_file_block_storage_test
+ ametsuchi
+ test_logger
+ )
+
add_library(ametsuchi_fixture INTERFACE)
target_link_libraries(ametsuchi_fixture INTERFACE
integration_framework_config_helper
diff --git a/test/module/irohad/ametsuchi/flat_file_block_storage_test.cpp b/test/module/irohad/ametsuchi/flat_file_block_storage_test.cpp
new file mode 100644
index 0000000000..7ee0e6c5d8
--- /dev/null
+++ b/test/module/irohad/ametsuchi/flat_file_block_storage_test.cpp
@@ -0,0 +1,180 @@
+/**
+ * Copyright Soramitsu Co., Ltd. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include "ametsuchi/impl/flat_file_block_storage.hpp"
+#include "ametsuchi/impl/flat_file_block_storage_factory.hpp"
+
+#include <gtest/gtest.h>
+#include <boost/filesystem.hpp>
+#include "framework/test_logger.hpp"
+#include "module/shared_model/interface_mocks.hpp"
+
+using namespace iroha::ametsuchi;
+using namespace boost::filesystem;
+
+using ::testing::_;
+using ::testing::Invoke;
+using ::testing::NiceMock;
+using ::testing::Return;
+
+class FlatFileBlockStorageTest : public ::testing::Test {
+ public:
+ FlatFileBlockStorageTest() {
+ ON_CALL(*block_, height()).WillByDefault(Return(height_));
+ }
+
+ protected:
+ void SetUp() override {
+ create_directory(path_provider_());
+ }
+
+ const std::string block_store_path_ =
+ (temp_directory_path() / unique_path()).string();
+
+ std::function<std::string()> path_provider_ = [&]() {
+ return block_store_path_;
+ };
+
+ std::shared_ptr<MockBlockJsonConverter> converter_ =
+ std::make_shared<NiceMock<MockBlockJsonConverter>>();
+ logger::LoggerManagerTreePtr log_manager_ = getTestLoggerManager();
+ std::shared_ptr<MockBlock> block_ = std::make_shared<NiceMock<MockBlock>>();
+ shared_model::interface::types::HeightType height_ = 1;
+};
+
+/**
+ * @given block storage factory
+ * @when create is called
+ * @then block storage is created
+ */
+TEST_F(FlatFileBlockStorageTest, Creation) {
+ auto block_storage =
+ FlatFileBlockStorageFactory(path_provider_, converter_, log_manager_)
+ .create();
+ ASSERT_TRUE(block_storage);
+}
+
+/**
+ * @given initialized block storage, single block with height_ inserted
+ * @when another block with height_ is inserted
+ * @then second insertion fails
+ */
+TEST_F(FlatFileBlockStorageTest, Insert) {
+ auto block_storage =
+ FlatFileBlockStorageFactory(path_provider_, converter_, log_manager_)
+ .create();
+ ASSERT_TRUE(block_storage->insert(block_));
+ ASSERT_FALSE(block_storage->insert(block_));
+}
+
+/**
+ * @given initialized block storage, single block with height_ inserted
+ * @when block with height_ is fetched
+ * @then it is returned
+ */
+TEST_F(FlatFileBlockStorageTest, FetchExisting) {
+ auto block_storage =
+ FlatFileBlockStorageFactory(path_provider_, converter_, log_manager_)
+ .create();
+ ASSERT_TRUE(block_storage->insert(block_));
+
+ shared_model::interface::Block *raw_block;
+
+ EXPECT_CALL(*converter_, deserialize(_))
+ .WillOnce(Invoke([&](const shared_model::interface::types::JsonType &)
+ -> iroha::expected::Result<
+ std::unique_ptr<shared_model::interface::Block>,
+ std::string> {
+ auto return_block = std::make_unique<MockBlock>();
+ raw_block = return_block.get();
+ return iroha::expected::makeValue<
+ std::unique_ptr<shared_model::interface::Block>>(
+ std::move(return_block));
+ }));
+ std::shared_ptr<const shared_model::interface::Block> block_var =
+ *(block_storage->fetch(height_));
+
+ ASSERT_EQ(raw_block, block_var.get());
+}
+
+/**
+ * @given initialized block storage without blocks
+ * @when block with height_ is fetched
+ * @then nothing is returned
+ */
+TEST_F(FlatFileBlockStorageTest, FetchNonexistent) {
+ auto block_storage =
+ FlatFileBlockStorageFactory(path_provider_, converter_, log_manager_)
+ .create();
+ auto block_var = block_storage->fetch(height_);
+ ASSERT_FALSE(block_var);
+}
+
+/**
+ * @given initialized block storage, single block with height_ inserted
+ * @when size is fetched
+ * @then 1 is returned
+ */
+TEST_F(FlatFileBlockStorageTest, Size) {
+ auto block_storage =
+ FlatFileBlockStorageFactory(path_provider_, converter_, log_manager_)
+ .create();
+ ASSERT_TRUE(block_storage->insert(block_));
+
+ ASSERT_EQ(1, block_storage->size());
+}
+
+/**
+ * @given initialized block storage, single block with height_ inserted
+ * @when storage is cleared with clear
+ * @then no blocks are left in storage
+ */
+TEST_F(FlatFileBlockStorageTest, Clear) {
+ auto block_storage =
+ FlatFileBlockStorageFactory(path_provider_, converter_, log_manager_)
+ .create();
+ ASSERT_TRUE(block_storage->insert(block_));
+
+ block_storage->clear();
+
+ auto block_var = block_storage->fetch(height_);
+
+ ASSERT_FALSE(block_var);
+}
+
+/**
+ * @given initialized block storage, single block with height_ inserted
+ * @when forEach is called
+ * @then block with height_ is visited, lambda is invoked once
+ */
+TEST_F(FlatFileBlockStorageTest, ForEach) {
+ auto block_storage =
+ FlatFileBlockStorageFactory(path_provider_, converter_, log_manager_)
+ .create();
+ ASSERT_TRUE(block_storage->insert(block_));
+
+ shared_model::interface::Block *raw_block;
+
+ EXPECT_CALL(*converter_, deserialize(_))
+ .WillOnce(Invoke([&](const shared_model::interface::types::JsonType &)
+ -> iroha::expected::Result<
+ std::unique_ptr<shared_model::interface::Block>,
+ std::string> {
+ auto return_block = std::make_unique<MockBlock>();
+ raw_block = return_block.get();
+ return iroha::expected::makeValue<
+ std::unique_ptr<shared_model::interface::Block>>(
+ std::move(return_block));
+ }));
+
+ size_t count = 0;
+
+ block_storage->forEach([&count, &raw_block](const auto &block) {
+ ++count;
+ ASSERT_EQ(raw_block, block.get());
+ });
+
+ ASSERT_EQ(1, count);
+}
diff --git a/test/module/irohad/ametsuchi/flat_file_test.cpp b/test/module/irohad/ametsuchi/flat_file_test.cpp
index e4d5a1647e..f6dc7f9ea5 100644
--- a/test/module/irohad/ametsuchi/flat_file_test.cpp
+++ b/test/module/irohad/ametsuchi/flat_file_test.cpp
@@ -52,6 +52,12 @@ TEST_F(BlStore_Test, Read_Write_Test) {
ASSERT_EQ(*res, block);
}
+/**
+ * @given initialized FlatFile storage and 3 blocks are inserted into it
+ * @when storage removed, file for a second block removed and new storage is
+ * created on the same directory
+ * @then last block id is still 3
+ */
TEST_F(BlStore_Test, BlockStoreWhenRemoveBlock) {
log_->info("----------| Simulate removal of the block |----------");
// Remove file in the middle of the block store
@@ -79,7 +85,7 @@ TEST_F(BlStore_Test, BlockStoreWhenRemoveBlock) {
ASSERT_TRUE(store);
auto bl_store = std::move(*store);
auto res = bl_store->last_id();
- ASSERT_EQ(res, 1);
+ ASSERT_EQ(res, 3);
}
TEST_F(BlStore_Test, BlockStoreWhenAbsentFolder) {
@@ -120,15 +126,6 @@ TEST_F(BlStore_Test, BlockStoreInitializationFromNonemptyFolder) {
ASSERT_EQ(bl_store1->last_id(), bl_store2->last_id());
}
-/**
- * @given empty folder name
- * @then check consistency fails
- */
-TEST_F(BlStore_Test, EmptyDumpDir) {
- auto res = FlatFile::check_consistency("", flat_file_log_);
- ASSERT_FALSE(res);
-}
-
/**
* @given empty folder with block store
* @when block id does not exist
@@ -217,3 +214,48 @@ TEST_F(BlStore_Test, WriteDeniedFolder) {
auto res = bl_store->add(id, block);
ASSERT_FALSE(res);
}
+
+/**
+ * @given initialized FlatFile storage
+ * @when several blocks with non-consecutive ids are inserted
+ * @then all inserted blocks are available, block 1 is not available
+ */
+TEST_F(BlStore_Test, RandomNumbers) {
+ auto store = FlatFile::create(block_store_path, flat_file_log_);
+ ASSERT_TRUE(store);
+ auto bl_store = std::move(*store);
+ bl_store->add(5, block);
+ bl_store->add(22, block);
+ bl_store->add(11, block);
+ ASSERT_TRUE(bl_store->get(5));
+ ASSERT_TRUE(bl_store->get(22));
+ ASSERT_TRUE(bl_store->get(11));
+ ASSERT_FALSE(bl_store->get(1));
+}
+
+/**
+ * @given initialized FlatFile storage
+ * @when 3 blocks with non-consecutive ids are inserted, storage removed, and
+ * new storage is created on the same directory
+ * @then only blocks with inserted ids are available
+ */
+TEST_F(BlStore_Test, RemoveAndCreateNew) {
+ {
+ auto store = FlatFile::create(block_store_path, flat_file_log_);
+ ASSERT_TRUE(store);
+ auto bl_store = std::move(*store);
+
+ bl_store->add(4, block);
+ bl_store->add(17, block);
+ bl_store->add(7, block);
+ }
+
+ auto store = FlatFile::create(block_store_path, flat_file_log_);
+ ASSERT_TRUE(store);
+ auto bl_store = std::move(*store);
+
+ ASSERT_TRUE(bl_store->get(4));
+ ASSERT_TRUE(bl_store->get(17));
+ ASSERT_TRUE(bl_store->get(7));
+ ASSERT_FALSE(bl_store->get(1));
+}
diff --git a/test/module/irohad/ametsuchi/in_memory_block_storage_test.cpp b/test/module/irohad/ametsuchi/in_memory_block_storage_test.cpp
index b30d927f97..37cb983a11 100644
--- a/test/module/irohad/ametsuchi/in_memory_block_storage_test.cpp
+++ b/test/module/irohad/ametsuchi/in_memory_block_storage_test.cpp
@@ -77,7 +77,7 @@ TEST_F(InMemoryBlockStorageTest, FetchNonexistent) {
}
/**
- * @given initialized block storage, single block with id_ inserted
+ * @given initialized block storage, single block with height_ inserted
* @when size is fetched
* @then 1 is returned
*/
@@ -88,7 +88,7 @@ TEST_F(InMemoryBlockStorageTest, Size) {
}
/**
- * @given initialized block storage, single block with id_ inserted
+ * @given initialized block storage, single block with height_ inserted
* @when storage is cleared with clear
* @then no blocks are left in storage
*/
@@ -103,9 +103,9 @@ TEST_F(InMemoryBlockStorageTest, Clear) {
}
/**
- * @given initialized block storage, single block with id_ inserted
+ * @given initialized block storage, single block with height_ inserted
* @when forEach is called
- * @then block with id_ is visited, lambda is invoked once
+ * @then block with height_ is visited, lambda is invoked once
*/
TEST_F(InMemoryBlockStorageTest, ForEach) {
ASSERT_TRUE(block_storage_.insert(block_));
diff --git a/test/module/shared_model/interface_mocks.hpp b/test/module/shared_model/interface_mocks.hpp
index 2d387c0a44..34fb5b5e50 100644
--- a/test/module/shared_model/interface_mocks.hpp
+++ b/test/module/shared_model/interface_mocks.hpp
@@ -13,6 +13,7 @@
#include "interfaces/common_objects/common_objects_factory.hpp"
#include "interfaces/common_objects/peer.hpp"
#include "interfaces/iroha_internal/block.hpp"
+#include "interfaces/iroha_internal/block_json_converter.hpp"
#include "interfaces/iroha_internal/proposal.hpp"
#include "interfaces/iroha_internal/transaction_batch.hpp"
#include "interfaces/iroha_internal/unsafe_proposal_factory.hpp"
@@ -260,4 +261,18 @@ struct MockAccount : public shared_model::interface::Account {
MOCK_CONST_METHOD0(clone, MockAccount *());
};
+struct MockBlockJsonConverter
+ : public shared_model::interface::BlockJsonConverter {
+ MOCK_CONST_METHOD1(
+ serialize,
+ iroha::expected::Result<shared_model::interface::types::JsonType,
+ std::string>(
+ const shared_model::interface::Block &block));
+ MOCK_CONST_METHOD1(
+ deserialize,
+ iroha::expected::Result<std::unique_ptr<shared_model::interface::Block>,
+ std::string>(
+ const shared_model::interface::types::JsonType &json));
+};
+
#endif // IROHA_SHARED_MODEL_INTERFACE_MOCKS_HPP