From ebd170ff8cc1bfea5f367044de25f74480c9ddb3 Mon Sep 17 00:00:00 2001 From: Dominik Durner Date: Mon, 31 Jul 2023 10:12:06 +0200 Subject: [PATCH] Add some more datastructure tests --- README.md | 5 +- include/cloud/aws.hpp | 10 ++-- include/cloud/azure.hpp | 4 +- include/cloud/gcp.hpp | 10 ++-- include/cloud/provider.hpp | 14 +++--- include/network/http_helper.hpp | 2 +- include/network/original_message.hpp | 4 +- src/cloud/aws.cpp | 6 +-- src/cloud/azure.cpp | 2 +- src/cloud/gcp.cpp | 6 +-- src/cloud/provider.cpp | 12 ++--- src/network/http_helper.cpp | 2 +- test/unit/cloud/aws_test.cpp | 2 +- test/unit/cloud/azure_test.cpp | 2 +- test/unit/cloud/gcp_test.cpp | 2 +- test/unit/utils/ring_buffer_test.cpp | 67 ++++++++++++++++++++++++++ test/unit/utils/unordered_map_test.cpp | 55 +++++++++++++++++++++ 17 files changed, 164 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 46c6ed1..f5c5080 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ In this repository, we present AnyBlob. AnyBlob is a universal download manager that allows to retrieve and upload objects to different cloud object stores. Our download manager uses less CPU resources than cloud-vendor provided libraries while retaining maximum throughput performance. AnyBlob leverages IO\_uring for superior performance per core. -For experimental results, please visit our research paper at []. +For experimental results, please visit our research paper at [PVLDB 16](https://www.vldb.org/pvldb/vol16/p2734-durner.pdf). ## Building AnyBlob @@ -42,9 +42,10 @@ For coverage testing you can simply `make coverage` and open the coverage report ## Cite this work -If you are using AnyBlob in our scientific work, please cite: +If you are using AnyBlob in your scientific work, please cite: ``` Exploiting Cloud Object Storage for High-Performance Analytics Dominik Durner, Viktor Leis, and Thomas Neumann +PVLDB 16, 11 (2023), 49th International Conference on Very Large Data Bases ``` diff --git a/include/cloud/aws.hpp b/include/cloud/aws.hpp index 4d7b159..943f48e 100644 --- a/include/cloud/aws.hpp +++ b/include/cloud/aws.hpp @@ -111,9 +111,9 @@ class AWS : public Provider { /// Builds the http request for downloading a blob or listing the directory [[nodiscard]] std::unique_ptr> getRequest(const std::string& filePath, const std::pair& range) const override; /// Builds the http request for putting objects without the object data itself - [[nodiscard]] std::unique_ptr> putRequestGeneric(const std::string& filePath, const std::string_view object, uint16_t part, const std::string_view uploadId) const override; + [[nodiscard]] std::unique_ptr> putRequestGeneric(const std::string& filePath, std::string_view object, uint16_t part, std::string_view uploadId) const override; /// Builds the http request for putting objects without the object data itself - [[nodiscard]] std::unique_ptr> putRequest(const std::string& filePath, const std::string_view object) const override { + [[nodiscard]] std::unique_ptr> putRequest(const std::string& filePath, std::string_view object) const override { return putRequestGeneric(filePath, object, 0, ""); } // Builds the http request for deleting an objects @@ -121,11 +121,11 @@ class AWS : public Provider { return deleteRequestGeneric(filePath, ""); } /// Builds the http request for deleting objects - [[nodiscard]] std::unique_ptr> deleteRequestGeneric(const std::string& filePath, const std::string_view uploadId) const override; + [[nodiscard]] std::unique_ptr> deleteRequestGeneric(const std::string& filePath, std::string_view uploadId) const override; /// Builds the http request for creating multipart put objects [[nodiscard]] std::unique_ptr> createMultiPartRequest(const std::string& filePath) const override; /// Builds the http request for completing multipart put objects - [[nodiscard]] std::unique_ptr> completeMultiPartRequest(const std::string& filePath, const std::string_view uploadId, const std::vector& etags) const override; + [[nodiscard]] std::unique_ptr> completeMultiPartRequest(const std::string& filePath, std::string_view uploadId, const std::vector& etags) const override; /// Get the address of the server [[nodiscard]] std::string getAddress() const override; @@ -135,7 +135,7 @@ class AWS : public Provider { /// Builds the info http request [[nodiscard]] static std::unique_ptr> downloadInstanceInfo(const std::string& info = "instance-type"); /// Get the IAM address - [[nodiscard]] static constexpr const char* getIAMAddress() { return "169.254.169.254"; } + [[nodiscard]] static constexpr std::string_view getIAMAddress() { return "169.254.169.254"; } /// Get the port of the IAM server [[nodiscard]] static constexpr uint32_t getIAMPort() { return 80; } diff --git a/include/cloud/azure.hpp b/include/cloud/azure.hpp index 63c7ec2..e2bbbb5 100644 --- a/include/cloud/azure.hpp +++ b/include/cloud/azure.hpp @@ -77,7 +77,7 @@ class Azure : public Provider { /// Builds the http request for downloading a blob or listing the directory [[nodiscard]] std::unique_ptr> getRequest(const std::string& filePath, const std::pair& range) const override; /// Builds the http request for putting objects without the object data itself - [[nodiscard]] std::unique_ptr> putRequest(const std::string& filePath, const std::string_view object) const override; + [[nodiscard]] std::unique_ptr> putRequest(const std::string& filePath, std::string_view object) const override; // Builds the http request for deleting an objects [[nodiscard]] std::unique_ptr> deleteRequest(const std::string& filePath) const override; @@ -89,7 +89,7 @@ class Azure : public Provider { /// Builds the info http request [[nodiscard]] static std::unique_ptr> downloadInstanceInfo(); /// Get the IAM address - [[nodiscard]] static constexpr const char* getIAMAddress() { + [[nodiscard]] static constexpr std::string_view getIAMAddress() { return "169.254.169.254"; } /// Get the port of the IAM server diff --git a/include/cloud/gcp.hpp b/include/cloud/gcp.hpp index d7c237d..b81392b 100644 --- a/include/cloud/gcp.hpp +++ b/include/cloud/gcp.hpp @@ -79,9 +79,9 @@ class GCP : public Provider { /// Builds the http request for downloading a blob or listing the directory [[nodiscard]] std::unique_ptr> getRequest(const std::string& filePath, const std::pair& range) const override; /// Builds the http request for putting objects without the object data itself - [[nodiscard]] std::unique_ptr> putRequestGeneric(const std::string& filePath, const std::string_view object, uint16_t part, const std::string_view uploadId) const override; + [[nodiscard]] std::unique_ptr> putRequestGeneric(const std::string& filePath, std::string_view object, uint16_t part, std::string_view uploadId) const override; /// Builds the http request for putting objects without the object data itself - [[nodiscard]] std::unique_ptr> putRequest(const std::string& filePath, const std::string_view object) const override { + [[nodiscard]] std::unique_ptr> putRequest(const std::string& filePath, std::string_view object) const override { return putRequestGeneric(filePath, object, 0, ""); } // Builds the http request for deleting an objects @@ -89,11 +89,11 @@ class GCP : public Provider { return deleteRequestGeneric(filePath, ""); } /// Builds the http request for deleting objects - [[nodiscard]] std::unique_ptr> deleteRequestGeneric(const std::string& filePath, const std::string_view uploadId) const override; + [[nodiscard]] std::unique_ptr> deleteRequestGeneric(const std::string& filePath, std::string_view uploadId) const override; /// Builds the http request for creating multipart put objects [[nodiscard]] std::unique_ptr> createMultiPartRequest(const std::string& filePath) const override; /// Builds the http request for completing multipart put objects - [[nodiscard]] std::unique_ptr> completeMultiPartRequest(const std::string& filePath, const std::string_view uploadId, const std::vector& etags) const override; + [[nodiscard]] std::unique_ptr> completeMultiPartRequest(const std::string& filePath, std::string_view uploadId, const std::vector& etags) const override; /// Get the address of the server [[nodiscard]] std::string getAddress() const override; @@ -103,7 +103,7 @@ class GCP : public Provider { /// Builds the info http request [[nodiscard]] static std::unique_ptr> downloadInstanceInfo(const std::string& info = "machine-type"); /// Get the IAM address - [[nodiscard]] static constexpr const char* getIAMAddress() { + [[nodiscard]] static constexpr std::string_view getIAMAddress() { return "169.254.169.254"; } /// Get the port of the IAM server diff --git a/include/cloud/provider.hpp b/include/cloud/provider.hpp index a4e3683..17271d8 100644 --- a/include/cloud/provider.hpp +++ b/include/cloud/provider.hpp @@ -73,7 +73,7 @@ class Provider { /// Builds the http request for downloading a blob or listing a directory [[nodiscard]] virtual std::unique_ptr> getRequest(const std::string& filePath, const std::pair& range) const = 0; /// Builds the http request for putting an object without the actual data (header only according to the data and length provided) - [[nodiscard]] virtual std::unique_ptr> putRequest(const std::string& filePath, const std::string_view object) const = 0; + [[nodiscard]] virtual std::unique_ptr> putRequest(const std::string& filePath, std::string_view object) const = 0; /// Builds the http request for deleting an object [[nodiscard]] virtual std::unique_ptr> deleteRequest(const std::string& filePath) const = 0; /// Get the address of the server @@ -84,13 +84,13 @@ class Provider { /// Is multipart upload supported, if size > 0? [[nodiscard]] virtual uint64_t multipartUploadSize() const { return 0; } /// Builds the http request for putting multipart objects without the object data itself - [[nodiscard]] virtual std::unique_ptr> putRequestGeneric(const std::string& /*filePath*/, const std::string_view /*object*/, uint16_t /*part*/, const std::string_view /*uploadId*/) const; + [[nodiscard]] virtual std::unique_ptr> putRequestGeneric(const std::string& /*filePath*/, std::string_view /*object*/, uint16_t /*part*/, std::string_view /*uploadId*/) const; /// Builds the http request for deleting multipart aborted objects - [[nodiscard]] virtual std::unique_ptr> deleteRequestGeneric(const std::string& /*filePath*/, const std::string_view /*uploadId*/) const; + [[nodiscard]] virtual std::unique_ptr> deleteRequestGeneric(const std::string& /*filePath*/, std::string_view /*uploadId*/) const; /// Builds the http request for creating multipart put objects [[nodiscard]] virtual std::unique_ptr> createMultiPartRequest(const std::string& /*filePath*/) const; /// Builds the http request for completing multipart put objects - [[nodiscard]] virtual std::unique_ptr> completeMultiPartRequest(const std::string& /*filePath*/, const std::string_view /*uploadId*/, const std::vector& /*etags*/) const; + [[nodiscard]] virtual std::unique_ptr> completeMultiPartRequest(const std::string& /*filePath*/, std::string_view /*uploadId*/, const std::vector& /*etags*/) const; /// Initialize secret virtual void initSecret(network::TaskedSendReceiver& /*sendReceiver*/) {} @@ -101,7 +101,7 @@ class Provider { /// Gets the cloud provider type [[nodiscard]] CloudService getType() { return _type; } /// Is it a remote file? - [[nodiscard]] static bool isRemoteFile(const std::string_view fileName) noexcept; + [[nodiscard]] static bool isRemoteFile(std::string_view fileName) noexcept; /// Get the path of the parent dir without the remote info [[nodiscard]] static std::string getRemoteParentDirectory(std::string fileName) noexcept; /// Get a region and bucket name @@ -109,9 +109,9 @@ class Provider { /// Get the key from a keyFile [[nodiscard]] static std::string getKey(const std::string& keyFile); /// Get the etag from the upload header - [[nodiscard]] static std::string getETag(const std::string_view header); + [[nodiscard]] static std::string getETag(std::string_view header); /// Get the upload id from the multipart request body - [[nodiscard]] static std::string getUploadId(const std::string_view body); + [[nodiscard]] static std::string getUploadId(std::string_view body); /// Create a provider (keyId is access email for GCP/Azure) [[nodiscard]] static std::unique_ptr makeProvider(const std::string& filepath, bool https = true, const std::string& keyId = "", const std::string& keyFile = "", network::TaskedSendReceiver* sendReceiver = nullptr); diff --git a/include/network/http_helper.hpp b/include/network/http_helper.hpp index 0ea9144..1e0bb04 100644 --- a/include/network/http_helper.hpp +++ b/include/network/http_helper.hpp @@ -47,7 +47,7 @@ class HTTPHelper { private: /// Detect the protocol - [[nodiscard]] static Info detect(const std::string_view s); + [[nodiscard]] static Info detect(std::string_view s); public: /// Retrieve the content without http meta info, note that this changes data diff --git a/include/network/original_message.hpp b/include/network/original_message.hpp index d0d32ec..b3d3662 100644 --- a/include/network/original_message.hpp +++ b/include/network/original_message.hpp @@ -42,7 +42,7 @@ struct OriginalMessage { uint64_t putLength; /// The constructor - OriginalMessage(std::unique_ptr> message, std::string hostname, uint32_t port, uint8_t* receiveBuffer = nullptr, uint64_t bufferSize = 0, uint64_t traceId = 0) : message(std::move(message)), result(receiveBuffer, bufferSize), hostname(hostname), port(port), traceId(traceId), putData(nullptr), putLength() {} + OriginalMessage(std::unique_ptr> message, std::string_view hostname, uint32_t port, uint8_t* receiveBuffer = nullptr, uint64_t bufferSize = 0, uint64_t traceId = 0) : message(std::move(message)), result(receiveBuffer, bufferSize), hostname(hostname), port(port), traceId(traceId), putData(nullptr), putLength() {} /// The destructor virtual ~OriginalMessage() = default; @@ -72,7 +72,7 @@ struct OriginalCallbackMessage : public OriginalMessage { Callback callback; /// The constructor - OriginalCallbackMessage(Callback&& callback, std::unique_ptr> message, std::string hostname, uint32_t port, uint8_t* receiveBuffer = nullptr, uint64_t bufferSize = 0, uint64_t traceId = 0) : OriginalMessage(std::move(message), hostname, port, receiveBuffer, bufferSize, traceId), callback(std::forward(callback)) {} + OriginalCallbackMessage(Callback&& callback, std::unique_ptr> message, std::string_view hostname, uint32_t port, uint8_t* receiveBuffer = nullptr, uint64_t bufferSize = 0, uint64_t traceId = 0) : OriginalMessage(std::move(message), hostname, port, receiveBuffer, bufferSize, traceId), callback(std::forward(callback)) {} /// The destructor virtual ~OriginalCallbackMessage() override = default; diff --git a/src/cloud/aws.cpp b/src/cloud/aws.cpp index 6ef01f1..a198d36 100644 --- a/src/cloud/aws.cpp +++ b/src/cloud/aws.cpp @@ -228,7 +228,7 @@ unique_ptr> AWS::getRequest(const string& filePath, c return make_unique>(reinterpret_cast(httpHeader.data()), reinterpret_cast(httpHeader.data() + httpHeader.size())); } //--------------------------------------------------------------------------- -unique_ptr> AWS::putRequestGeneric(const string& filePath, const string_view object, uint16_t part, const string_view uploadId) const +unique_ptr> AWS::putRequestGeneric(const string& filePath, string_view object, uint16_t part, string_view uploadId) const // Builds the http request for putting objects without the object data itself { if (!validKeys()) @@ -270,7 +270,7 @@ unique_ptr> AWS::putRequestGeneric(const string& file return make_unique>(reinterpret_cast(httpHeader.data()), reinterpret_cast(httpHeader.data() + httpHeader.size())); } //--------------------------------------------------------------------------- -unique_ptr> AWS::deleteRequestGeneric(const string& filePath, const string_view uploadId) const +unique_ptr> AWS::deleteRequestGeneric(const string& filePath, string_view uploadId) const // Builds the http request for deleting an objects { if (!validKeys()) @@ -345,7 +345,7 @@ unique_ptr> AWS::createMultiPartRequest(const string& return make_unique>(reinterpret_cast(httpHeader.data()), reinterpret_cast(httpHeader.data() + httpHeader.size())); } //--------------------------------------------------------------------------- -unique_ptr> AWS::completeMultiPartRequest(const string& filePath, const string_view uploadId, const std::vector& etags) const +unique_ptr> AWS::completeMultiPartRequest(const string& filePath, string_view uploadId, const std::vector& etags) const // Builds the http request for completing multipart upload objects { if (!validKeys()) diff --git a/src/cloud/azure.cpp b/src/cloud/azure.cpp index 31c19d0..fec4905 100644 --- a/src/cloud/azure.cpp +++ b/src/cloud/azure.cpp @@ -121,7 +121,7 @@ unique_ptr> Azure::getRequest(const string& filePath, return make_unique>(reinterpret_cast(httpHeader.data()), reinterpret_cast(httpHeader.data() + httpHeader.size())); } //--------------------------------------------------------------------------- -unique_ptr> Azure::putRequest(const string& filePath, const string_view object) const +unique_ptr> Azure::putRequest(const string& filePath, string_view object) const // Builds the http request for putting objects without the object data itself { AzureSigner::Request request; diff --git a/src/cloud/gcp.cpp b/src/cloud/gcp.cpp index bef1819..53e59e8 100644 --- a/src/cloud/gcp.cpp +++ b/src/cloud/gcp.cpp @@ -104,7 +104,7 @@ unique_ptr> GCP::getRequest(const string& filePath, c return make_unique>(reinterpret_cast(httpHeader.data()), reinterpret_cast(httpHeader.data() + httpHeader.size())); } //--------------------------------------------------------------------------- -unique_ptr> GCP::putRequestGeneric(const string& filePath, const string_view object, uint16_t part, const string_view uploadId) const +unique_ptr> GCP::putRequestGeneric(const string& filePath, string_view object, uint16_t part, string_view uploadId) const // Builds the http request for putting objects without the object data itself { GCPSigner::Request request; @@ -139,7 +139,7 @@ unique_ptr> GCP::putRequestGeneric(const string& file return make_unique>(reinterpret_cast(httpHeader.data()), reinterpret_cast(httpHeader.data() + httpHeader.size())); } //--------------------------------------------------------------------------- -unique_ptr> GCP::deleteRequestGeneric(const string& filePath, const string_view uploadId) const +unique_ptr> GCP::deleteRequestGeneric(const string& filePath, string_view uploadId) const // Builds the http request for deleting objects { GCPSigner::Request request; @@ -196,7 +196,7 @@ unique_ptr> GCP::createMultiPartRequest(const string& return make_unique>(reinterpret_cast(httpHeader.data()), reinterpret_cast(httpHeader.data() + httpHeader.size())); } //--------------------------------------------------------------------------- -unique_ptr> GCP::completeMultiPartRequest(const string& filePath, const string_view uploadId, const std::vector& etags) const +unique_ptr> GCP::completeMultiPartRequest(const string& filePath, string_view uploadId, const std::vector& etags) const // Builds the http request for completing multipart upload objects { string content = "\n"; diff --git a/src/cloud/provider.cpp b/src/cloud/provider.cpp index fe14ac3..7cbc06f 100644 --- a/src/cloud/provider.cpp +++ b/src/cloud/provider.cpp @@ -40,7 +40,7 @@ string Provider::getRemoteParentDirectory(string fileName) noexcept { return fileName.substr(0, pos + 1); } //--------------------------------------------------------------------------- -bool Provider::isRemoteFile(const string_view fileName) noexcept +bool Provider::isRemoteFile(string_view fileName) noexcept // Is it a remote file? { for (auto i = 0u; i < remoteFileCount; i++) @@ -93,7 +93,7 @@ string Provider::getKey(const string& keyFile) return string((istreambuf_iterator(ifs)), (istreambuf_iterator())); } //--------------------------------------------------------------------------- -string Provider::getETag(const string_view header) +string Provider::getETag(string_view header) // Get the etag from the upload header { string needle = "ETag: \""; @@ -105,7 +105,7 @@ string Provider::getETag(const string_view header) return string(header.substr(pos, end - pos)); } //--------------------------------------------------------------------------- -string Provider::getUploadId(const string_view body) +string Provider::getUploadId(string_view body) // Get the upload id from the multipart request body { string needle = ""; @@ -117,13 +117,13 @@ string Provider::getUploadId(const string_view body) return string(body.substr(pos, end - pos)); } //--------------------------------------------------------------------------- -unique_ptr> Provider::putRequestGeneric(const string& /*filePath*/, const string_view /*object*/, uint16_t /*part*/, const string_view /*uploadId*/) const +unique_ptr> Provider::putRequestGeneric(const string& /*filePath*/, string_view /*object*/, uint16_t /*part*/, string_view /*uploadId*/) const // Builds the http request for putting multipart objects without the object data itself { return nullptr; } //--------------------------------------------------------------------------- -unique_ptr> Provider::deleteRequestGeneric(const string& /*filePath*/, const string_view /*uploadId*/) const +unique_ptr> Provider::deleteRequestGeneric(const string& /*filePath*/, string_view /*uploadId*/) const // Builds the http request for delete multipart objects { return nullptr; @@ -135,7 +135,7 @@ unique_ptr> Provider::createMultiPartRequest(const st return nullptr; } //--------------------------------------------------------------------------- -unique_ptr> Provider::completeMultiPartRequest(const string& /*filePath*/, const string_view /*uploadId*/, const vector& /*etags*/) const +unique_ptr> Provider::completeMultiPartRequest(const string& /*filePath*/, string_view /*uploadId*/, const vector& /*etags*/) const // Builds the http request for completing multipart put objects { return nullptr; diff --git a/src/network/http_helper.cpp b/src/network/http_helper.cpp index 864e63c..d588401 100644 --- a/src/network/http_helper.cpp +++ b/src/network/http_helper.cpp @@ -16,7 +16,7 @@ namespace network { //--------------------------------------------------------------------------- using namespace std; //--------------------------------------------------------------------------- -HTTPHelper::Info HTTPHelper::detect(const string_view header) +HTTPHelper::Info HTTPHelper::detect(string_view header) // Detect the protocol { auto info = Info{0, 0, Protocol::Unknown, Encoding::Unknown}; diff --git a/test/unit/cloud/aws_test.cpp b/test/unit/cloud/aws_test.cpp index 6dd1531..9e5857c 100644 --- a/test/unit/cloud/aws_test.cpp +++ b/test/unit/cloud/aws_test.cpp @@ -28,7 +28,7 @@ class AWSTester { auto provider = Provider::makeProvider("s3://test:test/", false, "test", "test"); AWS& aws = *static_cast(provider.get()); - REQUIRE(!strcmp(aws.getIAMAddress(), "169.254.169.254")); + REQUIRE(!aws.getIAMAddress().compare("169.254.169.254")); REQUIRE(aws.getIAMPort() == 80); auto dv = aws.downloadInstanceInfo(); diff --git a/test/unit/cloud/azure_test.cpp b/test/unit/cloud/azure_test.cpp index 1a059e4..31e5c50 100644 --- a/test/unit/cloud/azure_test.cpp +++ b/test/unit/cloud/azure_test.cpp @@ -29,7 +29,7 @@ class AzureTester { auto provider = Provider::makeProvider("azure://test/", false, "test", ""); Azure& azure = *static_cast(provider.get()); - REQUIRE(!strcmp(azure.getIAMAddress(), "169.254.169.254")); + REQUIRE(!azure.getIAMAddress().compare("169.254.169.254")); REQUIRE(azure.getIAMPort() == 80); auto dv = azure.downloadInstanceInfo(); diff --git a/test/unit/cloud/gcp_test.cpp b/test/unit/cloud/gcp_test.cpp index 08e510c..fb45185 100644 --- a/test/unit/cloud/gcp_test.cpp +++ b/test/unit/cloud/gcp_test.cpp @@ -69,7 +69,7 @@ class GCPTester { auto provider = Provider::makeProvider("gcp://test:test/", false, "test@test.com", key); GCP& gcp = *static_cast(provider.get()); - REQUIRE(!strcmp(gcp.getIAMAddress(), "169.254.169.254")); + REQUIRE(!gcp.getIAMAddress().compare("169.254.169.254")); REQUIRE(gcp.getIAMPort() == 80); auto dv = gcp.downloadInstanceInfo(); diff --git a/test/unit/utils/ring_buffer_test.cpp b/test/unit/utils/ring_buffer_test.cpp index 1ade0d9..d199b22 100644 --- a/test/unit/utils/ring_buffer_test.cpp +++ b/test/unit/utils/ring_buffer_test.cpp @@ -1,5 +1,6 @@ #include "utils/ring_buffer.hpp" #include "catch2/single_include/catch2/catch.hpp" +#include //--------------------------------------------------------------------------- // AnyBlob - Universal Cloud Object Storage Library // Dominik Durner, 2022 @@ -27,6 +28,72 @@ TEST_CASE("ring_buffer") { REQUIRE(rb.empty()); } //--------------------------------------------------------------------------- +TEST_CASE("single_threaded_insert_multi_threaded_consume") { + RingBuffer rb(1000); + for (int i = 0; i < 1000; i++) { + REQUIRE(rb.insert(i) != ~0ull); + } + std::vector threads; + for (int i = 0; i < 10; i++) { + threads.push_back(std::thread([&] { + for (int j = 0; j < 100; j++) { + REQUIRE(rb.consume().value() < 1000ul); + } + })); + } + for (auto& th : threads) { + th.join(); + } + REQUIRE(!rb.consume().has_value()); +} +//--------------------------------------------------------------------------- +TEST_CASE("multi_threaded_ring_buffer") { + RingBuffer rb(1000); + std::vector threads; + for (int i = 0; i < 10; i++) { + threads.push_back(std::thread([&] { + for (int j = 0; j < 100; j++) { + REQUIRE(rb.insert(j) != ~0ull); + } + })); + } + for (auto& th : threads) { + th.join(); + } + for (int i = 0; i < 1000; i++) { + REQUIRE(rb.consume().value() < 100ul); + } + REQUIRE(!rb.consume().has_value()); +} +//--------------------------------------------------------------------------- +TEST_CASE("multi_threaded_ring_buffer_multi_threaded_consume") { + RingBuffer rb(1000); + std::vector threads; + for (int i = 0; i < 10; i++) { + threads.push_back(std::thread([&] { + for (int j = 0; j < 100; j++) { + REQUIRE(rb.insert(j) != ~0ull); + } + })); + } + for (auto& th : threads) { + th.join(); + } + // Consume multi-threaded + threads.clear(); + for (int i = 0; i < 10; i++) { + threads.push_back(std::thread([&] { + for (int j = 0; j < 100; j++) { + REQUIRE(rb.consume().value() < 100ul); + } + })); + } + for (auto& th : threads) { + th.join(); + } + REQUIRE(!rb.consume().has_value()); +} +//--------------------------------------------------------------------------- } // namespace test } // namespace utils } // namespace anyblob diff --git a/test/unit/utils/unordered_map_test.cpp b/test/unit/utils/unordered_map_test.cpp index ca06dd3..6e753c5 100644 --- a/test/unit/utils/unordered_map_test.cpp +++ b/test/unit/utils/unordered_map_test.cpp @@ -1,5 +1,6 @@ #include "utils/unordered_map.hpp" #include "catch2/single_include/catch2/catch.hpp" +#include //--------------------------------------------------------------------------- // AnyBlob - Universal Cloud Object Storage Library // Dominik Durner, 2022 @@ -28,6 +29,60 @@ TEST_CASE("unordered_map") { REQUIRE(ht.size() == 1); } //--------------------------------------------------------------------------- +TEST_CASE("unordered_map_multi_threaded") { + // Insert 1000 elements in 10 threads + UnorderedMap ht(128); + std::thread t[10]; + for (int i = 0; i < 10; i++) { + t[i] = std::thread([&ht, i]() { + for (int j = 0; j < 10; j++) { + std::ignore = ht.insert(i * 10 + j, i * 10 + j); + } + }); + } + for (int i = 0; i < 10; i++) { + t[i].join(); + } + // Find the elements multu-threaded + for (int i = 0; i < 10; i++) { + t[i] = std::thread([&ht, i]() { + for (int j = 0; j < 10; j++) { + REQUIRE(ht.find(i * 10 + j) != ht.end()); + } + }); + } + for (int i = 0; i < 10; i++) { + t[i].join(); + } +} +//--------------------------------------------------------------------------- +TEST_CASE("unordered_map_multi_threaded_delete") { + UnorderedMap ht(128); + // Insert 1000 elements in 10 threads + std::thread t[10]; + for (int i = 0; i < 10; i++) { + t[i] = std::thread([&ht, i]() { + for (int j = 0; j < 10; j++) { + std::ignore = ht.insert(i * 10 + j, i * 10 + j); + } + }); + } + for (int i = 0; i < 10; i++) { + t[i].join(); + } + // Delete the elements multi-threaded + for (int i = 0; i < 10; i++) { + t[i] = std::thread([&ht, i]() { + for (int j = 0; j < 10; j++) { + REQUIRE(ht.erase(i * 10 + j)); + } + }); + } + for (int i = 0; i < 10; i++) { + t[i].join(); + } +} +//--------------------------------------------------------------------------- } // namespace test } // namespace utils } // namespace anyblob