From 16e487b2ca217f636e4496adc8dd38472a9d8547 Mon Sep 17 00:00:00 2001 From: Dominik Durner Date: Mon, 15 May 2023 23:59:00 +0200 Subject: [PATCH] Added Delete Blob Methods --- include/cloud/aws.hpp | 2 ++ include/cloud/azure.hpp | 2 ++ include/cloud/gcp.hpp | 2 ++ include/cloud/ibm.hpp | 2 +- include/cloud/minio.hpp | 2 +- include/cloud/oracle.hpp | 2 +- include/cloud/provider.hpp | 4 +++ include/network/delete_transaction.hpp | 43 ++++++++++++++++++++++++++ include/network/http_helper.hpp | 3 +- include/utils/ring_buffer.hpp | 4 +-- src/cloud/aws.cpp | 37 ++++++++++++++++++++-- src/cloud/azure.cpp | 24 ++++++++++++++ src/cloud/gcp.cpp | 25 +++++++++++++++ src/network/http_helper.cpp | 7 ++++- test/integration/minio.cpp | 17 ++++++++++ test/unit/cloud/aws_test.cpp | 6 ++++ test/unit/cloud/azure_test.cpp | 6 ++++ test/unit/cloud/gcp_test.cpp | 4 +++ 18 files changed, 183 insertions(+), 9 deletions(-) create mode 100644 include/network/delete_transaction.hpp diff --git a/include/cloud/aws.hpp b/include/cloud/aws.hpp index 8d895d7..f42fe0b 100644 --- a/include/cloud/aws.hpp +++ b/include/cloud/aws.hpp @@ -107,6 +107,8 @@ class AWS : public Provider { [[nodiscard]] std::unique_ptr> getRequest(const std::string& filePath, const std::pair& range) const override; /// Builds the http request for putting objects without the object data itself [[nodiscard]] std::unique_ptr> putRequest(const std::string& filePath, const std::string_view object) const override; + // Builds the http request for deleting an objects + [[nodiscard]] std::unique_ptr> deleteRequest(const std::string& filePath) const override; /// Get the address of the server [[nodiscard]] std::string getAddress() const override; diff --git a/include/cloud/azure.hpp b/include/cloud/azure.hpp index ca4e5dc..63c7ec2 100644 --- a/include/cloud/azure.hpp +++ b/include/cloud/azure.hpp @@ -78,6 +78,8 @@ class Azure : public Provider { [[nodiscard]] std::unique_ptr> getRequest(const std::string& filePath, const std::pair& range) const override; /// Builds the http request for putting objects without the object data itself [[nodiscard]] std::unique_ptr> putRequest(const std::string& filePath, const std::string_view object) const override; + // Builds the http request for deleting an objects + [[nodiscard]] std::unique_ptr> deleteRequest(const std::string& filePath) const override; /// Get the address of the server [[nodiscard]] std::string getAddress() const override; diff --git a/include/cloud/gcp.hpp b/include/cloud/gcp.hpp index 1105bbb..fe8e520 100644 --- a/include/cloud/gcp.hpp +++ b/include/cloud/gcp.hpp @@ -76,6 +76,8 @@ class GCP : public Provider { [[nodiscard]] std::unique_ptr> getRequest(const std::string& filePath, const std::pair& range) const override; /// Builds the http request for putting objects without the object data itself [[nodiscard]] std::unique_ptr> putRequest(const std::string& filePath, const std::string_view object) const override; + // Builds the http request for deleting an objects + [[nodiscard]] std::unique_ptr> deleteRequest(const std::string& filePath) const override; /// Get the address of the server [[nodiscard]] std::string getAddress() const override; diff --git a/include/cloud/ibm.hpp b/include/cloud/ibm.hpp index 264e793..c9d340a 100644 --- a/include/cloud/ibm.hpp +++ b/include/cloud/ibm.hpp @@ -33,7 +33,7 @@ class IBM : public AWS { /// Get the address of the server [[nodiscard]] std::string getAddress() const override; /// Get the instance details - Provider::Instance getInstanceDetails(network::TaskedSendReceiver& sendReceiver) override; + [[nodiscard]] Provider::Instance getInstanceDetails(network::TaskedSendReceiver& sendReceiver) override; friend Provider; }; diff --git a/include/cloud/minio.hpp b/include/cloud/minio.hpp index bd9dfa9..b804b1f 100644 --- a/include/cloud/minio.hpp +++ b/include/cloud/minio.hpp @@ -28,7 +28,7 @@ class MinIO : public AWS { /// Get the address of the server [[nodiscard]] std::string getAddress() const override; /// Get the instance details - Provider::Instance getInstanceDetails(network::TaskedSendReceiver& sendReceiver) override; + [[nodiscard]] Provider::Instance getInstanceDetails(network::TaskedSendReceiver& sendReceiver) override; friend Provider; }; diff --git a/include/cloud/oracle.hpp b/include/cloud/oracle.hpp index 35a07a1..9a8dfad 100644 --- a/include/cloud/oracle.hpp +++ b/include/cloud/oracle.hpp @@ -28,7 +28,7 @@ class Oracle : public AWS { /// Get the address of the server [[nodiscard]] std::string getAddress() const override; /// Get the instance details - Provider::Instance getInstanceDetails(network::TaskedSendReceiver& sendReceiver) override; + [[nodiscard]] Provider::Instance getInstanceDetails(network::TaskedSendReceiver& sendReceiver) override; friend Provider; }; diff --git a/include/cloud/provider.hpp b/include/cloud/provider.hpp index 7d46d62..a618c3a 100644 --- a/include/cloud/provider.hpp +++ b/include/cloud/provider.hpp @@ -16,6 +16,7 @@ class TaskedSendReceiver; class Transaction; class GetTransaction; class PutTransaction; +class DeleteTransaction; struct OriginalMessage; }; // namespace network namespace utils { @@ -76,6 +77,8 @@ class Provider { [[nodiscard]] virtual std::unique_ptr> getRequest(const std::string& filePath, const std::pair& range) const = 0; /// Builds the http request for putting an object without the actual data (header only according to the data and length provided) [[nodiscard]] virtual std::unique_ptr> putRequest(const std::string& filePath, const std::string_view object) const = 0; + /// Builds the http request for deleting an object + [[nodiscard]] virtual std::unique_ptr> deleteRequest(const std::string& filePath) const = 0; /// Get the address of the server [[nodiscard]] virtual std::string getAddress() const = 0; /// Get the port of the server @@ -108,6 +111,7 @@ class Provider { friend network::Transaction; friend network::GetTransaction; friend network::PutTransaction; + friend network::DeleteTransaction; }; //--------------------------------------------------------------------------- } // namespace cloud diff --git a/include/network/delete_transaction.hpp b/include/network/delete_transaction.hpp new file mode 100644 index 0000000..fcf8ef9 --- /dev/null +++ b/include/network/delete_transaction.hpp @@ -0,0 +1,43 @@ +#pragma once +#include "cloud/provider.hpp" +#include "network/original_message.hpp" +#include "network/transaction.hpp" +#include +//--------------------------------------------------------------------------- +// AnyBlob - Universal Cloud Object Storage Library +// Dominik Durner, 2023 +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// SPDX-License-Identifier: MPL-2.0 +//--------------------------------------------------------------------------- +namespace anyblob { +//--------------------------------------------------------------------------- +namespace network { +//--------------------------------------------------------------------------- +/// Specialization of a delete request +class DeleteTransaction : public Transaction { + public: + /// The default constructor + DeleteTransaction() = default; + /// The explicit constructor + explicit DeleteTransaction(const cloud::Provider* provider) : Transaction(provider) {} + + /// Build a new delete request for synchronous calls + inline void addRequest(const std::string& remotePath, uint8_t* result = nullptr, uint64_t capacity = 0, uint64_t traceId = 0) { + assert(provider); + auto originalMsg = std::make_unique(provider->deleteRequest(remotePath), provider->getAddress(), provider->getPort(), result, capacity, traceId); + messages.push_back(std::move(originalMsg)); + } + + /// Build a new delete request with callback + template + inline void addRequest(Callback&& callback, const std::string& remotePath, uint8_t* result = nullptr, uint64_t capacity = 0, uint64_t traceId = 0) { + assert(provider); + auto originalMsg = std::make_unique>(std::forward(callback), provider->deleteRequest(remotePath), provider->getAddress(), provider->getPort(), result, capacity, traceId); + messages.push_back(std::move(originalMsg)); + } +}; +//--------------------------------------------------------------------------- +} // namespace network +} // namespace anyblob diff --git a/include/network/http_helper.hpp b/include/network/http_helper.hpp index 8f003d1..0ea9144 100644 --- a/include/network/http_helper.hpp +++ b/include/network/http_helper.hpp @@ -23,7 +23,8 @@ class HTTPHelper { HTTP_1_0_OK, HTTP_1_1_Partial, HTTP_1_1_OK, - HTTP_1_1_Created + HTTP_1_1_Created, + HTTP_1_1_No_Content }; /// The encoding diff --git a/include/utils/ring_buffer.hpp b/include/utils/ring_buffer.hpp index 933bb9f..a69eb90 100644 --- a/include/utils/ring_buffer.hpp +++ b/include/utils/ring_buffer.hpp @@ -44,7 +44,7 @@ class RingBuffer { /// The buffer size uint64_t _size; /// The insert cache-aligned struct - struct alignas(64) { + struct { /// The current insert head std::atomic pending; /// The current insert counter such that we can safely insert before seen takes element @@ -53,7 +53,7 @@ class RingBuffer { SpinMutex mutex; } _insert; /// The seen cache-alignedstruct - struct alignas(64) { + struct { /// The seen head std::atomic pending; /// The seen counter diff --git a/src/cloud/aws.cpp b/src/cloud/aws.cpp index b25b3df..aa3183c 100644 --- a/src/cloud/aws.cpp +++ b/src/cloud/aws.cpp @@ -66,9 +66,8 @@ Provider::Instance AWS::getInstanceDetails(network::TaskedSendReceiver& sendRece return instance; return AWSInstance{string(s), 0, 0, ""}; - } else { - return AWSInstance{"minio", 0, 0, "10"}; } + return AWSInstance{"aws", 0, 0, ""}; } //--------------------------------------------------------------------------- string AWS::getInstanceRegion(network::TaskedSendReceiver& sendReceiver) @@ -264,6 +263,40 @@ unique_ptr> AWS::putRequest(const string& filePath, c return make_unique>(reinterpret_cast(httpHeader.data()), reinterpret_cast(httpHeader.data() + httpHeader.size())); } //--------------------------------------------------------------------------- +unique_ptr> AWS::deleteRequest(const string& filePath) const +// Builds the http request for deleting an objects +{ + if (!validKeys()) + return nullptr; + + AWSSigner::Request request; + request.method = "DELETE"; + request.type = "HTTP/1.1"; + + // If an endpoint is defined, we use the path-style request. The default is the usage of virtual hosted-style requests. + if (_settings.endpoint.empty()) + request.path = "/" + filePath; + else + request.path = "/" + _settings.bucket + "/" + filePath; + request.bodyData = nullptr; + request.bodyLength = 0; + request.headers.emplace("Host", getAddress()); + request.headers.emplace("x-amz-date", testEnviornment ? fakeAMZTimestamp : buildAMZTimestamp()); + request.headers.emplace("x-amz-request-payer", "requester"); + if (!_secret->sessionToken.empty()) + request.headers.emplace("x-amz-security-token", _secret->sessionToken); + + auto canonical = AWSSigner::createCanonicalRequest(request); + + AWSSigner::StringToSign stringToSign = {.request = request, .requestSHA = canonical.second, .region = _settings.region, .service = "s3"}; + const auto uri = AWSSigner::createSignedRequest(_secret->keyId, _secret->secret, stringToSign); + auto httpHeader = request.method + " " + uri + " " + request.type + "\r\n"; + for (auto& h : request.headers) + httpHeader += h.first + ": " + h.second + "\r\n"; + httpHeader += "\r\n"; + return make_unique>(reinterpret_cast(httpHeader.data()), reinterpret_cast(httpHeader.data() + httpHeader.size())); +} +//--------------------------------------------------------------------------- uint32_t AWS::getPort() const // Gets the port of AWS S3 on http { diff --git a/src/cloud/azure.cpp b/src/cloud/azure.cpp index 52116f1..31c19d0 100644 --- a/src/cloud/azure.cpp +++ b/src/cloud/azure.cpp @@ -147,6 +147,30 @@ unique_ptr> Azure::putRequest(const string& filePath, return make_unique>(reinterpret_cast(httpHeader.data()), reinterpret_cast(httpHeader.data() + httpHeader.size())); } //--------------------------------------------------------------------------- +unique_ptr> Azure::deleteRequest(const string& filePath) const +// Builds the http request for deleting objects +{ + AzureSigner::Request request; + request.method = "DELETE"; + request.type = "HTTP/1.1"; + request.path = "/" + _settings.container + "/" + filePath; + request.bodyData = nullptr; + request.bodyLength = 0; + + auto date = testEnviornment ? fakeXMSTimestamp : buildXMSTimestamp(); + request.headers.emplace("x-ms-date", date); + request.headers.emplace("Host", getAddress()); + + request.path = AzureSigner::createSignedRequest(_secret->accountName, _secret->privateKey, request); + + auto httpHeader = request.method + " " + request.path + " " + request.type + "\r\n"; + for (auto& h : request.headers) + httpHeader += h.first + ": " + h.second + "\r\n"; + httpHeader += "\r\n"; + + return make_unique>(reinterpret_cast(httpHeader.data()), reinterpret_cast(httpHeader.data() + httpHeader.size())); +} +//--------------------------------------------------------------------------- uint32_t Azure::getPort() const // Gets the port of Azure on http { diff --git a/src/cloud/gcp.cpp b/src/cloud/gcp.cpp index 90f6d0f..35d4e5e 100644 --- a/src/cloud/gcp.cpp +++ b/src/cloud/gcp.cpp @@ -132,6 +132,31 @@ unique_ptr> GCP::putRequest(const string& filePath, c return make_unique>(reinterpret_cast(httpHeader.data()), reinterpret_cast(httpHeader.data() + httpHeader.size())); } //--------------------------------------------------------------------------- +unique_ptr> GCP::deleteRequest(const string& filePath) const +// Builds the http request for deleting objects +{ + GCPSigner::Request request; + request.method = "DELETE"; + request.type = "HTTP/1.1"; + request.path = "/" + filePath; + request.bodyData = nullptr; + request.bodyLength = 0; + + auto date = testEnviornment ? fakeAMZTimestamp : buildAMZTimestamp(); + request.queries.emplace("X-Goog-Date", date); + request.headers.emplace("Host", getAddress()); + + GCPSigner::StringToSign stringToSign = {.region = _settings.region, .service = "storage"}; + request.path = GCPSigner::createSignedRequest(_secret->serviceAccountEmail, _secret->privateKey, request, stringToSign); + + auto httpHeader = request.method + " " + request.path + " " + request.type + "\r\n"; + for (auto& h : request.headers) + httpHeader += h.first + ": " + h.second + "\r\n"; + httpHeader += "\r\n"; + + return make_unique>(reinterpret_cast(httpHeader.data()), reinterpret_cast(httpHeader.data() + httpHeader.size())); +} +//--------------------------------------------------------------------------- uint32_t GCP::getPort() const // Gets the port of GCP on http { diff --git a/src/network/http_helper.cpp b/src/network/http_helper.cpp index 3aa6448..864e63c 100644 --- a/src/network/http_helper.cpp +++ b/src/network/http_helper.cpp @@ -25,6 +25,7 @@ HTTPHelper::Info HTTPHelper::detect(const string_view header) string str_http_1_1 = "HTTP/1.1 200 OK"; string str_http_1_1_partial = "HTTP/1.1 206 Partial Content"; string str_http_1_1_created = "HTTP/1.1 201 Created"; + string str_http_1_1_no_conent = "HTTP/1.1 204 No Content"; if (header.length() >= str_http_1_0.size() && !strncmp(header.data(), str_http_1_0.c_str(), str_http_1_0.size())) info.protocol = Protocol::HTTP_1_0_OK; @@ -34,6 +35,8 @@ HTTPHelper::Info HTTPHelper::detect(const string_view header) info.protocol = Protocol::HTTP_1_1_Partial; else if (header.length() >= str_http_1_1_created.size() && !strncmp(header.data(), str_http_1_1_created.c_str(), str_http_1_1_created.size())) info.protocol = Protocol::HTTP_1_1_Created; + else if (header.length() >= str_http_1_1_no_conent.size() && !strncmp(header.data(), str_http_1_1_no_conent.c_str(), str_http_1_1_no_conent.size())) + info.protocol = Protocol::HTTP_1_1_No_Content; if (info.protocol != Protocol::Unknown) { if (header.npos != header.find("Transfer-Encoding: chunked")) { @@ -84,9 +87,11 @@ bool HTTPHelper::finished(const uint8_t* data, uint64_t length, unique_ptr if (ret) info->length -= info->headerLength; return ret; - } default: { + if (info->protocol == Protocol::HTTP_1_1_No_Content) + return true; + info = nullptr; throw runtime_error("Unsupported HTTP transfer protocol"); } diff --git a/test/integration/minio.cpp b/test/integration/minio.cpp index ccc9d88..063094d 100644 --- a/test/integration/minio.cpp +++ b/test/integration/minio.cpp @@ -2,6 +2,7 @@ #include "cloud/provider.hpp" #include "network/get_transaction.hpp" #include "network/put_transaction.hpp" +#include "network/delete_transaction.hpp" #include "network/tasked_send_receiver.hpp" #include #include @@ -102,6 +103,22 @@ TEST_CASE("MinIO Integration") { REQUIRE(!rawDataString.compare(content[i++])); } } + { + // Create the put request + anyblob::network::DeleteTransaction deleteTxn(provider.get()); + for (auto i = 0u; i < 2; i++) + deleteTxn.addRequest(fileName[i]); + + // Upload the request synchronously with the scheduler object on this thread + deleteTxn.processSync(sendReceiver); + + // Check the upload + for (const auto& it : deleteTxn) { + // Sucessful request + REQUIRE(it.success()); + } + } + } //--------------------------------------------------------------------------- } // namespace test diff --git a/test/unit/cloud/aws_test.cpp b/test/unit/cloud/aws_test.cpp index ab18ddc..6dd1531 100644 --- a/test/unit/cloud/aws_test.cpp +++ b/test/unit/cloud/aws_test.cpp @@ -62,6 +62,12 @@ class AWSTester { resultString += "\r\nx-amz-request-payer: requester\r\nx-amz-security-token: ABC\r\n\r\n"; REQUIRE(string_view(reinterpret_cast(dv->data()), dv->size()) == resultString); + dv = aws.deleteRequest("a/b/c.d"); + resultString = "DELETE /a/b/c.d? HTTP/1.1\r\nAuthorization: AWS4-HMAC-SHA256 Credential=ABC/21000101/test/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-request-payer;x-amz-security-token, Signature=2240aba5140727498bd7bcea6f58e68a4c91ef2532b3273834a8d54983ae9319\r\nHost: test.s3.test.amazonaws.com\r\nx-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\r\nx-amz-date: "; + resultString += aws.fakeAMZTimestamp; + resultString += "\r\nx-amz-request-payer: requester\r\nx-amz-security-token: ABC\r\n\r\n"; + REQUIRE(string_view(reinterpret_cast(dv->data()), dv->size()) == resultString); + auto vec = AWSInstance::getInstanceDetails(); REQUIRE(vec.size() > 0); diff --git a/test/unit/cloud/azure_test.cpp b/test/unit/cloud/azure_test.cpp index 4ac570e..1a059e4 100644 --- a/test/unit/cloud/azure_test.cpp +++ b/test/unit/cloud/azure_test.cpp @@ -50,6 +50,12 @@ class AzureTester { resultString += "\r\nx-ms-version: 2015-02-21\r\n\r\n"; REQUIRE(string_view(reinterpret_cast(dv->data()), dv->size()) == resultString); + dv = azure.deleteRequest("a/b/c.d"); + resultString = "DELETE /test/a/b/c.d HTTP/1.1\r\nAuthorization: SharedKey test:nuGDW7QRI5/DB5Xt9vET/YEmipJ4UGjn64h4A+BFaL0=\r\nHost: test.blob.core.windows.net\r\nx-ms-date: "; + resultString += azure.fakeXMSTimestamp; + resultString += "\r\nx-ms-version: 2015-02-21\r\n\r\n"; + REQUIRE(string_view(reinterpret_cast(dv->data()), dv->size()) == resultString); + Provider::testEnviornment = false; } }; diff --git a/test/unit/cloud/gcp_test.cpp b/test/unit/cloud/gcp_test.cpp index 19e9c86..08e510c 100644 --- a/test/unit/cloud/gcp_test.cpp +++ b/test/unit/cloud/gcp_test.cpp @@ -88,6 +88,10 @@ class GCPTester { resultString += "\r\nHost: test.storage.googleapis.com\r\n\r\n"; REQUIRE(string_view(reinterpret_cast(dv->data()), dv->size()) == resultString); + dv = gcp.deleteRequest("a/b/c.d"); + resultString = "DELETE /a/b/c.d?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test%40test.com%2F21000101%2Ftest%2Fstorage%2Fgoog4_request&X-Goog-Date=21000101T000000Z&X-Goog-Expires=3600&X-Goog-SignedHeaders=host&x-goog-signature=bcd0bf851dd3121aa2b06a53b7abfd368c99d3424a37e87b71e45dc2c966e66d84ff6cee740835e07b808eba439ec9ffdc8c147138754e12eb68b89f1eaf47a08dc6b7a09d8aab85740e7e1007091caca7427a5f6d11998102214abd3086d403894bc38f4be97a30b9fa99b9247516636074558a9cb0caea7ae12de67780c2ff0c19db60136e3e7d5c9e196123a22b6aeb634a953e6bd3a0e18218dbaee362776fd7d50ccff4afc5ee3e5ed5c5c2610c8c2e30d786d7d5e8a272914942541f914395fa0b71a94ce4684a493b007d83bffc2ee2e06765cab000d5fdbbe07a46e26d3bb11c47617c53d62a4450af03c5100f6641fa7bd3ce816e859610cbd0b9d7f2a9f49f9c1ad25ea47c4e54afd1b91ce785fe7232515f89982d82163ea97df6cf50404fae8c5d738aa219b3907478d265c684033c12fc535b8c27863914881620df1462a4307f2b61b38291bb2e040f037a13b5841d5908aa069b2c4b878c972e3a646bce4a254d47ccee3a1da670ff40a614bf0a631729fa4e2fd161ce98ec HTTP/1.1\r\nHost: test.storage.googleapis.com\r\n\r\n"; + REQUIRE(string_view(reinterpret_cast(dv->data()), dv->size()) == resultString); + ignore = GCPInstance::getInstanceDetails(); Provider::testEnviornment = false;