Added Delete Blob Methods

include/cloud/aws.hpp

@@ -107,6 +107,8 @@ class AWS : public Provider {
     [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> getRequest(const std::string& filePath, const std::pair<uint64_t, uint64_t>& range) const override;
     /// Builds the http request for putting objects without the object data itself
     [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> putRequest(const std::string& filePath, const std::string_view object) const override;
+    // Builds the http request for deleting an objects
+    [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> deleteRequest(const std::string& filePath) const override;
 
     /// Get the address of the server
     [[nodiscard]] std::string getAddress() const override;

include/cloud/azure.hpp

@@ -78,6 +78,8 @@ class Azure : public Provider {
     [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> getRequest(const std::string& filePath, const std::pair<uint64_t, uint64_t>& range) const override;
     /// Builds the http request for putting objects without the object data itself
     [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> putRequest(const std::string& filePath, const std::string_view object) const override;
+    // Builds the http request for deleting an objects
+    [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> deleteRequest(const std::string& filePath) const override;
 
     /// Get the address of the server
     [[nodiscard]] std::string getAddress() const override;

include/cloud/gcp.hpp

@@ -76,6 +76,8 @@ class GCP : public Provider {
     [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> getRequest(const std::string& filePath, const std::pair<uint64_t, uint64_t>& range) const override;
     /// Builds the http request for putting objects without the object data itself
     [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> putRequest(const std::string& filePath, const std::string_view object) const override;
+    // Builds the http request for deleting an objects
+    [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> deleteRequest(const std::string& filePath) const override;
 
     /// Get the address of the server
     [[nodiscard]] std::string getAddress() const override;

include/cloud/ibm.hpp

@@ -33,7 +33,7 @@ class IBM : public AWS {
     /// Get the address of the server
     [[nodiscard]] std::string getAddress() const override;
     /// Get the instance details
-    Provider::Instance getInstanceDetails(network::TaskedSendReceiver& sendReceiver) override;
+    [[nodiscard]] Provider::Instance getInstanceDetails(network::TaskedSendReceiver& sendReceiver) override;
 
     friend Provider;
 };

include/cloud/minio.hpp

@@ -28,7 +28,7 @@ class MinIO : public AWS {
     /// Get the address of the server
     [[nodiscard]] std::string getAddress() const override;
     /// Get the instance details
-    Provider::Instance getInstanceDetails(network::TaskedSendReceiver& sendReceiver) override;
+    [[nodiscard]] Provider::Instance getInstanceDetails(network::TaskedSendReceiver& sendReceiver) override;
 
     friend Provider;
 };

include/cloud/oracle.hpp

@@ -28,7 +28,7 @@ class Oracle : public AWS {
     /// Get the address of the server
     [[nodiscard]] std::string getAddress() const override;
     /// Get the instance details
-    Provider::Instance getInstanceDetails(network::TaskedSendReceiver& sendReceiver) override;
+    [[nodiscard]] Provider::Instance getInstanceDetails(network::TaskedSendReceiver& sendReceiver) override;
 
     friend Provider;
 };

include/cloud/provider.hpp

@@ -16,6 +16,7 @@ class TaskedSendReceiver;
 class Transaction;
 class GetTransaction;
 class PutTransaction;
+class DeleteTransaction;
 struct OriginalMessage;
 }; // namespace network
 namespace utils {
@@ -76,6 +77,8 @@ class Provider {
     [[nodiscard]] virtual std::unique_ptr<utils::DataVector<uint8_t>> getRequest(const std::string& filePath, const std::pair<uint64_t, uint64_t>& range) const = 0;
     /// Builds the http request for putting an object without the actual data (header only according to the data and length provided)
     [[nodiscard]] virtual std::unique_ptr<utils::DataVector<uint8_t>> putRequest(const std::string& filePath, const std::string_view object) const = 0;
+    /// Builds the http request for deleting an object
+    [[nodiscard]] virtual std::unique_ptr<utils::DataVector<uint8_t>> deleteRequest(const std::string& filePath) const = 0;
     /// Get the address of the server
     [[nodiscard]] virtual std::string getAddress() const = 0;
     /// Get the port of the server
@@ -108,6 +111,7 @@ class Provider {
     friend network::Transaction;
     friend network::GetTransaction;
     friend network::PutTransaction;
+    friend network::DeleteTransaction;
 };
 //---------------------------------------------------------------------------
 } // namespace cloud

include/network/delete_transaction.hpp

@@ -0,0 +1,43 @@
+#pragma once
+#include "cloud/provider.hpp"
+#include "network/original_message.hpp"
+#include "network/transaction.hpp"
+#include <cassert>
+//---------------------------------------------------------------------------
+// AnyBlob - Universal Cloud Object Storage Library
+// Dominik Durner, 2023
+//
+// This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0.
+// If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+// SPDX-License-Identifier: MPL-2.0
+//---------------------------------------------------------------------------
+namespace anyblob {
+//---------------------------------------------------------------------------
+namespace network {
+//---------------------------------------------------------------------------
+/// Specialization of a delete request
+class DeleteTransaction : public Transaction {
+    public:
+    /// The default constructor
+    DeleteTransaction() = default;
+    /// The explicit constructor
+    explicit DeleteTransaction(const cloud::Provider* provider) : Transaction(provider) {}
+
+    /// Build a new delete request for synchronous calls
+    inline void addRequest(const std::string& remotePath, uint8_t* result = nullptr, uint64_t capacity = 0, uint64_t traceId = 0) {
+        assert(provider);
+        auto originalMsg = std::make_unique<network::OriginalMessage>(provider->deleteRequest(remotePath), provider->getAddress(), provider->getPort(), result, capacity, traceId);
+        messages.push_back(std::move(originalMsg));
+    }
+
+    /// Build a new delete request with callback
+    template <typename Callback>
+    inline void addRequest(Callback&& callback, const std::string& remotePath, uint8_t* result = nullptr, uint64_t capacity = 0, uint64_t traceId = 0) {
+        assert(provider);
+        auto originalMsg = std::make_unique<network::OriginalCallbackMessage<Callback>>(std::forward<Callback&&>(callback), provider->deleteRequest(remotePath), provider->getAddress(), provider->getPort(), result, capacity, traceId);
+        messages.push_back(std::move(originalMsg));
+    }
+};
+//---------------------------------------------------------------------------
+} // namespace network
+} // namespace anyblob

include/network/http_helper.hpp

@@ -23,7 +23,8 @@ class HTTPHelper {
         HTTP_1_0_OK,
         HTTP_1_1_Partial,
         HTTP_1_1_OK,
-        HTTP_1_1_Created
+        HTTP_1_1_Created,
+        HTTP_1_1_No_Content
     };
 
     /// The encoding

include/utils/ring_buffer.hpp

@@ -44,7 +44,7 @@ class RingBuffer {
     /// The buffer size
     uint64_t _size;
     /// The insert cache-aligned struct
-    struct alignas(64) {
+    struct {
         /// The current insert head
         std::atomic<uint64_t> pending;
         /// The current insert counter such that we can safely insert before seen takes element
@@ -53,7 +53,7 @@ class RingBuffer {
         SpinMutex mutex;
     } _insert;
     /// The seen cache-alignedstruct
-    struct alignas(64) {
+    struct {
         /// The seen head
         std::atomic<uint64_t> pending;
         /// The seen counter

src/cloud/aws.cpp

@@ -66,9 +66,8 @@ Provider::Instance AWS::getInstanceDetails(network::TaskedSendReceiver& sendRece
                 return instance;
 
         return AWSInstance{string(s), 0, 0, ""};
-    } else {
-        return AWSInstance{"minio", 0, 0, "10"};
     }
+    return AWSInstance{"aws", 0, 0, ""};
 }
 //---------------------------------------------------------------------------
 string AWS::getInstanceRegion(network::TaskedSendReceiver& sendReceiver)
@@ -264,6 +263,40 @@ unique_ptr<utils::DataVector<uint8_t>> AWS::putRequest(const string& filePath, c
     return make_unique<utils::DataVector<uint8_t>>(reinterpret_cast<uint8_t*>(httpHeader.data()), reinterpret_cast<uint8_t*>(httpHeader.data() + httpHeader.size()));
 }
 //---------------------------------------------------------------------------
+unique_ptr<utils::DataVector<uint8_t>> AWS::deleteRequest(const string& filePath) const
+// Builds the http request for deleting an objects
+{
+    if (!validKeys())
+        return nullptr;
+
+    AWSSigner::Request request;
+    request.method = "DELETE";
+    request.type = "HTTP/1.1";
+
+    // If an endpoint is defined, we use the path-style request. The default is the usage of virtual hosted-style requests.
+    if (_settings.endpoint.empty())
+        request.path = "/" + filePath;
+    else
+        request.path = "/" + _settings.bucket + "/" + filePath;
+    request.bodyData = nullptr;
+    request.bodyLength = 0;
+    request.headers.emplace("Host", getAddress());
+    request.headers.emplace("x-amz-date", testEnviornment ? fakeAMZTimestamp : buildAMZTimestamp());
+    request.headers.emplace("x-amz-request-payer", "requester");
+    if (!_secret->sessionToken.empty())
+        request.headers.emplace("x-amz-security-token", _secret->sessionToken);
+
+    auto canonical = AWSSigner::createCanonicalRequest(request);
+
+    AWSSigner::StringToSign stringToSign = {.request = request, .requestSHA = canonical.second, .region = _settings.region, .service = "s3"};
+    const auto uri = AWSSigner::createSignedRequest(_secret->keyId, _secret->secret, stringToSign);
+    auto httpHeader = request.method + " " + uri + " " + request.type + "\r\n";
+    for (auto& h : request.headers)
+        httpHeader += h.first + ": " + h.second + "\r\n";
+    httpHeader += "\r\n";
+    return make_unique<utils::DataVector<uint8_t>>(reinterpret_cast<uint8_t*>(httpHeader.data()), reinterpret_cast<uint8_t*>(httpHeader.data() + httpHeader.size()));
+}
+//---------------------------------------------------------------------------
 uint32_t AWS::getPort() const
 // Gets the port of AWS S3 on http
 {

src/cloud/azure.cpp

@@ -147,6 +147,30 @@ unique_ptr<utils::DataVector<uint8_t>> Azure::putRequest(const string& filePath,
     return make_unique<utils::DataVector<uint8_t>>(reinterpret_cast<uint8_t*>(httpHeader.data()), reinterpret_cast<uint8_t*>(httpHeader.data() + httpHeader.size()));
 }
 //---------------------------------------------------------------------------
+unique_ptr<utils::DataVector<uint8_t>> Azure::deleteRequest(const string& filePath) const
+// Builds the http request for deleting objects
+{
+    AzureSigner::Request request;
+    request.method = "DELETE";
+    request.type = "HTTP/1.1";
+    request.path = "/" + _settings.container + "/" + filePath;
+    request.bodyData = nullptr;
+    request.bodyLength = 0;
+
+    auto date = testEnviornment ? fakeXMSTimestamp : buildXMSTimestamp();
+    request.headers.emplace("x-ms-date", date);
+    request.headers.emplace("Host", getAddress());
+
+    request.path = AzureSigner::createSignedRequest(_secret->accountName, _secret->privateKey, request);
+
+    auto httpHeader = request.method + " " + request.path + " " + request.type + "\r\n";
+    for (auto& h : request.headers)
+        httpHeader += h.first + ": " + h.second + "\r\n";
+    httpHeader += "\r\n";
+
+    return make_unique<utils::DataVector<uint8_t>>(reinterpret_cast<uint8_t*>(httpHeader.data()), reinterpret_cast<uint8_t*>(httpHeader.data() + httpHeader.size()));
+}
+//---------------------------------------------------------------------------
 uint32_t Azure::getPort() const
 // Gets the port of Azure on http
 {

src/cloud/gcp.cpp

@@ -132,6 +132,31 @@ unique_ptr<utils::DataVector<uint8_t>> GCP::putRequest(const string& filePath, c
     return make_unique<utils::DataVector<uint8_t>>(reinterpret_cast<uint8_t*>(httpHeader.data()), reinterpret_cast<uint8_t*>(httpHeader.data() + httpHeader.size()));
 }
 //---------------------------------------------------------------------------
+unique_ptr<utils::DataVector<uint8_t>> GCP::deleteRequest(const string& filePath) const
+// Builds the http request for deleting objects
+{
+    GCPSigner::Request request;
+    request.method = "DELETE";
+    request.type = "HTTP/1.1";
+    request.path = "/" + filePath;
+    request.bodyData = nullptr;
+    request.bodyLength = 0;
+
+    auto date = testEnviornment ? fakeAMZTimestamp : buildAMZTimestamp();
+    request.queries.emplace("X-Goog-Date", date);
+    request.headers.emplace("Host", getAddress());
+
+    GCPSigner::StringToSign stringToSign = {.region = _settings.region, .service = "storage"};
+    request.path = GCPSigner::createSignedRequest(_secret->serviceAccountEmail, _secret->privateKey, request, stringToSign);
+
+    auto httpHeader = request.method + " " + request.path + " " + request.type + "\r\n";
+    for (auto& h : request.headers)
+        httpHeader += h.first + ": " + h.second + "\r\n";
+    httpHeader += "\r\n";
+
+    return make_unique<utils::DataVector<uint8_t>>(reinterpret_cast<uint8_t*>(httpHeader.data()), reinterpret_cast<uint8_t*>(httpHeader.data() + httpHeader.size()));
+}
+//---------------------------------------------------------------------------
 uint32_t GCP::getPort() const
 // Gets the port of GCP on http
 {

src/network/http_helper.cpp

@@ -25,6 +25,7 @@ HTTPHelper::Info HTTPHelper::detect(const string_view header)
     string str_http_1_1 = "HTTP/1.1 200 OK";
     string str_http_1_1_partial = "HTTP/1.1 206 Partial Content";
     string str_http_1_1_created = "HTTP/1.1 201 Created";
+    string str_http_1_1_no_conent = "HTTP/1.1 204 No Content";
 
     if (header.length() >= str_http_1_0.size() && !strncmp(header.data(), str_http_1_0.c_str(), str_http_1_0.size()))
         info.protocol = Protocol::HTTP_1_0_OK;
@@ -34,6 +35,8 @@ HTTPHelper::Info HTTPHelper::detect(const string_view header)
         info.protocol = Protocol::HTTP_1_1_Partial;
     else if (header.length() >= str_http_1_1_created.size() && !strncmp(header.data(), str_http_1_1_created.c_str(), str_http_1_1_created.size()))
         info.protocol = Protocol::HTTP_1_1_Created;
+    else if (header.length() >= str_http_1_1_no_conent.size() && !strncmp(header.data(), str_http_1_1_no_conent.c_str(), str_http_1_1_no_conent.size()))
+        info.protocol = Protocol::HTTP_1_1_No_Content;
 
     if (info.protocol != Protocol::Unknown) {
         if (header.npos != header.find("Transfer-Encoding: chunked")) {
@@ -84,9 +87,11 @@ bool HTTPHelper::finished(const uint8_t* data, uint64_t length, unique_ptr<Info>
                 if (ret)
                     info->length -= info->headerLength;
                 return ret;
-
             }
             default: {
+                if (info->protocol == Protocol::HTTP_1_1_No_Content)
+                    return true;
+
                 info = nullptr;
                 throw runtime_error("Unsupported HTTP transfer protocol");
             }

test/integration/minio.cpp

@@ -2,6 +2,7 @@
 #include "cloud/provider.hpp"
 #include "network/get_transaction.hpp"
 #include "network/put_transaction.hpp"
+#include "network/delete_transaction.hpp"
 #include "network/tasked_send_receiver.hpp"
 #include <cstdlib>
 #include <cstring>
@@ -102,6 +103,22 @@ TEST_CASE("MinIO Integration") {
             REQUIRE(!rawDataString.compare(content[i++]));
         }
     }
+    {
+        // Create the put request
+        anyblob::network::DeleteTransaction deleteTxn(provider.get());
+        for (auto i = 0u; i < 2; i++)
+            deleteTxn.addRequest(fileName[i]);
+
+        // Upload the request synchronously with the scheduler object on this thread
+        deleteTxn.processSync(sendReceiver);
+
+        // Check the upload
+        for (const auto& it : deleteTxn) {
+            // Sucessful request
+            REQUIRE(it.success());
+        }
+    }
+
 }
 //---------------------------------------------------------------------------
 } // namespace test

test/unit/cloud/aws_test.cpp

@@ -62,6 +62,12 @@ class AWSTester {
         resultString += "\r\nx-amz-request-payer: requester\r\nx-amz-security-token: ABC\r\n\r\n";
         REQUIRE(string_view(reinterpret_cast<char*>(dv->data()), dv->size()) == resultString);
 
+        dv = aws.deleteRequest("a/b/c.d");
+        resultString = "DELETE /a/b/c.d? HTTP/1.1\r\nAuthorization: AWS4-HMAC-SHA256 Credential=ABC/21000101/test/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-request-payer;x-amz-security-token, Signature=2240aba5140727498bd7bcea6f58e68a4c91ef2532b3273834a8d54983ae9319\r\nHost: test.s3.test.amazonaws.com\r\nx-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\r\nx-amz-date: ";
+        resultString += aws.fakeAMZTimestamp;
+        resultString += "\r\nx-amz-request-payer: requester\r\nx-amz-security-token: ABC\r\n\r\n";
+        REQUIRE(string_view(reinterpret_cast<char*>(dv->data()), dv->size()) == resultString);
+
         auto vec = AWSInstance::getInstanceDetails();
         REQUIRE(vec.size() > 0);
 

test/unit/cloud/azure_test.cpp

@@ -50,6 +50,12 @@ class AzureTester {
         resultString += "\r\nx-ms-version: 2015-02-21\r\n\r\n";
         REQUIRE(string_view(reinterpret_cast<char*>(dv->data()), dv->size()) == resultString);
 
+        dv = azure.deleteRequest("a/b/c.d");
+        resultString = "DELETE /test/a/b/c.d HTTP/1.1\r\nAuthorization: SharedKey test:nuGDW7QRI5/DB5Xt9vET/YEmipJ4UGjn64h4A+BFaL0=\r\nHost: test.blob.core.windows.net\r\nx-ms-date: ";
+        resultString += azure.fakeXMSTimestamp;
+        resultString += "\r\nx-ms-version: 2015-02-21\r\n\r\n";
+        REQUIRE(string_view(reinterpret_cast<char*>(dv->data()), dv->size()) == resultString);
+
         Provider::testEnviornment = false;
     }
 };

test/unit/cloud/gcp_test.cpp

@@ -88,6 +88,10 @@ class GCPTester {
         resultString += "\r\nHost: test.storage.googleapis.com\r\n\r\n";
         REQUIRE(string_view(reinterpret_cast<char*>(dv->data()), dv->size()) == resultString);
 
+        dv = gcp.deleteRequest("a/b/c.d");
+        resultString = "DELETE /a/b/c.d?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test%40test.com%2F21000101%2Ftest%2Fstorage%2Fgoog4_request&X-Goog-Date=21000101T000000Z&X-Goog-Expires=3600&X-Goog-SignedHeaders=host&x-goog-signature=bcd0bf851dd3121aa2b06a53b7abfd368c99d3424a37e87b71e45dc2c966e66d84ff6cee740835e07b808eba439ec9ffdc8c147138754e12eb68b89f1eaf47a08dc6b7a09d8aab85740e7e1007091caca7427a5f6d11998102214abd3086d403894bc38f4be97a30b9fa99b9247516636074558a9cb0caea7ae12de67780c2ff0c19db60136e3e7d5c9e196123a22b6aeb634a953e6bd3a0e18218dbaee362776fd7d50ccff4afc5ee3e5ed5c5c2610c8c2e30d786d7d5e8a272914942541f914395fa0b71a94ce4684a493b007d83bffc2ee2e06765cab000d5fdbbe07a46e26d3bb11c47617c53d62a4450af03c5100f6641fa7bd3ce816e859610cbd0b9d7f2a9f49f9c1ad25ea47c4e54afd1b91ce785fe7232515f89982d82163ea97df6cf50404fae8c5d738aa219b3907478d265c684033c12fc535b8c27863914881620df1462a4307f2b61b38291bb2e040f037a13b5841d5908aa069b2c4b878c972e3a646bce4a254d47ccee3a1da670ff40a614bf0a631729fa4e2fd161ce98ec HTTP/1.1\r\nHost: test.storage.googleapis.com\r\n\r\n";
+        REQUIRE(string_view(reinterpret_cast<char*>(dv->data()), dv->size()) == resultString);
+
         ignore = GCPInstance::getInstanceDetails();
 
         Provider::testEnviornment = false;