Use central HttpRequest to allow future resigning of failed requests

include/cloud/aws.hpp

@@ -79,7 +79,6 @@ class AWS : public Provider {
     /// The session secret
     thread_local static std::shared_ptr<Secret> _sessionSecret;
 
-
     public:
     /// Get instance details
     [[nodiscard]] Provider::Instance getInstanceDetails(network::TaskedSendReceiver& sendReceiver) override;
@@ -131,7 +130,7 @@ class AWS : public Provider {
     [[nodiscard]] uint64_t multipartUploadSize() const override { return _multipartUploadSize; }
 
     /// Creates the generic http request and signs it
-    [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> buildRequest(AWSSigner::Request& request, std::string_view payload = "", bool initHeaders = true) const;
+    [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> buildRequest(network::HttpRequest& request, const uint8_t* bodyData = nullptr, uint64_t bodyLength = 0, bool initHeaders = true) const;
     /// Builds the http request for downloading a blob or listing the directory
     [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> getRequest(const std::string& filePath, const std::pair<uint64_t, uint64_t>& range) const override;
     /// Builds the http request for putting objects without the object data itself
@@ -149,7 +148,7 @@ class AWS : public Provider {
     /// Builds the http request for creating multipart put objects
     [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> createMultiPartRequest(const std::string& filePath) const override;
     /// Builds the http request for completing multipart put objects
-    [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> completeMultiPartRequest(const std::string& filePath, std::string_view uploadId, const std::vector<std::string>& etags) const override;
+    [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> completeMultiPartRequest(const std::string& filePath, std::string_view uploadId, const std::vector<std::string>& etags, std::string& content) const override;
     /// Builds the http request for getting the session token objects
     [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> getSessionToken(std::string_view type = "ReadWrite") const;
     /// Get the address of the server

include/cloud/aws_signer.hpp

@@ -1,5 +1,5 @@
 #pragma once
-#include <map>
+#include "network/http_request.hpp"
 #include <memory>
 #include <string>
 //---------------------------------------------------------------------------
@@ -17,40 +17,23 @@ namespace cloud {
 /// It follows the v4 docu: https://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html
 class AWSSigner {
     public:
-    struct Request {
-        /// The method
-        std::string method;
-        /// The type
-        std::string type;
-        /// The path - needs to be RFC 3986 conform
-        std::string path;
-        /// The queries - need to be RFC 3986 conform
-        std::map<std::string, std::string> queries;
-        /// The headers - need to be without trailing and leading whitespaces
-        std::map<std::string, std::string> headers;
-        /// The signed headers
-        std::string signedHeaders;
-        /// The payload hash
-        std::string payloadHash;
-        /// The unowned body data
-        const uint8_t* bodyData;
-        /// The unowned body length
-        uint64_t bodyLength;
-    };
-
     struct StringToSign {
         /// The canonical request
-        Request& request;
-        /// The request sha
-        std::string& requestSHA;
+        network::HttpRequest& request;
         /// The region
         std::string region;
         /// The service
         std::string service;
+        /// The request sha
+        std::string requestSHA;
+        /// The signed headers
+        std::string signedHeaders;
+        /// The payload hash
+        std::string payloadHash;
     };
 
     /// Creates the canonical request from the input
-    [[nodiscard]] static std::pair<std::string, std::string> createCanonicalRequest(Request& request);
+    static void encodeCanonicalRequest(network::HttpRequest& request, StringToSign& stringToSign, const uint8_t* bodyData = nullptr, uint64_t bodyLength = 0);
     /// Calculates the signature
     [[nodiscard]] static std::string createSignedRequest(const std::string& keyId, const std::string& secret, const StringToSign& stringToSign);
 

include/cloud/azure_signer.hpp

@@ -1,4 +1,5 @@
 #pragma once
+#include "network/http_request.hpp"
 #include <map>
 #include <memory>
 #include <string>
@@ -17,29 +18,8 @@ namespace cloud {
 /// It follows the v4 docu: https://cloud.google.com/storage/docs/access-control/signing-urls-manually
 class AzureSigner {
     public:
-    struct Request {
-        /// The method
-        std::string method;
-        /// The type
-        std::string type;
-        /// The path - needs to be RFC 3986 conform
-        std::string path;
-        /// The queries - need to be RFC 3986 conform
-        std::map<std::string, std::string> queries;
-        /// The headers - need to be without trailing and leading whitespaces
-        std::map<std::string, std::string> headers;
-        /// The signed headers
-        std::string signedHeaders;
-        /// The payload hash
-        std::string payloadHash;
-        /// The unowned body data
-        const uint8_t* bodyData;
-        /// The unowned body length
-        uint64_t bodyLength;
-    };
-
     /// Builds the signed url
-    [[nodiscard]] static std::string createSignedRequest(const std::string& serviceAccountEmail, const std::string& privateRSA, Request& request);
+    [[nodiscard]] static std::string createSignedRequest(const std::string& serviceAccountEmail, const std::string& privateRSA, network::HttpRequest& request);
 };
 //---------------------------------------------------------------------------
 }; // namespace cloud

include/cloud/gcp.hpp

@@ -92,7 +92,7 @@ class GCP : public Provider {
     /// Builds the http request for creating multipart put objects
     [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> createMultiPartRequest(const std::string& filePath) const override;
     /// Builds the http request for completing multipart put objects
-    [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> completeMultiPartRequest(const std::string& filePath, std::string_view uploadId, const std::vector<std::string>& etags) const override;
+    [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> completeMultiPartRequest(const std::string& filePath, std::string_view uploadId, const std::vector<std::string>& etags, std::string& content) const override;
 
     /// Get the address of the server
     [[nodiscard]] std::string getAddress() const override;

include/cloud/gcp_signer.hpp

@@ -1,4 +1,5 @@
 #pragma once
+#include "network/http_request.hpp"
 #include <map>
 #include <memory>
 #include <string>
@@ -17,36 +18,17 @@ namespace cloud {
 /// It follows the v4 docu: https://cloud.google.com/storage/docs/access-control/signing-urls-manually
 class GCPSigner {
     public:
-    struct Request {
-        /// The method
-        std::string method;
-        /// The type
-        std::string type;
-        /// The path - needs to be RFC 3986 conform
-        std::string path;
-        /// The queries - need to be RFC 3986 conform
-        std::map<std::string, std::string> queries;
-        /// The headers - need to be without trailing and leading whitespaces
-        std::map<std::string, std::string> headers;
-        /// The signed headers
-        std::string signedHeaders;
-        /// The payload hash
-        std::string payloadHash;
-        /// The unowned body data
-        const uint8_t* bodyData;
-        /// The unowned body length
-        uint64_t bodyLength;
-    };
-
     struct StringToSign {
         /// The region
         std::string region;
         /// The service
         std::string service;
+        /// The signed headers
+        std::string signedHeaders;
     };
 
     /// Builds the signed url
-    [[nodiscard]] static std::string createSignedRequest(const std::string& serviceAccountEmail, const std::string& privateRSA, Request& request, const StringToSign& stringToSign);
+    [[nodiscard]] static std::string createSignedRequest(const std::string& serviceAccountEmail, const std::string& privateRSA, network::HttpRequest& request, StringToSign& stringToSign);
 };
 //---------------------------------------------------------------------------
 }; // namespace cloud

include/cloud/provider.hpp

@@ -97,7 +97,7 @@ class Provider {
     /// Builds the http request for creating multipart put objects
     [[nodiscard]] virtual std::unique_ptr<utils::DataVector<uint8_t>> createMultiPartRequest(const std::string& /*filePath*/) const;
     /// Builds the http request for completing multipart put objects
-    [[nodiscard]] virtual std::unique_ptr<utils::DataVector<uint8_t>> completeMultiPartRequest(const std::string& /*filePath*/, std::string_view /*uploadId*/, const std::vector<std::string>& /*etags*/) const;
+    [[nodiscard]] virtual std::unique_ptr<utils::DataVector<uint8_t>> completeMultiPartRequest(const std::string& /*filePath*/, std::string_view /*uploadId*/, const std::vector<std::string>& /*etags*/, std::string& /*eTagsContent*/) const;
 
     /// Initialize secret
     virtual void initSecret(network::TaskedSendReceiver& /*sendReceiver*/) {}

include/network/http_helper.hpp

@@ -14,7 +14,7 @@ namespace anyblob {
 namespace network {
 //---------------------------------------------------------------------------
 /// Implements an helper to resolve http requests
-class HTTPHelper {
+class HttpHelper {
     public:
     /// The protocol detected by the helper
     enum class Protocol : uint8_t {

include/network/http_message.hpp

@@ -19,7 +19,7 @@ struct HTTPMessage : public MessageTask {
     /// The tcp settings
     IOUringSocket::TCPSettings tcpSettings;
     /// HTTP info header
-    std::unique_ptr<HTTPHelper::Info> info;
+    std::unique_ptr<HttpHelper::Info> info;
 
     /// The constructor
     HTTPMessage(OriginalMessage* sendingMessage, uint64_t chunkSize);

include/network/http_request.hpp

@@ -0,0 +1,73 @@
+#pragma once
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <string>
+#include <string_view>
+//---------------------------------------------------------------------------
+// AnyBlob - Universal Cloud Object Storage Library
+// Dominik Durner, 2024
+//
+// This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0.
+// If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+// SPDX-License-Identifier: MPL-2.0
+//---------------------------------------------------------------------------
+namespace anyblob {
+//---------------------------------------------------------------------------
+namespace utils {
+template <typename T>
+class DataVector;
+}
+//---------------------------------------------------------------------------
+namespace network {
+//---------------------------------------------------------------------------
+/// Implements an helper to serialize and deserialize http requests
+struct HttpRequest {
+    /// The method class
+    enum class Method : uint8_t {
+        GET,
+        PUT,
+        POST,
+        DELETE
+    };
+    enum class Type : uint8_t {
+        HTTP_1_0,
+        HTTP_1_1
+    };
+    /// The queries - need to be RFC 3986 conform
+    std::map<std::string, std::string> queries;
+    /// The headers - need to be without trailing and leading whitespaces
+    std::map<std::string, std::string> headers;
+    /// The method
+    Method method;
+    /// The type
+    Type type;
+    /// The path - needs to be RFC 3986 conform
+    std::string path;
+
+    /// Get the request method
+    static constexpr auto getRequestMethod(const HttpRequest& request) {
+        switch (request.method) {
+            case Method::GET: return "GET";
+            case Method::PUT: return "PUT";
+            case Method::POST: return "POST";
+            case Method::DELETE: return "DELETE";
+            default: return "";
+        }
+    }
+    /// Get the request type
+    static constexpr auto getRequestType(const HttpRequest& request) {
+        switch (request.type) {
+            case Type::HTTP_1_0: return "HTTP/1.0";
+            case Type::HTTP_1_1: return "HTTP/1.1";
+            default: return "";
+        }
+    }
+    /// Serialize the request
+    [[nodiscard]] static std::unique_ptr<utils::DataVector<uint8_t>> serialize(const HttpRequest& request);
+    /// Deserialize the request
+    [[nodiscard]] static HttpRequest deserialize(std::string_view data);
+};
+//---------------------------------------------------------------------------
+} // namespace network
+} // namespace anyblob

include/network/original_message.hpp

@@ -27,22 +27,22 @@ struct OriginalMessage {
     /// The result
     MessageResult result;
 
-    /// The hostname
-    std::string hostname;
-    /// The port
-    uint32_t port;
-
-    /// Optional trace info
-    uint64_t traceId;
-
     /// If it is a put request store the additional data
     /// The raw data ptr for put requests
     const uint8_t* putData;
     /// The length
     uint64_t putLength;
 
+    /// Optional trace info
+    uint64_t traceId;
+
+    /// The hostname
+    std::string hostname;
+    /// The port
+    uint32_t port;
+
     /// The constructor
-    OriginalMessage(std::unique_ptr<utils::DataVector<uint8_t>> message, std::string_view hostname, uint32_t port, uint8_t* receiveBuffer = nullptr, uint64_t bufferSize = 0, uint64_t traceId = 0) : message(std::move(message)), result(receiveBuffer, bufferSize), hostname(hostname), port(port), traceId(traceId), putData(nullptr), putLength() {}
+    OriginalMessage(std::unique_ptr<utils::DataVector<uint8_t>> message, std::string_view hostname, uint32_t port, uint8_t* receiveBuffer = nullptr, uint64_t bufferSize = 0, uint64_t traceId = 0) : message(std::move(message)), result(receiveBuffer, bufferSize), putData(nullptr), putLength(), traceId(traceId), hostname(hostname), port(port) {}
 
     /// The destructor
     virtual ~OriginalMessage() = default;

include/network/transaction.hpp

@@ -224,15 +224,18 @@ class Transaction {
                     }
                     if (_multipartUploads[position].outstanding.fetch_sub(1) == 1) {
                         if (_multipartUploads[position].state != MultipartUpload::State::Aborted) [[likely]] {
-                            auto finished = [&callback, &initalRequestResult, this](network::MessageResult& result) {
+                            auto contentData = std::make_unique<std::string>();
+                            auto content = contentData.get();
+                            auto finished = [&callback, &initalRequestResult, contentPtr = move(contentData), this](network::MessageResult& result) mutable {
                                 if (!result.success()) {
                                     initalRequestResult.state = network::MessageState::Cancelled;
                                     initalRequestResult.originError = &result;
                                 }
                                 _completedMultiparts++;
                                 std::forward<Callback>(callback)(initalRequestResult);
                             };
-                            auto originalMsg = makeCallbackMessage(std::move(finished), _provider->completeMultiPartRequest(remotePath, _multipartUploads[position].uploadId, _multipartUploads[position].eTags), _provider->getAddress(), _provider->getPort(), nullptr, 0, traceId);
+                            auto originalMsg = makeCallbackMessage(std::move(finished), _provider->completeMultiPartRequest(remotePath, _multipartUploads[position].uploadId, _multipartUploads[position].eTags, *content), _provider->getAddress(), _provider->getPort(), nullptr, 0, traceId);
+                            originalMsg->setPutRequestData(reinterpret_cast<const uint8_t*>(content->data()), content->size());
                             _multipartUploads[position].messages[parts] = std::move(originalMsg);
                         } else {
                             auto finished = [&callback, &initalRequestResult, position, this](network::MessageResult& /*result*/) {