From c1586c416bbc4f510e7431bb5351525d7881a7f0 Mon Sep 17 00:00:00 2001 From: Dominik Durner Date: Thu, 15 Feb 2024 17:49:13 +0100 Subject: [PATCH] Use central HttpRequest to allow future resigning of failed requests --- include/cloud/aws.hpp | 5 +- include/cloud/aws_signer.hpp | 35 +++------- include/cloud/azure_signer.hpp | 24 +------ include/cloud/gcp.hpp | 2 +- include/cloud/gcp_signer.hpp | 26 ++------ include/cloud/provider.hpp | 2 +- include/network/http_helper.hpp | 2 +- include/network/http_message.hpp | 2 +- include/network/http_request.hpp | 73 +++++++++++++++++++++ include/network/original_message.hpp | 18 +++--- include/network/transaction.hpp | 7 +- src/cloud/aws.cpp | 97 +++++++++++++--------------- src/cloud/aws_signer.cpp | 28 ++++---- src/cloud/azure.cpp | 50 +++++++------- src/cloud/azure_signer.cpp | 4 +- src/cloud/gcp.cpp | 93 +++++++++++++------------- src/cloud/gcp_signer.cpp | 10 +-- src/cloud/provider.cpp | 2 +- src/network/http_helper.cpp | 6 +- src/network/http_message.cpp | 2 +- src/network/http_request.cpp | 45 +++++++++++++ src/network/https_message.cpp | 2 +- 22 files changed, 300 insertions(+), 235 deletions(-) create mode 100644 include/network/http_request.hpp create mode 100644 src/network/http_request.cpp diff --git a/include/cloud/aws.hpp b/include/cloud/aws.hpp index 721a1cd..ef74827 100644 --- a/include/cloud/aws.hpp +++ b/include/cloud/aws.hpp @@ -79,7 +79,6 @@ class AWS : public Provider { /// The session secret thread_local static std::shared_ptr _sessionSecret; - public: /// Get instance details [[nodiscard]] Provider::Instance getInstanceDetails(network::TaskedSendReceiver& sendReceiver) override; @@ -131,7 +130,7 @@ class AWS : public Provider { [[nodiscard]] uint64_t multipartUploadSize() const override { return _multipartUploadSize; } /// Creates the generic http request and signs it - [[nodiscard]] std::unique_ptr> buildRequest(AWSSigner::Request& request, std::string_view payload = "", bool initHeaders = true) const; + [[nodiscard]] std::unique_ptr> buildRequest(network::HttpRequest& request, const uint8_t* bodyData = nullptr, uint64_t bodyLength = 0, bool initHeaders = true) const; /// Builds the http request for downloading a blob or listing the directory [[nodiscard]] std::unique_ptr> getRequest(const std::string& filePath, const std::pair& range) const override; /// Builds the http request for putting objects without the object data itself @@ -149,7 +148,7 @@ class AWS : public Provider { /// Builds the http request for creating multipart put objects [[nodiscard]] std::unique_ptr> createMultiPartRequest(const std::string& filePath) const override; /// Builds the http request for completing multipart put objects - [[nodiscard]] std::unique_ptr> completeMultiPartRequest(const std::string& filePath, std::string_view uploadId, const std::vector& etags) const override; + [[nodiscard]] std::unique_ptr> completeMultiPartRequest(const std::string& filePath, std::string_view uploadId, const std::vector& etags, std::string& content) const override; /// Builds the http request for getting the session token objects [[nodiscard]] std::unique_ptr> getSessionToken(std::string_view type = "ReadWrite") const; /// Get the address of the server diff --git a/include/cloud/aws_signer.hpp b/include/cloud/aws_signer.hpp index 9efb31f..23d3d2b 100644 --- a/include/cloud/aws_signer.hpp +++ b/include/cloud/aws_signer.hpp @@ -1,5 +1,5 @@ #pragma once -#include +#include "network/http_request.hpp" #include #include //--------------------------------------------------------------------------- @@ -17,40 +17,23 @@ namespace cloud { /// It follows the v4 docu: https://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html class AWSSigner { public: - struct Request { - /// The method - std::string method; - /// The type - std::string type; - /// The path - needs to be RFC 3986 conform - std::string path; - /// The queries - need to be RFC 3986 conform - std::map queries; - /// The headers - need to be without trailing and leading whitespaces - std::map headers; - /// The signed headers - std::string signedHeaders; - /// The payload hash - std::string payloadHash; - /// The unowned body data - const uint8_t* bodyData; - /// The unowned body length - uint64_t bodyLength; - }; - struct StringToSign { /// The canonical request - Request& request; - /// The request sha - std::string& requestSHA; + network::HttpRequest& request; /// The region std::string region; /// The service std::string service; + /// The request sha + std::string requestSHA; + /// The signed headers + std::string signedHeaders; + /// The payload hash + std::string payloadHash; }; /// Creates the canonical request from the input - [[nodiscard]] static std::pair createCanonicalRequest(Request& request); + static void encodeCanonicalRequest(network::HttpRequest& request, StringToSign& stringToSign, const uint8_t* bodyData = nullptr, uint64_t bodyLength = 0); /// Calculates the signature [[nodiscard]] static std::string createSignedRequest(const std::string& keyId, const std::string& secret, const StringToSign& stringToSign); diff --git a/include/cloud/azure_signer.hpp b/include/cloud/azure_signer.hpp index 727b87b..54ab27d 100644 --- a/include/cloud/azure_signer.hpp +++ b/include/cloud/azure_signer.hpp @@ -1,4 +1,5 @@ #pragma once +#include "network/http_request.hpp" #include #include #include @@ -17,29 +18,8 @@ namespace cloud { /// It follows the v4 docu: https://cloud.google.com/storage/docs/access-control/signing-urls-manually class AzureSigner { public: - struct Request { - /// The method - std::string method; - /// The type - std::string type; - /// The path - needs to be RFC 3986 conform - std::string path; - /// The queries - need to be RFC 3986 conform - std::map queries; - /// The headers - need to be without trailing and leading whitespaces - std::map headers; - /// The signed headers - std::string signedHeaders; - /// The payload hash - std::string payloadHash; - /// The unowned body data - const uint8_t* bodyData; - /// The unowned body length - uint64_t bodyLength; - }; - /// Builds the signed url - [[nodiscard]] static std::string createSignedRequest(const std::string& serviceAccountEmail, const std::string& privateRSA, Request& request); + [[nodiscard]] static std::string createSignedRequest(const std::string& serviceAccountEmail, const std::string& privateRSA, network::HttpRequest& request); }; //--------------------------------------------------------------------------- }; // namespace cloud diff --git a/include/cloud/gcp.hpp b/include/cloud/gcp.hpp index 2c19a43..497b648 100644 --- a/include/cloud/gcp.hpp +++ b/include/cloud/gcp.hpp @@ -92,7 +92,7 @@ class GCP : public Provider { /// Builds the http request for creating multipart put objects [[nodiscard]] std::unique_ptr> createMultiPartRequest(const std::string& filePath) const override; /// Builds the http request for completing multipart put objects - [[nodiscard]] std::unique_ptr> completeMultiPartRequest(const std::string& filePath, std::string_view uploadId, const std::vector& etags) const override; + [[nodiscard]] std::unique_ptr> completeMultiPartRequest(const std::string& filePath, std::string_view uploadId, const std::vector& etags, std::string& content) const override; /// Get the address of the server [[nodiscard]] std::string getAddress() const override; diff --git a/include/cloud/gcp_signer.hpp b/include/cloud/gcp_signer.hpp index ef909d3..73fd395 100644 --- a/include/cloud/gcp_signer.hpp +++ b/include/cloud/gcp_signer.hpp @@ -1,4 +1,5 @@ #pragma once +#include "network/http_request.hpp" #include #include #include @@ -17,36 +18,17 @@ namespace cloud { /// It follows the v4 docu: https://cloud.google.com/storage/docs/access-control/signing-urls-manually class GCPSigner { public: - struct Request { - /// The method - std::string method; - /// The type - std::string type; - /// The path - needs to be RFC 3986 conform - std::string path; - /// The queries - need to be RFC 3986 conform - std::map queries; - /// The headers - need to be without trailing and leading whitespaces - std::map headers; - /// The signed headers - std::string signedHeaders; - /// The payload hash - std::string payloadHash; - /// The unowned body data - const uint8_t* bodyData; - /// The unowned body length - uint64_t bodyLength; - }; - struct StringToSign { /// The region std::string region; /// The service std::string service; + /// The signed headers + std::string signedHeaders; }; /// Builds the signed url - [[nodiscard]] static std::string createSignedRequest(const std::string& serviceAccountEmail, const std::string& privateRSA, Request& request, const StringToSign& stringToSign); + [[nodiscard]] static std::string createSignedRequest(const std::string& serviceAccountEmail, const std::string& privateRSA, network::HttpRequest& request, StringToSign& stringToSign); }; //--------------------------------------------------------------------------- }; // namespace cloud diff --git a/include/cloud/provider.hpp b/include/cloud/provider.hpp index 67882e7..7475dba 100644 --- a/include/cloud/provider.hpp +++ b/include/cloud/provider.hpp @@ -97,7 +97,7 @@ class Provider { /// Builds the http request for creating multipart put objects [[nodiscard]] virtual std::unique_ptr> createMultiPartRequest(const std::string& /*filePath*/) const; /// Builds the http request for completing multipart put objects - [[nodiscard]] virtual std::unique_ptr> completeMultiPartRequest(const std::string& /*filePath*/, std::string_view /*uploadId*/, const std::vector& /*etags*/) const; + [[nodiscard]] virtual std::unique_ptr> completeMultiPartRequest(const std::string& /*filePath*/, std::string_view /*uploadId*/, const std::vector& /*etags*/, std::string& /*eTagsContent*/) const; /// Initialize secret virtual void initSecret(network::TaskedSendReceiver& /*sendReceiver*/) {} diff --git a/include/network/http_helper.hpp b/include/network/http_helper.hpp index 1e0bb04..4d96daf 100644 --- a/include/network/http_helper.hpp +++ b/include/network/http_helper.hpp @@ -14,7 +14,7 @@ namespace anyblob { namespace network { //--------------------------------------------------------------------------- /// Implements an helper to resolve http requests -class HTTPHelper { +class HttpHelper { public: /// The protocol detected by the helper enum class Protocol : uint8_t { diff --git a/include/network/http_message.hpp b/include/network/http_message.hpp index 9f094af..7e5bd57 100644 --- a/include/network/http_message.hpp +++ b/include/network/http_message.hpp @@ -19,7 +19,7 @@ struct HTTPMessage : public MessageTask { /// The tcp settings IOUringSocket::TCPSettings tcpSettings; /// HTTP info header - std::unique_ptr info; + std::unique_ptr info; /// The constructor HTTPMessage(OriginalMessage* sendingMessage, uint64_t chunkSize); diff --git a/include/network/http_request.hpp b/include/network/http_request.hpp new file mode 100644 index 0000000..575cfa3 --- /dev/null +++ b/include/network/http_request.hpp @@ -0,0 +1,73 @@ +#pragma once +#include +#include +#include +#include +#include +//--------------------------------------------------------------------------- +// AnyBlob - Universal Cloud Object Storage Library +// Dominik Durner, 2024 +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// SPDX-License-Identifier: MPL-2.0 +//--------------------------------------------------------------------------- +namespace anyblob { +//--------------------------------------------------------------------------- +namespace utils { +template +class DataVector; +} +//--------------------------------------------------------------------------- +namespace network { +//--------------------------------------------------------------------------- +/// Implements an helper to serialize and deserialize http requests +struct HttpRequest { + /// The method class + enum class Method : uint8_t { + GET, + PUT, + POST, + DELETE + }; + enum class Type : uint8_t { + HTTP_1_0, + HTTP_1_1 + }; + /// The queries - need to be RFC 3986 conform + std::map queries; + /// The headers - need to be without trailing and leading whitespaces + std::map headers; + /// The method + Method method; + /// The type + Type type; + /// The path - needs to be RFC 3986 conform + std::string path; + + /// Get the request method + static constexpr auto getRequestMethod(const HttpRequest& request) { + switch (request.method) { + case Method::GET: return "GET"; + case Method::PUT: return "PUT"; + case Method::POST: return "POST"; + case Method::DELETE: return "DELETE"; + default: return ""; + } + } + /// Get the request type + static constexpr auto getRequestType(const HttpRequest& request) { + switch (request.type) { + case Type::HTTP_1_0: return "HTTP/1.0"; + case Type::HTTP_1_1: return "HTTP/1.1"; + default: return ""; + } + } + /// Serialize the request + [[nodiscard]] static std::unique_ptr> serialize(const HttpRequest& request); + /// Deserialize the request + [[nodiscard]] static HttpRequest deserialize(std::string_view data); +}; +//--------------------------------------------------------------------------- +} // namespace network +} // namespace anyblob diff --git a/include/network/original_message.hpp b/include/network/original_message.hpp index b3d3662..c33663b 100644 --- a/include/network/original_message.hpp +++ b/include/network/original_message.hpp @@ -27,22 +27,22 @@ struct OriginalMessage { /// The result MessageResult result; - /// The hostname - std::string hostname; - /// The port - uint32_t port; - - /// Optional trace info - uint64_t traceId; - /// If it is a put request store the additional data /// The raw data ptr for put requests const uint8_t* putData; /// The length uint64_t putLength; + /// Optional trace info + uint64_t traceId; + + /// The hostname + std::string hostname; + /// The port + uint32_t port; + /// The constructor - OriginalMessage(std::unique_ptr> message, std::string_view hostname, uint32_t port, uint8_t* receiveBuffer = nullptr, uint64_t bufferSize = 0, uint64_t traceId = 0) : message(std::move(message)), result(receiveBuffer, bufferSize), hostname(hostname), port(port), traceId(traceId), putData(nullptr), putLength() {} + OriginalMessage(std::unique_ptr> message, std::string_view hostname, uint32_t port, uint8_t* receiveBuffer = nullptr, uint64_t bufferSize = 0, uint64_t traceId = 0) : message(std::move(message)), result(receiveBuffer, bufferSize), putData(nullptr), putLength(), traceId(traceId), hostname(hostname), port(port) {} /// The destructor virtual ~OriginalMessage() = default; diff --git a/include/network/transaction.hpp b/include/network/transaction.hpp index 1e630e4..82a1af5 100644 --- a/include/network/transaction.hpp +++ b/include/network/transaction.hpp @@ -224,7 +224,9 @@ class Transaction { } if (_multipartUploads[position].outstanding.fetch_sub(1) == 1) { if (_multipartUploads[position].state != MultipartUpload::State::Aborted) [[likely]] { - auto finished = [&callback, &initalRequestResult, this](network::MessageResult& result) { + auto contentData = std::make_unique(); + auto content = contentData.get(); + auto finished = [&callback, &initalRequestResult, contentPtr = move(contentData), this](network::MessageResult& result) mutable { if (!result.success()) { initalRequestResult.state = network::MessageState::Cancelled; initalRequestResult.originError = &result; @@ -232,7 +234,8 @@ class Transaction { _completedMultiparts++; std::forward(callback)(initalRequestResult); }; - auto originalMsg = makeCallbackMessage(std::move(finished), _provider->completeMultiPartRequest(remotePath, _multipartUploads[position].uploadId, _multipartUploads[position].eTags), _provider->getAddress(), _provider->getPort(), nullptr, 0, traceId); + auto originalMsg = makeCallbackMessage(std::move(finished), _provider->completeMultiPartRequest(remotePath, _multipartUploads[position].uploadId, _multipartUploads[position].eTags, *content), _provider->getAddress(), _provider->getPort(), nullptr, 0, traceId); + originalMsg->setPutRequestData(reinterpret_cast(content->data()), content->size()); _multipartUploads[position].messages[parts] = std::move(originalMsg); } else { auto finished = [&callback, &initalRequestResult, position, this](network::MessageResult& /*result*/) { diff --git a/src/cloud/aws.cpp b/src/cloud/aws.cpp index 813e9a5..2db8d95 100644 --- a/src/cloud/aws.cpp +++ b/src/cloud/aws.cpp @@ -61,8 +61,8 @@ Provider::Instance AWS::getInstanceDetails(network::TaskedSendReceiver& sendRece sendReceiver.sendSync(originalMsg.get()); sendReceiver.processSync(); auto& content = originalMsg->result.getDataVector(); - unique_ptr infoPtr; - auto s = network::HTTPHelper::retrieveContent(content.cdata(), content.size(), infoPtr); + unique_ptr infoPtr; + auto s = network::HttpHelper::retrieveContent(content.cdata(), content.size(), infoPtr); for (auto& instance : AWSInstance::getInstanceDetails()) if (!instance.type.compare(s)) @@ -81,8 +81,8 @@ string AWS::getInstanceRegion(network::TaskedSendReceiver& sendReceiver) sendReceiver.sendSync(originalMsg.get()); sendReceiver.processSync(); auto& content = originalMsg->result.getDataVector(); - unique_ptr infoPtr; - auto s = network::HTTPHelper::retrieveContent(content.cdata(), content.size(), infoPtr); + unique_ptr infoPtr; + auto s = network::HttpHelper::retrieveContent(content.cdata(), content.size(), infoPtr); return string(s); } //--------------------------------------------------------------------------- @@ -234,8 +234,8 @@ void AWS::initSecret(network::TaskedSendReceiver& sendReceiver) sendReceiver.sendSync(originalMsg.get()); sendReceiver.processSync(); auto& content = originalMsg->result.getDataVector(); - unique_ptr infoPtr; - auto s = network::HTTPHelper::retrieveContent(content.cdata(), content.size(), infoPtr); + unique_ptr infoPtr; + auto s = network::HttpHelper::retrieveContent(content.cdata(), content.size(), infoPtr); string iamUser; message = downloadSecret(s, iamUser); originalMsg = make_unique(move(message), getIAMAddress(), getIAMPort()); @@ -243,7 +243,7 @@ void AWS::initSecret(network::TaskedSendReceiver& sendReceiver) sendReceiver.processSync(); auto& secretContent = originalMsg->result.getDataVector(); infoPtr.reset(); - s = network::HTTPHelper::retrieveContent(secretContent.cdata(), secretContent.size(), infoPtr); + s = network::HttpHelper::retrieveContent(secretContent.cdata(), secretContent.size(), infoPtr); updateSecret(s, iamUser); _mutex.unlock(); } @@ -263,8 +263,8 @@ void AWS::initSecret(network::TaskedSendReceiver& sendReceiver) sendReceiver.sendSync(originalMsg.get()); sendReceiver.processSync(); auto& secretContent = originalMsg->result.getDataVector(); - unique_ptr infoPtr; - auto s = network::HTTPHelper::retrieveContent(secretContent.cdata(), secretContent.size(), infoPtr); + unique_ptr infoPtr; + auto s = network::HttpHelper::retrieveContent(secretContent.cdata(), secretContent.size(), infoPtr); updateSessionToken(s); _mutex.unlock(); } @@ -295,7 +295,7 @@ void AWS::initResolver(network::TaskedSendReceiver& sendReceiver) } } //--------------------------------------------------------------------------- -unique_ptr> AWS::buildRequest(AWSSigner::Request& request, string_view payload, bool initHeaders) const +unique_ptr> AWS::buildRequest(network::HttpRequest& request, const uint8_t* bodyData, uint64_t bodyLength, bool initHeaders) const // Creates and signs the request { shared_ptr secret; @@ -313,14 +313,14 @@ unique_ptr> AWS::buildRequest(AWSSigner::Request& req } } - auto canonical = AWSSigner::createCanonicalRequest(request); - AWSSigner::StringToSign stringToSign = {.request = request, .requestSHA = canonical.second, .region = _settings.region, .service = "s3"}; - auto httpHeader = request.method + " "; - httpHeader += AWSSigner::createSignedRequest(secret->keyId, secret->secret, stringToSign) + " " + request.type + "\r\n"; + AWSSigner::StringToSign stringToSign = {.request = request, .region = _settings.region, .service = "s3", .requestSHA = "", .signedHeaders = "", .payloadHash = ""}; + AWSSigner::encodeCanonicalRequest(request, stringToSign, bodyData, bodyLength); + string httpHeader = network::HttpRequest::getRequestMethod(request); + httpHeader += " "; + httpHeader += AWSSigner::createSignedRequest(secret->keyId, secret->secret, stringToSign) + " " + network::HttpRequest::getRequestType(request) + "\r\n"; for (auto& h : request.headers) httpHeader += h.first + ": " + h.second + "\r\n"; httpHeader += "\r\n"; - httpHeader += payload; return make_unique>(reinterpret_cast(httpHeader.data()), reinterpret_cast(httpHeader.data() + httpHeader.size())); } //--------------------------------------------------------------------------- @@ -330,17 +330,15 @@ unique_ptr> AWS::getRequest(const string& filePath, c if (!validKeys() || (_settings.zonal && !validSession())) return nullptr; - AWSSigner::Request request; - request.method = "GET"; - request.type = "HTTP/1.1"; + network::HttpRequest request; + request.method = network::HttpRequest::Method::GET; + request.type = network::HttpRequest::Type::HTTP_1_1; // If an endpoint is defined, we use the path-style request. The default is the usage of virtual hosted-style requests. if (_settings.endpoint.empty()) request.path = "/" + filePath; else request.path = "/" + _settings.bucket + "/" + filePath; - request.bodyData = nullptr; - request.bodyLength = 0; if (range.first != range.second) { stringstream rangeString; @@ -357,10 +355,10 @@ unique_ptr> AWS::putRequestGeneric(const string& file if (!validKeys() || (_settings.zonal && !validSession())) return nullptr; - AWSSigner::Request request; - request.method = "PUT"; - request.type = "HTTP/1.1"; - request.bodyData = reinterpret_cast(object.data()); + network::HttpRequest request; + request.method = network::HttpRequest::Method::PUT; + request.type = network::HttpRequest::Type::HTTP_1_1; + auto bodyData = reinterpret_cast(object.data()); // If an endpoint is defined, we use the path-style request. The default is the usage of virtual hosted-style requests. if (_settings.endpoint.empty()) @@ -374,11 +372,11 @@ unique_ptr> AWS::putRequestGeneric(const string& file request.queries.emplace("uploadId", uploadId); } - request.bodyLength = object.size(); + auto bodyLength = object.size(); request.headers.emplace("Host", getAddress()); - request.headers.emplace("Content-Length", to_string(request.bodyLength)); + request.headers.emplace("Content-Length", to_string(bodyLength)); - return buildRequest(request); + return buildRequest(request, bodyData, bodyLength); } //--------------------------------------------------------------------------- unique_ptr> AWS::deleteRequestGeneric(const string& filePath, string_view uploadId) const @@ -387,9 +385,9 @@ unique_ptr> AWS::deleteRequestGeneric(const string& f if (!validKeys() || (_settings.zonal && !validSession())) return nullptr; - AWSSigner::Request request; - request.method = "DELETE"; - request.type = "HTTP/1.1"; + network::HttpRequest request; + request.method = network::HttpRequest::Method::DELETE; + request.type = network::HttpRequest::Type::HTTP_1_1; // If an endpoint is defined, we use the path-style request. The default is the usage of virtual hosted-style requests. if (_settings.endpoint.empty()) @@ -402,9 +400,6 @@ unique_ptr> AWS::deleteRequestGeneric(const string& f request.queries.emplace("uploadId", uploadId); } - request.bodyData = nullptr; - request.bodyLength = 0; - return buildRequest(request); } //--------------------------------------------------------------------------- @@ -414,9 +409,9 @@ unique_ptr> AWS::createMultiPartRequest(const string& if (!validKeys() || (_settings.zonal && !validSession())) return nullptr; - AWSSigner::Request request; - request.method = "POST"; - request.type = "HTTP/1.1"; + network::HttpRequest request; + request.method = network::HttpRequest::Method::POST; + request.type = network::HttpRequest::Type::HTTP_1_1; // If an endpoint is defined, we use the path-style request. The default is the usage of virtual hosted-style requests. if (_settings.endpoint.empty()) @@ -424,20 +419,18 @@ unique_ptr> AWS::createMultiPartRequest(const string& else request.path = "/" + _settings.bucket + "/" + filePath; request.queries.emplace("uploads", ""); - request.bodyData = nullptr; - request.bodyLength = 0; request.headers.emplace("Host", getAddress()); return buildRequest(request); } //--------------------------------------------------------------------------- -unique_ptr> AWS::completeMultiPartRequest(const string& filePath, string_view uploadId, const std::vector& etags) const +unique_ptr> AWS::completeMultiPartRequest(const string& filePath, string_view uploadId, const std::vector& etags, string& content) const // Builds the http request for completing multipart upload objects { if (!validKeys() || (_settings.zonal && !validSession())) return nullptr; - string content = "\n"; + content = "\n"; for (auto i = 0ull; i < etags.size(); i++) { content += "\n"; content += to_string(i + 1); @@ -447,9 +440,9 @@ unique_ptr> AWS::completeMultiPartRequest(const strin } content += "\n"; - AWSSigner::Request request; - request.method = "POST"; - request.type = "HTTP/1.1"; + network::HttpRequest request; + request.method = network::HttpRequest::Method::POST; + request.type = network::HttpRequest::Type::HTTP_1_1; // If an endpoint is defined, we use the path-style request. The default is the usage of virtual hosted-style requests. if (_settings.endpoint.empty()) @@ -458,11 +451,11 @@ unique_ptr> AWS::completeMultiPartRequest(const strin request.path = "/" + _settings.bucket + "/" + filePath; request.queries.emplace("uploadId", uploadId); - request.bodyData = reinterpret_cast(content.data()); - request.bodyLength = content.size(); - request.headers.emplace("Content-Length", to_string(content.size())); + auto bodyData = reinterpret_cast(content.data()); + auto bodyLength = content.size(); + request.headers.emplace("Content-Length", to_string(bodyLength)); - return buildRequest(request, content); + return buildRequest(request, bodyData, bodyLength); } //--------------------------------------------------------------------------- unique_ptr> AWS::getSessionToken(string_view type) const @@ -471,20 +464,18 @@ unique_ptr> AWS::getSessionToken(string_view type) co if (!validKeys()) return nullptr; - AWSSigner::Request request; - request.method = "GET"; - request.type = "HTTP/1.1"; + network::HttpRequest request; + request.method = network::HttpRequest::Method::GET; + request.type = network::HttpRequest::Type::HTTP_1_1; request.path = "/"; request.queries.emplace("session", ""); - request.bodyData = nullptr; - request.bodyLength = 0; request.headers.emplace("Host", _settings.bucket + ".s3.amazonaws.com"); request.headers.emplace("x-amz-create-session-mode", type); request.headers.emplace("x-amz-date", testEnviornment ? fakeAMZTimestamp : buildAMZTimestamp()); if (!_secret->token.empty()) request.headers.emplace("x-amz-security-token", _secret->token); - return buildRequest(request, "", false); + return buildRequest(request, nullptr, 0, false); } //--------------------------------------------------------------------------- uint32_t AWS::getPort() const diff --git a/src/cloud/aws_signer.cpp b/src/cloud/aws_signer.cpp index b86ba5d..738796a 100644 --- a/src/cloud/aws_signer.cpp +++ b/src/cloud/aws_signer.cpp @@ -18,13 +18,13 @@ namespace cloud { //--------------------------------------------------------------------------- using namespace std; //--------------------------------------------------------------------------- -pair AWSSigner::createCanonicalRequest(Request& request) +void AWSSigner::encodeCanonicalRequest(network::HttpRequest& request, StringToSign& stringToSign, const uint8_t* bodyData, uint64_t bodyLength) // Creates the canonical request (task 1) // https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html { stringstream requestStream; // Step 1, canonicalize request method - requestStream << request.method << "\n"; + requestStream << network::HttpRequest::getRequestMethod(request) << "\n"; // Step 2, canonicalize request path; assume that path is RFC 3986 conform if (request.path.empty()) @@ -43,21 +43,21 @@ pair AWSSigner::createCanonicalRequest(Request& request) } requestStream << "\n"; - if (request.bodyLength <= (1 << 10)) { + if (bodyLength <= (1 << 10)) { // Step 6, create sha256 payload string, earlier because of https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html - request.payloadHash = utils::sha256Encode(request.bodyData, request.bodyLength); - request.headers.emplace("x-amz-content-sha256", request.payloadHash); + stringToSign.payloadHash = utils::sha256Encode(bodyData, bodyLength); + request.headers.emplace("x-amz-content-sha256", stringToSign.payloadHash); // Step 6a, content-md5 for put - if ((request.method == "PUT" || request.method == "POST")) { - auto md5 = utils::md5Encode(request.bodyData, request.bodyLength); + if ((request.method == network::HttpRequest::Method::PUT || request.method == network::HttpRequest::Method::POST)) { + auto md5 = utils::md5Encode(bodyData, bodyLength); auto encodedResult = utils::base64Encode(reinterpret_cast(md5.data()), MD5_DIGEST_LENGTH); request.headers.emplace("Content-MD5", encodedResult); } } else { string unsignedPayload = "UNSIGNED-PAYLOAD"; - request.payloadHash = "UNSIGNED-PAYLOAD"; - request.headers.emplace("x-amz-content-sha256", request.payloadHash); + stringToSign.payloadHash = "UNSIGNED-PAYLOAD"; + request.headers.emplace("x-amz-content-sha256", stringToSign.payloadHash); } // Step 4, canonicalize headers, assume no unnecessary whitespaces in header @@ -84,17 +84,17 @@ pair AWSSigner::createCanonicalRequest(Request& request) if (++it != sorted.end()) signedRequests << ";"; } - request.signedHeaders = signedRequests.str(); - requestStream << request.signedHeaders; + stringToSign.signedHeaders = signedRequests.str(); + requestStream << stringToSign.signedHeaders; } requestStream << "\n"; // Step 6 continuing - requestStream << request.payloadHash; + requestStream << stringToSign.payloadHash; // Step 7, create sha256 request string and return both the request and the sha256 request string auto requestString = requestStream.str(); - return {requestString, utils::sha256Encode(reinterpret_cast(requestString.data()), requestString.length())}; + stringToSign.requestSHA = utils::sha256Encode(reinterpret_cast(requestString.data()), requestString.length()); } //--------------------------------------------------------------------------- string AWSSigner::createStringToSign(const StringToSign& stringToSign) @@ -138,7 +138,7 @@ string AWSSigner::createSignedRequest(const string& keyId, const string& secret, // https://docs.aws.amazon.com/general/latest/gr/sigv4-add-signature-to-request.html (task 4) stringstream authorization; authorization << "AWS4-HMAC-SHA256" - << " Credential=" << keyId << "/" << date << "/" << stringToSign.region << "/" << stringToSign.service << "/" << kRequest << ", SignedHeaders=" << stringToSign.request.signedHeaders << ", Signature=" << signature; + << " Credential=" << keyId << "/" << date << "/" << stringToSign.region << "/" << stringToSign.service << "/" << kRequest << ", SignedHeaders=" << stringToSign.signedHeaders << ", Signature=" << signature; stringToSign.request.headers.emplace("Authorization", authorization.str()); diff --git a/src/cloud/azure.cpp b/src/cloud/azure.cpp index d070ed4..e3c9bcf 100644 --- a/src/cloud/azure.cpp +++ b/src/cloud/azure.cpp @@ -55,8 +55,8 @@ Provider::Instance Azure::getInstanceDetails(network::TaskedSendReceiver& sendRe sendReceiver.sendSync(originalMsg.get()); sendReceiver.processSync(); auto& content = originalMsg->result.getDataVector(); - unique_ptr infoPtr; - auto s = network::HTTPHelper::retrieveContent(content.cdata(), content.size(), infoPtr); + unique_ptr infoPtr; + auto s = network::HttpHelper::retrieveContent(content.cdata(), content.size(), infoPtr); string needle = "\"vmSize\" : \""; auto pos = s.find(needle); @@ -80,8 +80,8 @@ string Azure::getRegion(network::TaskedSendReceiver& sendReceiver) sendReceiver.sendSync(originalMsg.get()); sendReceiver.processSync(); auto& content = originalMsg->result.getDataVector(); - unique_ptr infoPtr; - auto s = network::HTTPHelper::retrieveContent(content.cdata(), content.size(), infoPtr); + unique_ptr infoPtr; + auto s = network::HttpHelper::retrieveContent(content.cdata(), content.size(), infoPtr); string needle = "\"location\" : \""; auto pos = s.find(needle); @@ -97,12 +97,10 @@ string Azure::getRegion(network::TaskedSendReceiver& sendReceiver) unique_ptr> Azure::getRequest(const string& filePath, const pair& range) const // Builds the http request for downloading a blob { - AzureSigner::Request request; - request.method = "GET"; - request.type = "HTTP/1.1"; + network::HttpRequest request; + request.method = network::HttpRequest::Method::GET; + request.type = network::HttpRequest::Type::HTTP_1_1; request.path = "/" + _settings.container + "/" + filePath; - request.bodyData = nullptr; - request.bodyLength = 0; request.headers.emplace("x-ms-date", testEnviornment ? fakeXMSTimestamp : buildXMSTimestamp()); request.headers.emplace("Host", getAddress()); @@ -114,7 +112,10 @@ unique_ptr> Azure::getRequest(const string& filePath, request.path = AzureSigner::createSignedRequest(_secret->accountName, _secret->privateKey, request); - auto httpHeader = request.method + " " + request.path + " " + request.type + "\r\n"; + string httpHeader = network::HttpRequest::getRequestMethod(request); + httpHeader += " " + request.path + " "; + httpHeader += network::HttpRequest::getRequestType(request); + httpHeader += "\r\n"; for (auto& h : request.headers) httpHeader += h.first + ": " + h.second + "\r\n"; httpHeader += "\r\n"; @@ -125,22 +126,24 @@ unique_ptr> Azure::getRequest(const string& filePath, unique_ptr> Azure::putRequest(const string& filePath, string_view object) const // Builds the http request for putting objects without the object data itself { - AzureSigner::Request request; - request.method = "PUT"; - request.type = "HTTP/1.1"; + network::HttpRequest request; + request.method = network::HttpRequest::Method::PUT; + request.type = network::HttpRequest::Type::HTTP_1_1; request.path = "/" + _settings.container + "/" + filePath; - request.bodyData = reinterpret_cast(object.data()); - request.bodyLength = object.size(); + auto bodyLength = object.size(); auto date = testEnviornment ? fakeXMSTimestamp : buildXMSTimestamp(); request.headers.emplace("x-ms-date", date); request.headers.emplace("x-ms-blob-type", "BlockBlob"); request.headers.emplace("Host", getAddress()); - request.headers.emplace("Content-Length", to_string(request.bodyLength)); + request.headers.emplace("Content-Length", to_string(bodyLength)); request.path = AzureSigner::createSignedRequest(_secret->accountName, _secret->privateKey, request); - auto httpHeader = request.method + " " + request.path + " " + request.type + "\r\n"; + string httpHeader = network::HttpRequest::getRequestMethod(request); + httpHeader += " " + request.path + " "; + httpHeader += network::HttpRequest::getRequestType(request); + httpHeader += "\r\n"; for (auto& h : request.headers) httpHeader += h.first + ": " + h.second + "\r\n"; httpHeader += "\r\n"; @@ -151,12 +154,10 @@ unique_ptr> Azure::putRequest(const string& filePath, unique_ptr> Azure::deleteRequest(const string& filePath) const // Builds the http request for deleting objects { - AzureSigner::Request request; - request.method = "DELETE"; - request.type = "HTTP/1.1"; + network::HttpRequest request; + request.method = network::HttpRequest::Method::DELETE; + request.type = network::HttpRequest::Type::HTTP_1_1; request.path = "/" + _settings.container + "/" + filePath; - request.bodyData = nullptr; - request.bodyLength = 0; auto date = testEnviornment ? fakeXMSTimestamp : buildXMSTimestamp(); request.headers.emplace("x-ms-date", date); @@ -164,7 +165,10 @@ unique_ptr> Azure::deleteRequest(const string& filePa request.path = AzureSigner::createSignedRequest(_secret->accountName, _secret->privateKey, request); - auto httpHeader = request.method + " " + request.path + " " + request.type + "\r\n"; + string httpHeader = network::HttpRequest::getRequestMethod(request); + httpHeader += " " + request.path + " "; + httpHeader += network::HttpRequest::getRequestType(request); + httpHeader += "\r\n"; for (auto& h : request.headers) httpHeader += h.first + ": " + h.second + "\r\n"; httpHeader += "\r\n"; diff --git a/src/cloud/azure_signer.cpp b/src/cloud/azure_signer.cpp index 05ef3dc..08d75c4 100644 --- a/src/cloud/azure_signer.cpp +++ b/src/cloud/azure_signer.cpp @@ -18,14 +18,14 @@ namespace cloud { //--------------------------------------------------------------------------- using namespace std; //--------------------------------------------------------------------------- -string AzureSigner::createSignedRequest(const string& accountName, const string& privateRSA, Request& request) +string AzureSigner::createSignedRequest(const string& accountName, const string& privateRSA, network::HttpRequest& request) // Creates the canonical request { auto decodedKey = utils::base64Decode(reinterpret_cast(privateRSA.data()), privateRSA.size()); stringstream requestStream; // canonicalize request method - requestStream << request.method << "\n"; + requestStream << network::HttpRequest::getRequestMethod(request) << "\n"; // Set the version request.headers.emplace("x-ms-version", "2015-02-21"); diff --git a/src/cloud/gcp.cpp b/src/cloud/gcp.cpp index b28e465..7162370 100644 --- a/src/cloud/gcp.cpp +++ b/src/cloud/gcp.cpp @@ -2,8 +2,8 @@ #include "cloud/gcp_signer.hpp" #include "network/http_helper.hpp" #include "network/original_message.hpp" -#include "network/tasked_send_receiver.hpp" #include "network/resolver.hpp" +#include "network/tasked_send_receiver.hpp" #include "utils/data_vector.hpp" #include #include @@ -49,8 +49,8 @@ Provider::Instance GCP::getInstanceDetails(network::TaskedSendReceiver& sendRece sendReceiver.sendSync(originalMsg.get()); sendReceiver.processSync(); auto& content = originalMsg->result.getDataVector(); - unique_ptr infoPtr; - auto s = network::HTTPHelper::retrieveContent(content.cdata(), content.size(), infoPtr); + unique_ptr infoPtr; + auto s = network::HttpHelper::retrieveContent(content.cdata(), content.size(), infoPtr); auto machineType = s.substr(s.find("machineTypes/")); for (auto& instance : GCPInstance::getInstanceDetails()) @@ -68,8 +68,8 @@ string GCP::getInstanceRegion(network::TaskedSendReceiver& sendReceiver) sendReceiver.sendSync(originalMsg.get()); sendReceiver.processSync(); auto& content = originalMsg->result.getDataVector(); - unique_ptr infoPtr; - auto s = network::HTTPHelper::retrieveContent(content.cdata(), content.size(), infoPtr); + unique_ptr infoPtr; + auto s = network::HttpHelper::retrieveContent(content.cdata(), content.size(), infoPtr); auto region = s.substr(s.find("zones/")); region = region.substr(0, region.size() - 2); return string(region); @@ -78,12 +78,10 @@ string GCP::getInstanceRegion(network::TaskedSendReceiver& sendReceiver) unique_ptr> GCP::getRequest(const string& filePath, const pair& range) const // Builds the http request for downloading a blob { - GCPSigner::Request request; - request.method = "GET"; - request.type = "HTTP/1.1"; + network::HttpRequest request; + request.method = network::HttpRequest::Method::GET; + request.type = network::HttpRequest::Type::HTTP_1_1; request.path = "/" + filePath; - request.bodyData = nullptr; - request.bodyLength = 0; request.queries.emplace("X-Goog-Date", testEnviornment ? fakeAMZTimestamp : buildAMZTimestamp()); request.headers.emplace("Host", getAddress()); @@ -93,10 +91,13 @@ unique_ptr> GCP::getRequest(const string& filePath, c request.headers.emplace("Range", rangeString.str()); } - GCPSigner::StringToSign stringToSign = {.region = _settings.region, .service = "storage"}; + GCPSigner::StringToSign stringToSign = {.region = _settings.region, .service = "storage", .signedHeaders = ""}; request.path = GCPSigner::createSignedRequest(_secret->serviceAccountEmail, _secret->privateKey, request, stringToSign); - auto httpHeader = request.method + " " + request.path + " " + request.type + "\r\n"; + string httpHeader = network::HttpRequest::getRequestMethod(request); + httpHeader += " " + request.path + " "; + httpHeader += network::HttpRequest::getRequestType(request); + httpHeader += "\r\n"; for (auto& h : request.headers) httpHeader += h.first + ": " + h.second + "\r\n"; httpHeader += "\r\n"; @@ -107,9 +108,9 @@ unique_ptr> GCP::getRequest(const string& filePath, c unique_ptr> GCP::putRequestGeneric(const string& filePath, string_view object, uint16_t part, string_view uploadId) const // Builds the http request for putting objects without the object data itself { - GCPSigner::Request request; - request.method = "PUT"; - request.type = "HTTP/1.1"; + network::HttpRequest request; + request.method = network::HttpRequest::Method::PUT; + request.type = network::HttpRequest::Type::HTTP_1_1; request.path = "/" + filePath; // Is it a multipart upload? @@ -118,20 +119,20 @@ unique_ptr> GCP::putRequestGeneric(const string& file request.queries.emplace("uploadId", uploadId); } - request.bodyData = reinterpret_cast(object.data()); - request.bodyLength = object.size(); - auto date = testEnviornment ? fakeAMZTimestamp : buildAMZTimestamp(); request.queries.emplace("X-Goog-Date", date); request.headers.emplace("Host", getAddress()); request.headers.emplace("Date", date); - request.headers.emplace("Content-Length", to_string(request.bodyLength)); + request.headers.emplace("Content-Length", to_string(object.size())); - GCPSigner::StringToSign stringToSign = {.region = _settings.region, .service = "storage"}; + GCPSigner::StringToSign stringToSign = {.region = _settings.region, .service = "storage", .signedHeaders = ""}; request.path = GCPSigner::createSignedRequest(_secret->serviceAccountEmail, _secret->privateKey, request, stringToSign); - auto httpHeader = request.method + " " + request.path + " " + request.type + "\r\n"; + string httpHeader = network::HttpRequest::getRequestMethod(request); + httpHeader += " " + request.path + " "; + httpHeader += network::HttpRequest::getRequestType(request); + httpHeader += "\r\n"; for (auto& h : request.headers) httpHeader += h.first + ": " + h.second + "\r\n"; httpHeader += "\r\n"; @@ -142,12 +143,10 @@ unique_ptr> GCP::putRequestGeneric(const string& file unique_ptr> GCP::deleteRequestGeneric(const string& filePath, string_view uploadId) const // Builds the http request for deleting objects { - GCPSigner::Request request; - request.method = "DELETE"; - request.type = "HTTP/1.1"; + network::HttpRequest request; + request.method = network::HttpRequest::Method::DELETE; + request.type = network::HttpRequest::Type::HTTP_1_1; request.path = "/" + filePath; - request.bodyData = nullptr; - request.bodyLength = 0; // Is it a multipart upload? if (!uploadId.empty()) { @@ -158,10 +157,13 @@ unique_ptr> GCP::deleteRequestGeneric(const string& f request.queries.emplace("X-Goog-Date", date); request.headers.emplace("Host", getAddress()); - GCPSigner::StringToSign stringToSign = {.region = _settings.region, .service = "storage"}; + GCPSigner::StringToSign stringToSign = {.region = _settings.region, .service = "storage", .signedHeaders = ""}; request.path = GCPSigner::createSignedRequest(_secret->serviceAccountEmail, _secret->privateKey, request, stringToSign); - auto httpHeader = request.method + " " + request.path + " " + request.type + "\r\n"; + string httpHeader = network::HttpRequest::getRequestMethod(request); + httpHeader += " " + request.path + " "; + httpHeader += network::HttpRequest::getRequestType(request); + httpHeader += "\r\n"; for (auto& h : request.headers) httpHeader += h.first + ": " + h.second + "\r\n"; httpHeader += "\r\n"; @@ -172,23 +174,24 @@ unique_ptr> GCP::deleteRequestGeneric(const string& f unique_ptr> GCP::createMultiPartRequest(const string& filePath) const // Builds the http request for creating multipart upload objects { - GCPSigner::Request request; - request.method = "POST"; - request.type = "HTTP/1.1"; + network::HttpRequest request; + request.method = network::HttpRequest::Method::POST; + request.type = network::HttpRequest::Type::HTTP_1_1; request.path = "/" + filePath; request.queries.emplace("uploads", ""); - request.bodyData = nullptr; - request.bodyLength = 0; auto date = testEnviornment ? fakeAMZTimestamp : buildAMZTimestamp(); request.queries.emplace("X-Goog-Date", date); request.headers.emplace("Host", getAddress()); request.headers.emplace("Date", date); - GCPSigner::StringToSign stringToSign = {.region = _settings.region, .service = "storage"}; + GCPSigner::StringToSign stringToSign = {.region = _settings.region, .service = "storage", .signedHeaders = ""}; request.path = GCPSigner::createSignedRequest(_secret->serviceAccountEmail, _secret->privateKey, request, stringToSign); - auto httpHeader = request.method + " " + request.path + " " + request.type + "\r\n"; + string httpHeader = network::HttpRequest::getRequestMethod(request); + httpHeader += " " + request.path + " "; + httpHeader += network::HttpRequest::getRequestType(request); + httpHeader += "\r\n"; for (auto& h : request.headers) httpHeader += h.first + ": " + h.second + "\r\n"; httpHeader += "\r\n"; @@ -196,10 +199,10 @@ unique_ptr> GCP::createMultiPartRequest(const string& return make_unique>(reinterpret_cast(httpHeader.data()), reinterpret_cast(httpHeader.data() + httpHeader.size())); } //--------------------------------------------------------------------------- -unique_ptr> GCP::completeMultiPartRequest(const string& filePath, string_view uploadId, const std::vector& etags) const +unique_ptr> GCP::completeMultiPartRequest(const string& filePath, string_view uploadId, const std::vector& etags, string& content) const // Builds the http request for completing multipart upload objects { - string content = "\n"; + content = "\n"; for (auto i = 0ull; i < etags.size(); i++) { content += "\n"; content += to_string(i + 1); @@ -209,13 +212,11 @@ unique_ptr> GCP::completeMultiPartRequest(const strin } content += "\n"; - GCPSigner::Request request; - request.method = "POST"; - request.type = "HTTP/1.1"; + network::HttpRequest request; + request.method = network::HttpRequest::Method::POST; + request.type = network::HttpRequest::Type::HTTP_1_1; request.path = "/" + filePath; request.queries.emplace("uploadId", uploadId); - request.bodyData = reinterpret_cast(content.data()); - request.bodyLength = content.size(); auto date = testEnviornment ? fakeAMZTimestamp : buildAMZTimestamp(); request.queries.emplace("X-Goog-Date", date); @@ -223,10 +224,14 @@ unique_ptr> GCP::completeMultiPartRequest(const strin request.headers.emplace("Date", date); request.headers.emplace("Content-Length", to_string(content.size())); - GCPSigner::StringToSign stringToSign = {.region = _settings.region, .service = "storage"}; + GCPSigner::StringToSign stringToSign = {.region = _settings.region, .service = "storage", .signedHeaders = ""}; request.path = GCPSigner::createSignedRequest(_secret->serviceAccountEmail, _secret->privateKey, request, stringToSign); - auto httpHeaderMessage = request.method + " " + request.path + " " + request.type + "\r\n"; + string httpHeaderMessage = network::HttpRequest::getRequestMethod(request); + httpHeaderMessage += " " + request.path + " "; + httpHeaderMessage += network::HttpRequest::getRequestType(request); + httpHeaderMessage += "\r\n"; + for (auto& h : request.headers) httpHeaderMessage += h.first + ": " + h.second + "\r\n"; httpHeaderMessage += "\r\n" + content; diff --git a/src/cloud/gcp_signer.cpp b/src/cloud/gcp_signer.cpp index ae171e0..0a05fcd 100644 --- a/src/cloud/gcp_signer.cpp +++ b/src/cloud/gcp_signer.cpp @@ -18,12 +18,12 @@ namespace cloud { //--------------------------------------------------------------------------- using namespace std; //--------------------------------------------------------------------------- -string GCPSigner::createSignedRequest(const string& serviceAccountEmail, const string& privateRSA, Request& request, const StringToSign& stringToSign) +string GCPSigner::createSignedRequest(const string& serviceAccountEmail, const string& privateRSA, network::HttpRequest& request, StringToSign& stringToSign) // Creates the canonical request { stringstream requestStream; // canonicalize request method - requestStream << request.method << "\n"; + requestStream << network::HttpRequest::getRequestMethod(request) << "\n"; // canonicalize request path; assume that path is RFC 3986 conform if (request.path.empty()) @@ -54,7 +54,7 @@ string GCPSigner::createSignedRequest(const string& serviceAccountEmail, const s if (++it != sorted.end()) signedRequests << ";"; } - request.signedHeaders = signedRequests.str(); + stringToSign.signedHeaders = signedRequests.str(); } sorted.clear(); @@ -68,7 +68,7 @@ string GCPSigner::createSignedRequest(const string& serviceAccountEmail, const s request.queries.emplace("X-Goog-Algorithm", "GOOG4-RSA-SHA256"); request.queries.emplace("X-Goog-Credential", serviceAccountEmail + "/" + credentialScope.str()); request.queries.emplace("X-Goog-Expires", "3600"); - request.queries.emplace("X-Goog-SignedHeaders", request.signedHeaders); + request.queries.emplace("X-Goog-SignedHeaders", stringToSign.signedHeaders); if (request.queries.size()) { auto it = request.queries.begin(); @@ -92,7 +92,7 @@ string GCPSigner::createSignedRequest(const string& serviceAccountEmail, const s } requestStream << query.str() << "\n"; requestStream << headers.str() << "\n"; - requestStream << request.signedHeaders << "\n"; + requestStream << stringToSign.signedHeaders << "\n"; requestStream << "UNSIGNED-PAYLOAD"; // create sha256 request string and return both the request and the sha256 request string diff --git a/src/cloud/provider.cpp b/src/cloud/provider.cpp index 59a7912..f7fc857 100644 --- a/src/cloud/provider.cpp +++ b/src/cloud/provider.cpp @@ -175,7 +175,7 @@ unique_ptr> Provider::createMultiPartRequest(const st return nullptr; } //--------------------------------------------------------------------------- -unique_ptr> Provider::completeMultiPartRequest(const string& /*filePath*/, string_view /*uploadId*/, const vector& /*etags*/) const +unique_ptr> Provider::completeMultiPartRequest(const string& /*filePath*/, string_view /*uploadId*/, const vector& /*etags*/, string& /*content*/) const // Builds the http request for completing multipart put objects { return nullptr; diff --git a/src/network/http_helper.cpp b/src/network/http_helper.cpp index 87de1fc..2345f83 100644 --- a/src/network/http_helper.cpp +++ b/src/network/http_helper.cpp @@ -17,7 +17,7 @@ namespace network { //--------------------------------------------------------------------------- using namespace std; //--------------------------------------------------------------------------- -HTTPHelper::Info HTTPHelper::detect(string_view header) +HttpHelper::Info HttpHelper::detect(string_view header) // Detect the protocol { static constexpr auto unknown = Info{0, 0, Protocol::Unknown, Encoding::Unknown}; @@ -68,7 +68,7 @@ HTTPHelper::Info HTTPHelper::detect(string_view header) return info; } //--------------------------------------------------------------------------- -string_view HTTPHelper::retrieveContent(const uint8_t* data, uint64_t length, unique_ptr& info) +string_view HttpHelper::retrieveContent(const uint8_t* data, uint64_t length, unique_ptr& info) // Retrieve the content without http meta info, note that this changes data { string_view sv(reinterpret_cast(data), length); @@ -79,7 +79,7 @@ string_view HTTPHelper::retrieveContent(const uint8_t* data, uint64_t length, un return {}; } //--------------------------------------------------------------------------- -bool HTTPHelper::finished(const uint8_t* data, uint64_t length, unique_ptr& info) +bool HttpHelper::finished(const uint8_t* data, uint64_t length, unique_ptr& info) // Detect end / content { if (!info) { diff --git a/src/network/http_message.cpp b/src/network/http_message.cpp index b55bb6c..7ba0556 100644 --- a/src/network/http_message.cpp +++ b/src/network/http_message.cpp @@ -94,7 +94,7 @@ MessageState HTTPMessage::execute(IOUringSocket& socket) try { // check whether finished http - if (HTTPHelper::finished(receive.data(), static_cast(receiveBufferOffset), info)) { + if (HttpHelper::finished(receive.data(), static_cast(receiveBufferOffset), info)) { socket.disconnect(request->fd, originalMessage->hostname, originalMessage->port, &tcpSettings, static_cast(sendBufferOffset + receiveBufferOffset)); originalMessage->result.size = info->length; originalMessage->result.offset = info->headerLength; diff --git a/src/network/http_request.cpp b/src/network/http_request.cpp new file mode 100644 index 0000000..5174420 --- /dev/null +++ b/src/network/http_request.cpp @@ -0,0 +1,45 @@ +#include "network/http_request.hpp" +#include "utils/data_vector.hpp" +#include "utils/utils.hpp" +//--------------------------------------------------------------------------- +// AnyBlob - Universal Cloud Object Storage Library +// Dominik Durner, 2024 +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// SPDX-License-Identifier: MPL-2.0 +//--------------------------------------------------------------------------- +namespace anyblob { +namespace network { +//--------------------------------------------------------------------------- +using namespace std; +//--------------------------------------------------------------------------- +HttpRequest HttpRequest::deserialize(string_view /*data*/) +// Deserialize the http header +{ + HttpRequest request; + return request; +} +//--------------------------------------------------------------------------- +unique_ptr> HttpRequest::serialize(const HttpRequest& request) +// Serialize an http header +{ + string httpHeader = getRequestMethod(request); + httpHeader += " "; + auto it = request.queries.begin(); + while (it != request.queries.end()) { + httpHeader += utils::encodeUrlParameters(it->first) + "=" + utils::encodeUrlParameters(it->second); + if (++it != request.queries.end()) + httpHeader += "&"; + } + httpHeader += " "; + httpHeader += getRequestType(request); + httpHeader += "\r\n"; + for (auto& h : request.headers) + httpHeader += h.first + ": " + h.second + "\r\n"; + httpHeader += "\r\n"; + return make_unique>(reinterpret_cast(httpHeader.data()), reinterpret_cast(httpHeader.data() + httpHeader.size())); +} +//--------------------------------------------------------------------------- +} // namespace network +} // namespace anyblob diff --git a/src/network/https_message.cpp b/src/network/https_message.cpp index b457a9b..5f3e085 100644 --- a/src/network/https_message.cpp +++ b/src/network/https_message.cpp @@ -101,7 +101,7 @@ MessageState HTTPSMessage::execute(IOUringSocket& socket) receive.resize(receive.size() - (chunkSize - static_cast(result))); receiveBufferOffset += result; try { - if (HTTPHelper::finished(receive.data(), static_cast(receiveBufferOffset), info)) { + if (HttpHelper::finished(receive.data(), static_cast(receiveBufferOffset), info)) { originalMessage->result.size = info->length; originalMessage->result.offset = info->headerLength; state = MessageState::TLSShutdown;