Integrating s3 express one zone API and added verifyKeyRequest method

.clang-format

@@ -1,36 +1,68 @@
 ---
-Language:        Cpp
-# BasedOnStyle:  Chromium
+Language: Cpp
 AccessModifierOffset: 0
 AlignAfterOpenBracket: Align
-AlignConsecutiveAssignments: false
-AlignConsecutiveDeclarations: false
+AlignArrayOfStructures: None
+AlignConsecutiveAssignments:
+  Enabled: false
+  AcrossEmptyLines: false
+  AcrossComments: false
+  AlignCompound: false
+  PadOperators: true
+AlignConsecutiveBitFields:
+  Enabled: false
+  AcrossEmptyLines: false
+  AcrossComments: false
+  AlignCompound: false
+  PadOperators: true
+AlignConsecutiveDeclarations:
+  Enabled: false
+  AcrossEmptyLines: false
+  AcrossComments: false
+  AlignCompound: false
+  PadOperators: true
+AlignConsecutiveMacros:
+  Enabled: false
+  AcrossEmptyLines: false
+  AcrossComments: false
+  AlignCompound: false
+  PadOperators: true
 AlignEscapedNewlines: Left
-AlignOperands:   false
-AlignTrailingComments: false
+AlignOperands: Align
+AlignTrailingComments:
+  Kind: Never
+  OverEmptyLines: 0
+AllowAllArgumentsOnNextLine: true
 AllowAllParametersOfDeclarationOnNextLine: true
-AllowShortBlocksOnASingleLine: true
+AllowShortBlocksOnASingleLine: Empty
 AllowShortCaseLabelsOnASingleLine: true
 AllowShortFunctionsOnASingleLine: All
-AllowShortIfStatementsOnASingleLine: true
+AllowShortIfStatementsOnASingleLine: OnlyFirstIf
 AllowShortLoopsOnASingleLine: true
+AllowShortLambdasOnASingleLine: All
 AlwaysBreakAfterDefinitionReturnType: None
 AlwaysBreakAfterReturnType: None
 BinPackArguments: true
 BinPackParameters: true
 BraceWrapping:
-  AfterClass:      false
-  AfterControlStatement: false
-  AfterEnum:       false
-  AfterFunction:   false
-  AfterNamespace:  false
+  AfterClass: false
+  AfterControlStatement: MultiLine
+  AfterEnum: false
+  AfterFunction: false
+  AfterNamespace: false
   AfterObjCDeclaration: false
-  AfterStruct:     false
-  AfterUnion:      false
+  AfterStruct: false
+  AfterUnion: false
   AfterExternBlock: false
-  BeforeCatch:     false
-  BeforeElse:      false
-  IndentBraces:    false
+  BeforeCatch: false
+  BeforeElse: false
+  IndentBraces: false
+  AfterCaseLabel: false
+  BeforeLambdaBody: false
+  BeforeWhile: false
+  SplitEmptyFunction: false
+  SplitEmptyRecord: false
+  SplitEmptyNamespace: false
 BreakBeforeBinaryOperators: None
 BreakBeforeBraces: Attach
 BreakBeforeInheritanceComma: false
@@ -39,36 +71,50 @@ BreakConstructorInitializersBeforeComma: false
 BreakConstructorInitializers: BeforeColon
 BreakAfterJavaFieldAnnotations: false
 BreakStringLiterals: true
-ColumnLimit:     0
-CommentPragmas:  '^ IWYU pragma:'
+ColumnLimit: 0
+CommentPragmas: "^ IWYU pragma:"
+CompactNamespaces: false
 ConstructorInitializerAllOnOneLineOrOnePerLine: true
 ConstructorInitializerIndentWidth: 4
 ContinuationIndentWidth: 4
 Cpp11BracedListStyle: true
 DerivePointerAlignment: false
-DisableFormat:   false
+DisableFormat: false
 ExperimentalAutoDetectBinPacking: true
-ForEachMacros:   [ foreach, Q_FOREACH, BOOST_FOREACH ]
+FixNamespaceComments: true
+ForEachMacros:
+  - foreach
+  - Q_FOREACH
+  - BOOST_FOREACH
 IncludeCategories:
-  - Regex:           '^"'
-    Priority:        1
-  - Regex:           '^<llvm/.*>'
-    Priority:        2
-  - Regex:           '^<.*\.h>'
-    Priority:        4
-  - Regex:           '^<'
-    Priority:        3
-  - Regex:           '.\*'
-    Priority:        5
-IncludeIsMainRegex: '([-_](test|unittest))?$'
+  - Regex: ^"
+    Priority: 1
+  - Regex: ^<llvm/.*>
+    Priority: 2
+  - Regex: ^<.*\.h>
+    Priority: 4
+  - Regex: ^<
+    Priority: 3
+  - Regex: .\*
+    Priority: 5
+IncludeIsMainRegex: ([-_](test|unittest))?$
+IndentAccessModifiers: false
 IndentCaseLabels: true
-IndentWidth:     4
+IndentCaseBlocks: false
+IndentExternBlock: AfterExternBlock
+IndentGotoLabels: true
+IndentPPDirectives: None
+IndentRequiresClause: false
+IndentWidth: 4
 IndentWrappedFunctionNames: false
+InsertBraces: false
+InsertNewlineAtEOF: true
+InsertTrailingCommas: None
 JavaScriptQuotes: Leave
 JavaScriptWrapImports: true
 KeepEmptyLinesAtTheStartOfBlocks: false
-MacroBlockBegin: ''
-MacroBlockEnd:   ''
+MacroBlockBegin: ""
+MacroBlockEnd: ""
 MaxEmptyLinesToKeep: 1
 NamespaceIndentation: None
 ObjCBlockIndentWidth: 4
@@ -81,20 +127,29 @@ PenaltyBreakString: 1000
 PenaltyExcessCharacter: 1000000
 PenaltyReturnTypeOnItsOwnLine: 200
 PointerAlignment: Left
-ReflowComments:  false
-SortIncludes:    true
+ReflowComments: false
+SortIncludes: CaseSensitive
 SpaceAfterCStyleCast: true
 SpaceAfterTemplateKeyword: true
 SpaceBeforeAssignmentOperators: true
 SpaceBeforeParens: ControlStatements
+SpaceBeforeParensOptions:
+  AfterControlStatements: true
+  AfterForeachMacros: true
+  AfterFunctionDefinitionName: false
+  AfterFunctionDeclarationName: false
+  AfterIfMacros: true
+  AfterOverloadedOperator: false
+  AfterRequiresInClause: false
+  AfterRequiresInExpression: false
+  BeforeNonEmptyParentheses: false
 SpaceInEmptyParentheses: false
 SpacesBeforeTrailingComments: 1
-SpacesInAngles:  false
+SpacesInAngles: Leave
 SpacesInContainerLiterals: false
 SpacesInCStyleCastParentheses: false
 SpacesInParentheses: false
 SpacesInSquareBrackets: false
-Standard:        Auto
-TabWidth:        4
-UseTab:          Never
-...
+Standard: Latest
+TabWidth: 4
+UseTab: Never

example/benchmark/include/network/s3_send_receiver.hpp

@@ -15,7 +15,6 @@
 #include <mutex>
 #include <thread>
 #include <unordered_map>
-#include <unistd.h>
 #include <aws/core/Aws.h>
 #include <aws/core/client/ClientConfiguration.h>
 #include <aws/core/utils/Outcome.h>
@@ -25,6 +24,7 @@
 #include <aws/s3-crt/model/GetObjectRequest.h>
 #include <aws/s3/S3Client.h>
 #include <aws/s3/model/GetObjectRequest.h>
+#include <unistd.h>
 // ---------------------------------------------------------------------------
 // AnyBlob - Universal Cloud Object Storage Library
 // Dominik Durner, 2021

example/benchmark/src/benchmark/bandwidth.cpp

@@ -3,10 +3,10 @@
 #include "cloud/aws_resolver.hpp"
 #include "cloud/azure.hpp"
 #include "cloud/gcp.hpp"
-#include "network/transaction.hpp"
 #include "network/original_message.hpp"
 #include "network/s3_send_receiver.hpp"
 #include "network/tasked_send_receiver.hpp"
+#include "network/transaction.hpp"
 #include "perfevent/PerfEvent.hpp"
 #include "utils/timer.hpp"
 #include "utils/utils.hpp"

example/simple/main.cpp

@@ -1,6 +1,6 @@
 #include "cloud/provider.hpp"
-#include "network/transaction.hpp"
 #include "network/tasked_send_receiver.hpp"
+#include "network/transaction.hpp"
 #include <cstring>
 #include <iostream>
 //---------------------------------------------------------------------------

include/cloud/aws.hpp

@@ -1,8 +1,11 @@
 #pragma once
 #include "cloud/aws_instances.hpp"
+#include "cloud/aws_signer.hpp"
 #include "cloud/provider.hpp"
 #include "utils/data_vector.hpp"
 #include <cassert>
+#include <memory>
+#include <mutex>
 #include <string>
 //---------------------------------------------------------------------------
 // AnyBlob - Universal Cloud Object Storage Library
@@ -37,6 +40,8 @@ class AWS : public Provider {
         std::string endpoint;
         /// The port
         int port = 80;
+        /// Is zonal request required?
+        bool zonal = false;
     };
 
     /// The secret
@@ -48,9 +53,9 @@ class AWS : public Provider {
         /// The secret
         std::string secret;
         /// The session token
-        std::string sessionToken;
-        /// The expieration
-        int64_t experiation;
+        std::string token;
+        /// The expiration
+        int64_t expiration;
     };
 
     /// The fake AMZ timestamp
@@ -61,10 +66,19 @@ class AWS : public Provider {
     protected:
     /// The settings
     Settings _settings;
-    /// The secret
-    std::unique_ptr<Secret> _secret;
+    /// The global secret
+    std::shared_ptr<Secret> _globalSecret;
+    /// The global session secret
+    std::shared_ptr<Secret> _globalSessionSecret;
     /// The multipart upload size
     uint64_t _multipartUploadSize = 128ull << 20;
+    /// The secret mutex
+    std::mutex _mutex;
+    /// The thread local secret
+    thread_local static std::shared_ptr<Secret> _secret;
+    /// The session secret
+    thread_local static std::shared_ptr<Secret> _sessionSecret;
+
 
     public:
     /// Get instance details
@@ -75,7 +89,7 @@ class AWS : public Provider {
     void initResolver(network::TaskedSendReceiver& sendReceiver) override;
 
     /// The constructor
-    explicit AWS(const RemoteInfo& info) : _settings({info.bucket, info.region, info.endpoint, info.port}) {
+    explicit AWS(const RemoteInfo& info) : _settings({info.bucket, info.region, info.endpoint, info.port, info.zonal}), _mutex() {
         // Check for compatible clouds
         assert(info.provider == Provider::CloudService::AWS || info.provider == Provider::CloudService::MinIO || info.provider == Provider::CloudService::Oracle || info.provider == Provider::CloudService::IBM);
         // Requires a bucket
@@ -87,27 +101,37 @@ class AWS : public Provider {
     }
     /// The custom endpoint constructor
     AWS(const RemoteInfo& info, const std::string& keyId, const std::string& key) : AWS(info) {
-        _secret = std::make_unique<Secret>();
-        _secret->keyId = keyId;
-        _secret->secret = key;
+        _globalSecret = std::make_unique<Secret>();
+        // At init it is fine to simply overwrite
+        _globalSecret->keyId = keyId;
+        _globalSecret->secret = key;
+        _secret = _globalSecret;
     }
 
     private:
     /// Initialize secret
     void initSecret(network::TaskedSendReceiver& sendReceiver) override;
+    /// Get a local copy of the global secret
+    void getSecret() override;
     /// Builds the secret http request
     [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> downloadIAMUser() const;
     /// Builds the secret http request
-    [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> downloadSecret(std::string_view content);
+    [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> downloadSecret(std::string_view content, std::string& iamUser);
     /// Update secret
-    bool updateSecret(std::string_view content);
+    bool updateSecret(std::string_view content, std::string_view iamUser);
+    /// Update session token
+    bool updateSessionToken(std::string_view content);
     /// Checks whether the keys are still valid
-    [[nodiscard]] bool validKeys() const;
+    [[nodiscard]] bool validKeys(uint32_t offset = 60) const;
+    /// Checks whether the keys are still valid
+    [[nodiscard]] bool validSession(uint32_t offset = 60) const;
     /// Get the settings
     [[nodiscard]] inline Settings getSettings() { return _settings; }
     /// Allows multipart upload if size > 0
     [[nodiscard]] uint64_t multipartUploadSize() const override { return _multipartUploadSize; }
 
+    /// Creates the generic http request and signs it
+    [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> buildRequest(AWSSigner::Request& request, std::string_view payload = "", bool initHeaders = true) const;
     /// Builds the http request for downloading a blob or listing the directory
     [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> getRequest(const std::string& filePath, const std::pair<uint64_t, uint64_t>& range) const override;
     /// Builds the http request for putting objects without the object data itself
@@ -126,7 +150,8 @@ class AWS : public Provider {
     [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> createMultiPartRequest(const std::string& filePath) const override;
     /// Builds the http request for completing multipart put objects
     [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> completeMultiPartRequest(const std::string& filePath, std::string_view uploadId, const std::vector<std::string>& etags) const override;
-
+    /// Builds the http request for getting the session token objects
+    [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> getSessionToken(std::string_view type = "ReadWrite") const;
     /// Get the address of the server
     [[nodiscard]] std::string getAddress() const override;
     /// Get the port of the server

include/cloud/gcp.hpp

@@ -75,7 +75,6 @@ class GCP : public Provider {
     /// Allows multipart upload if size > 0
     [[nodiscard]] uint64_t multipartUploadSize() const override { return 128ull << 20; }
 
-
     /// Builds the http request for downloading a blob or listing the directory
     [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> getRequest(const std::string& filePath, const std::pair<uint64_t, uint64_t>& range) const override;
     /// Builds the http request for putting objects without the object data itself
@@ -84,7 +83,7 @@ class GCP : public Provider {
     [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> putRequest(const std::string& filePath, std::string_view object) const override {
         return putRequestGeneric(filePath, object, 0, "");
     }
-   // Builds the http request for deleting an objects
+    // Builds the http request for deleting an objects
     [[nodiscard]] std::unique_ptr<utils::DataVector<uint8_t>> deleteRequest(const std::string& filePath) const override {
         return deleteRequestGeneric(filePath, "");
     }