From 6ab8fca34e0fd7c435c496ff73025f4bb8205c8a Mon Sep 17 00:00:00 2001 From: LucioDonda Date: Thu, 4 Jul 2024 14:18:42 -0300 Subject: [PATCH 01/31] feat: initial commit of queue component --- src/agent/queue/CMakeLists.txt | 20 +++ src/agent/queue/include/persistence.hpp | 43 ++++++ src/agent/queue/include/queue.hpp | 173 ++++++++++++++++++++++++ src/agent/queue/include/shared.hpp | 17 +++ src/agent/queue/src/main.cpp | 30 ++++ 5 files changed, 283 insertions(+) create mode 100644 src/agent/queue/CMakeLists.txt create mode 100644 src/agent/queue/include/persistence.hpp create mode 100644 src/agent/queue/include/queue.hpp create mode 100644 src/agent/queue/include/shared.hpp create mode 100644 src/agent/queue/src/main.cpp diff --git a/src/agent/queue/CMakeLists.txt b/src/agent/queue/CMakeLists.txt new file mode 100644 index 00000000000..1279f3cf4ef --- /dev/null +++ b/src/agent/queue/CMakeLists.txt @@ -0,0 +1,20 @@ +cmake_minimum_required(VERSION 3.18) + +# Set the project name and C++ standard +project(QueueTest LANGUAGES CXX) +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED True) + +# Find the nlohmann::json package using vcpkg +find_package(nlohmann_json REQUIRED) + +# Add the include directory for shared headers +include_directories(${CMAKE_SOURCE_DIR}/include) +include_directories(${CMAKE_SOURCE_DIR}/src) + +# Add the executable +add_executable(QueueTest + src/main.cpp) + +# Link the nlohmann::json library to your project +target_link_libraries(QueueTest PRIVATE nlohmann_json::nlohmann_json) diff --git a/src/agent/queue/include/persistence.hpp b/src/agent/queue/include/persistence.hpp new file mode 100644 index 00000000000..ee3ec700619 --- /dev/null +++ b/src/agent/queue/include/persistence.hpp @@ -0,0 +1,43 @@ +#include +#include +#include + +/** + * @brief + * + */ +class Persistence { +private: + std::string filename; + +public: + Persistence(const std::string& fname) : filename(fname) {} + + bool createOrCheckFile() { + std::cout << "createOrCheckFile " << std::endl; + return false; // Failed to create file + } + + bool writeLine(const std::string& line) { + std::cout << "writeLine: " << line << std::endl; + + return true; + } + + std::string readLine(int lineNumber) { + std::cout << "readLine number " << lineNumber << std::endl; + return ""; // Return empty string if line not found + } + + std::string readLineStartingWith(int number) { + std::cout << "readLineStartingWith: " << number << std::endl; + return ""; + } + + bool removeLine(const std::string& lineToRemove) { + + std::cout << "removeLine: " << lineToRemove << std::endl; + + return true; + } +}; \ No newline at end of file diff --git a/src/agent/queue/include/queue.hpp b/src/agent/queue/include/queue.hpp new file mode 100644 index 00000000000..c7ac4ba9764 --- /dev/null +++ b/src/agent/queue/include/queue.hpp @@ -0,0 +1,173 @@ +#include +#include +#include +#include +#include + +#include "persistence.hpp" +#include "shared.hpp" + +//TODO: move to a configuration setting +constexpr int DEFAULT_MAX = 10; + +/** + * @brief + * + */ +class PersistedQueue +{ +public: + PersistedQueue(MessageType m_queueType, int max_size = DEFAULT_MAX) + : m_queueType(m_queueType) + , m_max_size(max_size) + { + m_persistenceDest.createOrCheckFile(); + } + + const MessageType& getType() const + { + return m_queueType; + } + + int getSize() const + { + return m_size; + } + + void setSize(int m_size) + { + this->m_size = m_size; + } + + bool insertMessage(Message event) + { + m_persistenceDest.writeLine("test-line"); + m_size++; + return true; + }; + + bool removeMessage() + { + m_persistenceDest.removeLine("contenido?"); + m_size--; + return true; + }; + + Message getMessage(MessageType type) + { + return Message(type, m_persistenceDest.readLine(0)); + }; + +private: + MessageType m_queueType; // Unnecesary ? + int m_size = 0; + int m_max_size; + Persistence m_persistenceDest {DEFAULT_PERS_PATH}; +}; + +/** + * +*/ +class MultiTypeQueue +{ +private: + std::unordered_map m_queuesMap; + std::mutex m_mtx; + std::condition_variable m_cv; + int m_maxItems; + +public: + MultiTypeQueue(int size = DEFAULT_MAX) + : m_maxItems(size) + { + // Create a vector with 3 PersistedQueue elements + m_queuesMap = {{MessageType::STATE_LESS, PersistedQueue(MessageType::STATE_LESS, m_maxItems)}, + {MessageType::STATE_FULL, PersistedQueue(MessageType::STATE_FULL, m_maxItems)}, + {MessageType::COMMAND, PersistedQueue(MessageType::COMMAND, m_maxItems)}}; + } + + /** + * @brief: push message to a queue of t + * + * @param event + */ + void push(Message event) + { + std::unique_lock lock(m_mtx); + auto it = m_queuesMap.find(event.type); + if (it != m_queuesMap.end()) + { + while (it->second.getSize() == m_maxItems) + { + m_cv.wait(lock); + } + it->second.insertMessage(event); + m_cv.notify_one(); + } + else + { + // TODO: error / logging handling !!! + std::cout << "error didn't find the queue" << std::endl; + } + } + + // FIFO order + Message getLastMessage(MessageType type) + { + std::unique_lock lock(m_mtx); + auto it = m_queuesMap.find(type); + if (it != m_queuesMap.end()) + { + auto event = it->second.getMessage(type); + m_cv.notify_one(); + return event; + } + else + { + // TODO: error / logging handling !!! + std::cout << "error didn't find the queue" << std::endl; + } + } + + void popLastMessage(MessageType type) + { + std::unique_lock lock(m_mtx); + auto it = m_queuesMap.find(type); + if (it != m_queuesMap.end()) + { + // Handle return value + m_cv.notify_one(); + it->second.removeMessage(); + } + else + { + // TODO: error / logging handling !!! + std::cout << "error didn't find the queue" << std::endl; + } + } + + // void updateLast(Message event) { + // std::unique_lock lock(m_mtx); + // while (m_queuesMap[event.type].empty() ) { + // m_cv.wait(lock); + // } + // m_queuesMap[event.type].back() = event; + // m_cv.notify_one(); + // } + + bool isEmptyByType(MessageType type) + { + std::unique_lock lock(m_mtx); + auto it = m_queuesMap.find(type); + if (it != m_queuesMap.end()) + { + return it->second.getSize() == 0; + } + else + { + // TODO: error / logging handling !!! + std::cout << "error didn't find the queue" << std::endl; + } + return false; + } +}; diff --git a/src/agent/queue/include/shared.hpp b/src/agent/queue/include/shared.hpp new file mode 100644 index 00000000000..a4502788896 --- /dev/null +++ b/src/agent/queue/include/shared.hpp @@ -0,0 +1,17 @@ +#include + +constexpr char DEFAULT_PERS_PATH[] = "/home/vagrant/FILE"; + + +enum MessageType { + STATE_LESS, + STATE_FULL, + COMMAND +}; + +class Message { +public: + MessageType type; + nlohmann::json data; + Message(MessageType t, std::string d) : type(t), data(d) {} +}; \ No newline at end of file diff --git a/src/agent/queue/src/main.cpp b/src/agent/queue/src/main.cpp new file mode 100644 index 00000000000..e7390b176bb --- /dev/null +++ b/src/agent/queue/src/main.cpp @@ -0,0 +1,30 @@ +#include + +#include "queue.hpp" + +int main() +{ + MultiTypeQueue queue(3); // with 2 it get's blocked + + // Push events + queue.push(Message(STATE_LESS, R"({"Data" : "for STATE_LESS"})")); + queue.push(Message(STATE_LESS, R"({"Data" : "for STATE_LESS"})")); + queue.push(Message(STATE_LESS, R"({"Data" : "for STATE_LESS"})")); + queue.push(Message(COMMAND, R"({"Data" : "for COMMAND"})")); + + // Retrieve events + Message event1 = queue.getLastMessage(MessageType::STATE_LESS); + std::cout << "Retrieved event 1: " << event1.data << std::endl; + + Message event2 = queue.getLastMessage(MessageType::STATE_LESS); + std::cout << "Retrieved event 2: " << event2.data << std::endl; + + Message event3 = queue.getLastMessage(MessageType::STATE_LESS); + std::cout << "Retrieved event 3: " << event3.data << std::endl; + + // Check if a specific queue is empty + bool isEmptyType1 = queue.isEmptyByType(MessageType::STATE_FULL); + std::cout << "Is STATE_LESS queue empty? " << (isEmptyType1 ? "Yes" : "No") << std::endl; + + return 0; +} \ No newline at end of file From b710c15120adba0b3030602ef49c1cb69ce532c6 Mon Sep 17 00:00:00 2001 From: LucioDonda Date: Fri, 5 Jul 2024 19:20:42 -0300 Subject: [PATCH 02/31] feat: code style, test and additional methods --- src/agent/queue/CMakeLists.txt | 6 +- src/agent/queue/include/persistence.hpp | 128 +++++++++++++--- src/agent/queue/include/queue.hpp | 195 +++++++++++++++++++----- src/agent/queue/include/shared.hpp | 30 +++- src/agent/queue/src/main.cpp | 70 ++++++--- src/agent/queue/tests/CMakeLists.txt | 15 ++ src/agent/queue/tests/test_queue.cpp | 18 +++ 7 files changed, 379 insertions(+), 83 deletions(-) create mode 100644 src/agent/queue/tests/CMakeLists.txt create mode 100644 src/agent/queue/tests/test_queue.cpp diff --git a/src/agent/queue/CMakeLists.txt b/src/agent/queue/CMakeLists.txt index 1279f3cf4ef..4add5209838 100644 --- a/src/agent/queue/CMakeLists.txt +++ b/src/agent/queue/CMakeLists.txt @@ -10,10 +10,12 @@ find_package(nlohmann_json REQUIRED) # Add the include directory for shared headers include_directories(${CMAKE_SOURCE_DIR}/include) -include_directories(${CMAKE_SOURCE_DIR}/src) + +# TODO: test define +add_subdirectory(tests) # Add the executable -add_executable(QueueTest +add_executable(QueueTest src/main.cpp) # Link the nlohmann::json library to your project diff --git a/src/agent/queue/include/persistence.hpp b/src/agent/queue/include/persistence.hpp index ee3ec700619..9a4079d5f67 100644 --- a/src/agent/queue/include/persistence.hpp +++ b/src/agent/queue/include/persistence.hpp @@ -1,43 +1,137 @@ #include +#include +#include #include #include + +using json = nlohmann::json; + + /** * @brief * */ -class Persistence { +class Persistence +{ private: std::string filename; public: - Persistence(const std::string& fname) : filename(fname) {} + Persistence(const std::string& fname) + : filename(fname) + { + } - bool createOrCheckFile() { + bool createOrCheckFile() + { std::cout << "createOrCheckFile " << std::endl; - return false; // Failed to create file + std::ifstream file(filename); + if (file.good()) { + file.close(); + return true; // File exists + } else { + std::ofstream newFile(filename); + if (newFile.is_open()) { + newFile.close(); + return true; // File created successfully + } + return false; // Failed to create file + } } - bool writeLine(const std::string& line) { - std::cout << "writeLine: " << line << std::endl; - - return true; + bool writeLine(const json data) + { + std::cout << "writeLine: " << data << std::endl; + std::ofstream file(filename, std::ios::app); + if (file.is_open()) { + file << to_string(data) << std::endl; + file.close(); + return true; + } + return false; } - std::string readLine(int lineNumber) { - std::cout << "readLine number " << lineNumber << std::endl; - return ""; // Return empty string if line not found + /** + * @brief Reads and return N lines from the queue + * + * @param linesQuantity + * @return std::string + */ + json readNLines(int linesQuantity) + { + std::cout << "readNLines number " << linesQuantity << std::endl; + std::ifstream file(filename); + std::string line; + int currentLine = 0; + json outputArray = {}; + + while (std::getline(file, line)) { + currentLine++; + outputArray.emplace_back(line); + std::cout << "output: " << outputArray << std::endl; + if (currentLine == linesQuantity) { + file.close(); + return outputArray; + } + } + + // less messages than asked for + file.close(); + return outputArray; } - std::string readLineStartingWith(int number) { - std::cout << "readLineStartingWith: " << number << std::endl; - return ""; + /** + * @brief Pops N lines from the queue + * + * @param linesQuantity + * @return int number of lines actually popped + */ + int removeNLines(int linesQuantity) + { + std::cout << "removeNLines: " << linesQuantity << std::endl; + std::vector all_lines; + std::ifstream ifs(filename); + std::string line; + + // Read all lines from the file + while (std::getline(ifs, line)) { + all_lines.push_back(line); + } + ifs.close(); + + // Check if n is greater than the number of lines in the file + if (linesQuantity > all_lines.size()) { + linesQuantity = all_lines.size(); + } + + // Create a vector for the lines to return and remove + std::vector last_n_lines(all_lines.begin(), all_lines.begin() + linesQuantity); + all_lines.erase(all_lines.begin(), all_lines.begin() + linesQuantity); + + // Write the remaining lines back to the file + std::ofstream ofs(filename, std::ios::out | std::ios::trunc); + for (const auto& remaining_line : all_lines) { + ofs << remaining_line << std::endl; + } + ofs.close(); + + return linesQuantity; } - bool removeLine(const std::string& lineToRemove) { + int getItems() const + { + std::cout << "getItems " << std::endl; + std::vector all_lines; + std::ifstream ifs(filename); + std::string line; - std::cout << "removeLine: " << lineToRemove << std::endl; + // Read all lines from the file + while (std::getline(ifs, line)) { + all_lines.push_back(line); + } + ifs.close(); - return true; + return all_lines.size(); } }; \ No newline at end of file diff --git a/src/agent/queue/include/queue.hpp b/src/agent/queue/include/queue.hpp index c7ac4ba9764..5d12aff4967 100644 --- a/src/agent/queue/include/queue.hpp +++ b/src/agent/queue/include/queue.hpp @@ -2,17 +2,19 @@ #include #include #include +#include +#include #include #include "persistence.hpp" #include "shared.hpp" -//TODO: move to a configuration setting +// TODO: move to a configuration setting constexpr int DEFAULT_MAX = 10; /** - * @brief - * + * @brief + * */ class PersistedQueue { @@ -24,68 +26,166 @@ class PersistedQueue m_persistenceDest.createOrCheckFile(); } + // Delete copy constructor + PersistedQueue(const PersistedQueue&) = delete; + + // Delete copy assignment operator + PersistedQueue& operator=(const PersistedQueue&) = delete; + + // Delete move constructor + PersistedQueue(PersistedQueue&&) = delete; + + // Delete move assignment operator + PersistedQueue& operator=(PersistedQueue&&) = delete; + + // TODO + ~PersistedQueue() { + // m_persistenceDest.close(); + }; + + /** + * @brief Get the Type object + * + * @return const MessageType& + */ const MessageType& getType() const { return m_queueType; } + int getItemsAvailable() const + { + return m_persistenceDest.getItems(); + } + + /** + * @brief Get the Size object + * + * @return int + */ int getSize() const { return m_size; } + /** + * @brief Set the Size object + * + * @param m_size + */ void setSize(int m_size) { this->m_size = m_size; } + /** + * @brief + * + * @param event + * @return true + * @return false + */ bool insertMessage(Message event) { - m_persistenceDest.writeLine("test-line"); + std::unique_lock lock(m_mtx); + m_persistenceDest.writeLine(event.data); m_size++; + m_cv.notify_one(); return true; }; + /** + * @brief + * + * @return true + * @return false + */ bool removeMessage() { - m_persistenceDest.removeLine("contenido?"); - m_size--; + std::unique_lock lock(m_mtx); + auto linesRemoved = m_persistenceDest.removeNLines(1); + m_size = m_size - linesRemoved; return true; }; + bool removeNMessages(int qttyMessages) + { + std::unique_lock lock(m_mtx); + auto linesRemoved = m_persistenceDest.removeNLines(qttyMessages); + m_size = m_size - linesRemoved; + return true; + }; + + /** + * @brief Get the Message object + * + * @param type + * @return Message + */ Message getMessage(MessageType type) { - return Message(type, m_persistenceDest.readLine(0)); + std::unique_lock lock(m_mtx); + return Message(type, m_persistenceDest.readNLines(1)); + }; + + Message getNMessages(MessageType type, int n) + { + std::unique_lock lock(m_mtx); + return Message(type, m_persistenceDest.readNLines(n)); }; + bool empty() + { + return m_size == 0; + } + private: - MessageType m_queueType; // Unnecesary ? + MessageType m_queueType; int m_size = 0; int m_max_size; - Persistence m_persistenceDest {DEFAULT_PERS_PATH}; + Persistence m_persistenceDest {DEFAULT_PERS_PATH + MessageTypeName.at(m_queueType)}; + std::mutex m_mtx; + std::condition_variable m_cv; }; /** - * -*/ + * @brief + * + */ class MultiTypeQueue { private: - std::unordered_map m_queuesMap; - std::mutex m_mtx; - std::condition_variable m_cv; + // std::condition_variable m_cv; + std::unordered_map> m_queuesMap; int m_maxItems; public: + // Create a vector with 3 PersistedQueue elements MultiTypeQueue(int size = DEFAULT_MAX) : m_maxItems(size) { - // Create a vector with 3 PersistedQueue elements - m_queuesMap = {{MessageType::STATE_LESS, PersistedQueue(MessageType::STATE_LESS, m_maxItems)}, - {MessageType::STATE_FULL, PersistedQueue(MessageType::STATE_FULL, m_maxItems)}, - {MessageType::COMMAND, PersistedQueue(MessageType::COMMAND, m_maxItems)}}; + // Populate the map inside the constructor body + m_queuesMap[MessageType::STATE_LESS] = std::make_unique(MessageType::STATE_LESS, m_maxItems); + m_queuesMap[MessageType::STATE_FULL] = std::make_unique(MessageType::STATE_FULL, m_maxItems); + m_queuesMap[MessageType::COMMAND] = std::make_unique(MessageType::COMMAND, m_maxItems); } + // Delete copy constructor + MultiTypeQueue(const MultiTypeQueue&) = delete; + + // Delete copy assignment operator + MultiTypeQueue& operator=(const MultiTypeQueue&) = delete; + + // Delete move constructor + MultiTypeQueue(MultiTypeQueue&&) = delete; + + // Delete move assignment operator + MultiTypeQueue& operator=(MultiTypeQueue&&) = delete; + + // TODO + ~MultiTypeQueue() { + // m_queuesMap; + }; /** * @brief: push message to a queue of t * @@ -93,16 +193,14 @@ class MultiTypeQueue */ void push(Message event) { - std::unique_lock lock(m_mtx); auto it = m_queuesMap.find(event.type); if (it != m_queuesMap.end()) { - while (it->second.getSize() == m_maxItems) + while (it->second->getSize() == m_maxItems) { - m_cv.wait(lock); + std::cout << "waiting" << std::endl; } - it->second.insertMessage(event); - m_cv.notify_one(); + it->second->insertMessage(event); } else { @@ -112,32 +210,41 @@ class MultiTypeQueue } // FIFO order + /** + * @brief Get the Last Message object + * + * @param type + * @return Message + */ Message getLastMessage(MessageType type) { - std::unique_lock lock(m_mtx); + Message result(type,{}); auto it = m_queuesMap.find(type); if (it != m_queuesMap.end()) { - auto event = it->second.getMessage(type); - m_cv.notify_one(); - return event; + result = it->second->getMessage(type); } else { // TODO: error / logging handling !!! std::cout << "error didn't find the queue" << std::endl; } + return result; } + /** + * @brief + * + * @param type + */ void popLastMessage(MessageType type) { - std::unique_lock lock(m_mtx); auto it = m_queuesMap.find(type); if (it != m_queuesMap.end()) { + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); // Handle return value - m_cv.notify_one(); - it->second.removeMessage(); + it->second->removeMessage(); } else { @@ -146,22 +253,36 @@ class MultiTypeQueue } } - // void updateLast(Message event) { - // std::unique_lock lock(m_mtx); - // while (m_queuesMap[event.type].empty() ) { - // m_cv.wait(lock); + // TODO + /** + * @brief + * + * @param event + */ + // void updateLast(Message event) + // { + // std::unique_lock lock(m_mtx); + // while (m_queuesMap[event.type].empty()) + // { + // m_cv.wait(lock); // } - // m_queuesMap[event.type].back() = event; + // m_queuesMap[event.type] = event; // m_cv.notify_one(); // } + /** + * @brief + * + * @param type + * @return true + * @return false + */ bool isEmptyByType(MessageType type) { - std::unique_lock lock(m_mtx); auto it = m_queuesMap.find(type); if (it != m_queuesMap.end()) { - return it->second.getSize() == 0; + return it->second->empty(); } else { diff --git a/src/agent/queue/include/shared.hpp b/src/agent/queue/include/shared.hpp index a4502788896..dc24015ae17 100644 --- a/src/agent/queue/include/shared.hpp +++ b/src/agent/queue/include/shared.hpp @@ -1,17 +1,37 @@ -#include +#include -constexpr char DEFAULT_PERS_PATH[] = "/home/vagrant/FILE"; +#include +constexpr char DEFAULT_PERS_PATH[] = "/home/vagrant/FILE_"; -enum MessageType { +/** + * @brief + * + */ +enum MessageType { STATE_LESS, STATE_FULL, COMMAND }; +/** + * @brief + * + */ +std::map MessageTypeName { + {MessageType::STATE_LESS, "STATE_LESS"}, + {MessageType::STATE_FULL, "STATE_FULL"}, + {MessageType::COMMAND, "COMMAND"}, +}; + + +/** + * @brief + * + */ class Message { public: MessageType type; nlohmann::json data; - Message(MessageType t, std::string d) : type(t), data(d) {} -}; \ No newline at end of file + Message(MessageType t, nlohmann::json d) : type(t), data(d) {} +}; diff --git a/src/agent/queue/src/main.cpp b/src/agent/queue/src/main.cpp index e7390b176bb..fa66f0962ec 100644 --- a/src/agent/queue/src/main.cpp +++ b/src/agent/queue/src/main.cpp @@ -1,30 +1,56 @@ #include +#include +#include #include "queue.hpp" int main() { - MultiTypeQueue queue(3); // with 2 it get's blocked - - // Push events - queue.push(Message(STATE_LESS, R"({"Data" : "for STATE_LESS"})")); - queue.push(Message(STATE_LESS, R"({"Data" : "for STATE_LESS"})")); - queue.push(Message(STATE_LESS, R"({"Data" : "for STATE_LESS"})")); - queue.push(Message(COMMAND, R"({"Data" : "for COMMAND"})")); - - // Retrieve events - Message event1 = queue.getLastMessage(MessageType::STATE_LESS); - std::cout << "Retrieved event 1: " << event1.data << std::endl; - - Message event2 = queue.getLastMessage(MessageType::STATE_LESS); - std::cout << "Retrieved event 2: " << event2.data << std::endl; - - Message event3 = queue.getLastMessage(MessageType::STATE_LESS); - std::cout << "Retrieved event 3: " << event3.data << std::endl; - - // Check if a specific queue is empty - bool isEmptyType1 = queue.isEmptyByType(MessageType::STATE_FULL); - std::cout << "Is STATE_LESS queue empty? " << (isEmptyType1 ? "Yes" : "No") << std::endl; + MultiTypeQueue queue(10); // with 2 it get's blocked + + auto consumer1 = [&](int &count) { + // std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + int item; + for (int i = 0; i < count; ++i) + { + queue.popLastMessage(MessageType::STATE_LESS); + std::cout << "Popping event 1: " << std::endl; + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + }; + + auto consumer2 = [&](int &count) { + int item; + for (int i = 0; i < count; ++i) + { + queue.popLastMessage(MessageType::STATE_FULL); + std::cout << "Popping event 2: " << std::endl; + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + }; + + + auto producer = [&](int &count) { + for (int i = 0; i < count; ++i) + { + auto message = R"({"Data" : "for STATE_LESS)" + std::to_string(i) + R"("})"; + queue.push(Message(MessageType::STATE_LESS, message)); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); // Simulate work + } + }; + + int items_to_insert = 5; + int items_to_consume = 2; + + producer(items_to_insert); + + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + + std::thread consumer_thread1(consumer1, std::ref(items_to_consume)); + std::thread producer_thread1(consumer2, std::ref(items_to_consume)); + + producer_thread1.join(); + consumer_thread1.join(); return 0; -} \ No newline at end of file +} diff --git a/src/agent/queue/tests/CMakeLists.txt b/src/agent/queue/tests/CMakeLists.txt new file mode 100644 index 00000000000..847bda00bd7 --- /dev/null +++ b/src/agent/queue/tests/CMakeLists.txt @@ -0,0 +1,15 @@ +# enable_testing() + +# Link against Google Test +find_package(GTest REQUIRED) +# find_package(GTest CONFIG REQUIRED) + +include_directories(${GTEST_INCLUDE_DIRS}) + +# Create a test executable +add_executable(test_queue test_queue.cpp) +# add_test(AllTestsInMain main) + +# Link the test executable against Google Test +target_link_libraries(test_queue ${GTEST_LIBRARIES} ${GTEST_MAIN_LIBRARIES}) +# target_link_libraries(main PRIVATE GTest::gtest GTest::gtest_main GTest::gmock GTest::gmock_main) diff --git a/src/agent/queue/tests/test_queue.cpp b/src/agent/queue/tests/test_queue.cpp new file mode 100644 index 00000000000..e69d0e6c60b --- /dev/null +++ b/src/agent/queue/tests/test_queue.cpp @@ -0,0 +1,18 @@ +#include +#include "../include/queue.hpp" +// #include "../include/shared.hpp" + +// Example test case +TEST(test_queue, Base) { + // EXPECT_EQ(result.z, 9); + MultiTypeQueue queue(10); // with 2 it get's blocked + int items_to_insert = 5; + auto dataContent = R"({"Data" : "for STATE_LESS)" + std::to_string(0) + R"("})"; + queue.push(Message(MessageType::STATE_LESS, dataContent)); + auto messageResponse = queue.getLastMessage(MessageType::STATE_LESS); +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} From 7f4b7acd994a38cde2085ad128ef8e286f69eeb1 Mon Sep 17 00:00:00 2001 From: LucioDonda Date: Wed, 10 Jul 2024 17:09:35 -0300 Subject: [PATCH 03/31] feat: add corrections on maps and helper functions plus tests WIP --- src/agent/queue/include/persistence.hpp | 1 - src/agent/queue/include/queue.hpp | 44 +++-- src/agent/queue/include/shared.hpp | 37 +++- src/agent/queue/src/main.cpp | 2 +- src/agent/queue/tests/CMakeLists.txt | 3 +- src/agent/queue/tests/main.cpp | 6 + src/agent/queue/tests/queue_test.cpp | 243 ++++++++++++++++++++++++ src/agent/queue/tests/queue_test.hpp | 27 +++ src/agent/queue/tests/test_queue.cpp | 18 -- 9 files changed, 337 insertions(+), 44 deletions(-) create mode 100644 src/agent/queue/tests/main.cpp create mode 100644 src/agent/queue/tests/queue_test.cpp create mode 100644 src/agent/queue/tests/queue_test.hpp delete mode 100644 src/agent/queue/tests/test_queue.cpp diff --git a/src/agent/queue/include/persistence.hpp b/src/agent/queue/include/persistence.hpp index 9a4079d5f67..ef4c562faac 100644 --- a/src/agent/queue/include/persistence.hpp +++ b/src/agent/queue/include/persistence.hpp @@ -25,7 +25,6 @@ class Persistence bool createOrCheckFile() { - std::cout << "createOrCheckFile " << std::endl; std::ifstream file(filename); if (file.good()) { file.close(); diff --git a/src/agent/queue/include/queue.hpp b/src/agent/queue/include/queue.hpp index 5d12aff4967..3f67d402334 100644 --- a/src/agent/queue/include/queue.hpp +++ b/src/agent/queue/include/queue.hpp @@ -143,6 +143,7 @@ class PersistedQueue MessageType m_queueType; int m_size = 0; int m_max_size; + //TODO: make DEFAULT_PERS_PATH configurable Persistence m_persistenceDest {DEFAULT_PERS_PATH + MessageTypeName.at(m_queueType)}; std::mutex m_mtx; std::condition_variable m_cv; @@ -193,14 +194,13 @@ class MultiTypeQueue */ void push(Message event) { - auto it = m_queuesMap.find(event.type); - if (it != m_queuesMap.end()) + if (m_queuesMap.contains(event.type)) { - while (it->second->getSize() == m_maxItems) + while (m_queuesMap[event.type]->getSize() == m_maxItems) { std::cout << "waiting" << std::endl; } - it->second->insertMessage(event); + m_queuesMap[event.type]->insertMessage(event); } else { @@ -219,10 +219,9 @@ class MultiTypeQueue Message getLastMessage(MessageType type) { Message result(type,{}); - auto it = m_queuesMap.find(type); - if (it != m_queuesMap.end()) + if (m_queuesMap.contains(type)) { - result = it->second->getMessage(type); + result = m_queuesMap[type]->getMessage(type); } else { @@ -239,12 +238,11 @@ class MultiTypeQueue */ void popLastMessage(MessageType type) { - auto it = m_queuesMap.find(type); - if (it != m_queuesMap.end()) + if (m_queuesMap.contains(type)) { std::this_thread::sleep_for(std::chrono::milliseconds(1000)); // Handle return value - it->second->removeMessage(); + m_queuesMap[type]->removeMessage(); } else { @@ -279,10 +277,30 @@ class MultiTypeQueue */ bool isEmptyByType(MessageType type) { - auto it = m_queuesMap.find(type); - if (it != m_queuesMap.end()) + if (m_queuesMap.contains(type)) { - return it->second->empty(); + return m_queuesMap[type]->empty(); + } + else + { + // TODO: error / logging handling !!! + std::cout << "error didn't find the queue" << std::endl; + } + return false; + } + + /** + * @brief Get the Items By Type object + * + * @param type + * @return true + * @return false + */ + bool getItemsByType(MessageType type) + { + if (m_queuesMap.contains(type)) + { + return m_queuesMap[type]->getItemsAvailable(); } else { diff --git a/src/agent/queue/include/shared.hpp b/src/agent/queue/include/shared.hpp index dc24015ae17..47382aeb8d7 100644 --- a/src/agent/queue/include/shared.hpp +++ b/src/agent/queue/include/shared.hpp @@ -5,18 +5,19 @@ constexpr char DEFAULT_PERS_PATH[] = "/home/vagrant/FILE_"; /** - * @brief - * + * @brief + * */ -enum MessageType { +enum MessageType +{ STATE_LESS, STATE_FULL, COMMAND }; /** - * @brief - * + * @brief + * */ std::map MessageTypeName { {MessageType::STATE_LESS, "STATE_LESS"}, @@ -24,14 +25,30 @@ std::map MessageTypeName { {MessageType::COMMAND, "COMMAND"}, }; - /** - * @brief - * + * @brief + * */ -class Message { +class Message +{ public: MessageType type; nlohmann::json data; - Message(MessageType t, nlohmann::json d) : type(t), data(d) {} + Message(MessageType t, nlohmann::json d) + : type(t) + , data(d) + { + } + + /** + * @brief + * + * @param other + * @return true + * @return false + */ + bool operator==(const Message& other) const + { + return (this->type == other.type) && (this->data == other.data); + } }; diff --git a/src/agent/queue/src/main.cpp b/src/agent/queue/src/main.cpp index fa66f0962ec..a3d32a12d54 100644 --- a/src/agent/queue/src/main.cpp +++ b/src/agent/queue/src/main.cpp @@ -6,7 +6,7 @@ int main() { - MultiTypeQueue queue(10); // with 2 it get's blocked + MultiTypeQueue queue(10); auto consumer1 = [&](int &count) { // std::this_thread::sleep_for(std::chrono::milliseconds(1000)); diff --git a/src/agent/queue/tests/CMakeLists.txt b/src/agent/queue/tests/CMakeLists.txt index 847bda00bd7..c3e064cb6e3 100644 --- a/src/agent/queue/tests/CMakeLists.txt +++ b/src/agent/queue/tests/CMakeLists.txt @@ -6,8 +6,9 @@ find_package(GTest REQUIRED) include_directories(${GTEST_INCLUDE_DIRS}) +file(GLOB QUEUE_UNIT_TEST_SRC "*.cpp") # Create a test executable -add_executable(test_queue test_queue.cpp) +add_executable(test_queue ${QUEUE_UNIT_TEST_SRC}) # add_test(AllTestsInMain main) # Link the test executable against Google Test diff --git a/src/agent/queue/tests/main.cpp b/src/agent/queue/tests/main.cpp new file mode 100644 index 00000000000..5ebbc761adf --- /dev/null +++ b/src/agent/queue/tests/main.cpp @@ -0,0 +1,6 @@ +#include + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/src/agent/queue/tests/queue_test.cpp b/src/agent/queue/tests/queue_test.cpp new file mode 100644 index 00000000000..f52c757c620 --- /dev/null +++ b/src/agent/queue/tests/queue_test.cpp @@ -0,0 +1,243 @@ +#include + +#include "../include/queue.hpp" + +#include "queue_test.hpp" + +#define BIG_QUEUE_QTTY 10 +#define SMALL_QUEUE_QTTY 10 + +// TODO: redo for additional stores +bool cleanPersistence() +{ + + for (auto messageType : MessageTypeName) + { + auto filePath = DEFAULT_PERS_PATH + messageType.second; + std::error_code ec; + if (std::filesystem::remove(filePath, ec)) + { + continue; + } + else + { + std::ifstream file(filePath); + if (file.good()) + { + file.close(); + std::cerr << "Error removing file: " << ec.message() << std::endl; + return true; // File exists + } + else + { + continue; + } + } + } + return true; +} + +// TODO: rework after double check the output of persistence +bool compareJsonStrings(const nlohmann::json& jsonValue1, const nlohmann::json& jsonValue2) +{ + + std::string str1 = jsonValue1.get(); + std::string str2 = jsonValue2.get(); + // Remove outer quotes from output if present + auto removeQuotes = [](std::string& inputString) + { + if (inputString.front() == '"' && inputString.back() == '"') + { + inputString = inputString.substr(1, inputString.length() - 2); + } + }; + + std::cout << "str1: " << str1 << std::endl; + std::cout << "str2: " << str2 << std::endl; + + auto unescapeJsonString = [](std::string input) -> std::string + { + std::string result; + bool escapeNext = false; + for (char c : input) + { + if (escapeNext) + { + if (c == '"') + { + result += '"'; + } + else + { + result += '\\'; + result += c; + } + escapeNext = false; + } + else + { + if (c == '\\') + { + escapeNext = true; + } + else + { + result += c; + } + } + } + return result; + }; + + unescapeJsonString(str1); + unescapeJsonString(str2); + + try + { + // Parse both strings into json objects + nlohmann::json json1 = nlohmann::json::parse(str1); + nlohmann::json json2 = nlohmann::json::parse(str2); + + // Compare the json objects + return json1 == json2; + } + catch (const nlohmann::json::parse_error& e) + { + std::cerr << "JSON parsing error: " << e.what() << std::endl; + return false; + } +} + +void QueueTest::SetUp() +{ + EXPECT_TRUE(cleanPersistence()); +}; + +void QueueTest::TearDown() {}; + +// Push, get and check the queue is not empty +TEST_F(QueueTest, BasicPushGetNotEmpty) +{ + MultiTypeQueue queue(BIG_QUEUE_QTTY); + const MessageType messageTtpe {MessageType::STATE_LESS}; + const nlohmann::json dataContent = R"({"Data" : "for STATE_LESS)" + std::to_string(0) + R"("})"; + const Message messageToSend {messageTtpe, dataContent}; + + queue.push(messageToSend); + auto messageResponse = queue.getLastMessage(MessageType::STATE_LESS); + + auto typeSend = messageResponse.type; + auto typeReceived = messageResponse.type; + + EXPECT_TRUE(typeSend == typeReceived); + EXPECT_TRUE(compareJsonStrings(messageResponse.data[0], messageToSend.data)); + EXPECT_FALSE(queue.isEmptyByType(MessageType::STATE_LESS)); +} + +// push and pop on a full queue +TEST_F(QueueTest, BasicPushPopEmpty) +{ + MultiTypeQueue queue(BIG_QUEUE_QTTY); + const MessageType messageTtpe {MessageType::STATE_LESS}; + const nlohmann::json dataContent = R"({"Data" : "for STATE_LESS)" + std::to_string(0) + R"("})"; + const Message messageToSend {messageTtpe, dataContent}; + + queue.push(messageToSend); + auto messageResponse = queue.getLastMessage(MessageType::STATE_LESS); + + auto typeSend = messageResponse.type; + auto typeReceived = messageResponse.type; + + EXPECT_TRUE(typeSend == typeReceived); + EXPECT_TRUE(compareJsonStrings(messageResponse.data[0], messageToSend.data)); + EXPECT_TRUE(queue.isEmptyByType(MessageType::STATE_LESS)); +} + +// Push, pop and check the queue is empty +TEST_F(QueueTest, BasicPushPop) +{ + MultiTypeQueue queue(BIG_QUEUE_QTTY); + const MessageType messageTtpe {MessageType::STATE_LESS}; + const nlohmann::json dataContent = R"({"Data" : "for STATE_LESS)" + std::to_string(0) + R"("})"; + const Message messageToSend {messageTtpe, dataContent}; + + queue.push(messageToSend); + EXPECT_FALSE(queue.isEmptyByType(MessageType::STATE_LESS)); + + queue.popLastMessage(MessageType::STATE_LESS); + EXPECT_TRUE(queue.isEmptyByType(MessageType::STATE_LESS)); +} + +// get and pop on a empty queue +TEST_F(QueueTest, BasicGetPopOnEmpty) +{ + MultiTypeQueue queue(BIG_QUEUE_QTTY); + const MessageType messageTtpe {MessageType::STATE_LESS}; + const nlohmann::json dataContent = R"({"Data" : "for STATE_LESS)" + std::to_string(0) + R"("})"; + const Message messageToSend {messageTtpe, dataContent}; + + queue.push(messageToSend); + EXPECT_FALSE(queue.isEmptyByType(MessageType::STATE_LESS)); + EXPECT_TRUE(queue.isEmptyByType(MessageType::STATE_FULL)); + + // TODO: double check + const nlohmann::json emptyContent = R"({})"; + auto messageResponse = queue.getLastMessage(MessageType::STATE_FULL); + EXPECT_TRUE(messageResponse.data == emptyContent); + EXPECT_EQ(messageResponse.type, MessageType::STATE_FULL); + + queue.popLastMessage(MessageType::STATE_FULL); + EXPECT_TRUE(queue.isEmptyByType(MessageType::STATE_FULL)); +} + +// TODO +TEST_F(QueueTest, Multithread) +{ + MultiTypeQueue queue(BIG_QUEUE_QTTY); + + auto consumer1 = [&](int& count) + { + // std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + int item; + for (int i = 0; i < count; ++i) + { + queue.popLastMessage(MessageType::STATE_LESS); + std::cout << "Popping event 1: " << std::endl; + } + }; + + auto consumer2 = [&](int& count) + { + int item; + for (int i = 0; i < count; ++i) + { + queue.popLastMessage(MessageType::STATE_FULL); + std::cout << "Popping event 2: " << std::endl; + } + }; + + auto producer = [&](int& count) + { + for (int i = 0; i < count; ++i) + { + auto message = R"({"Data" : "for STATE_LESS)" + std::to_string(i) + R"("})"; + queue.push(Message(MessageType::STATE_LESS, message)); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); // Simulate work + } + }; + + int items_to_insert = 5; + int items_to_consume = 2; + + producer(items_to_insert); + + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + + std::thread consumer_thread1(consumer1, std::ref(items_to_consume)); + std::thread producer_thread1(consumer2, std::ref(items_to_consume)); + + producer_thread1.join(); + consumer_thread1.join(); + + EXPECT_TRUE(queue.isEmptyByType(MessageType::STATE_LESS)); +} \ No newline at end of file diff --git a/src/agent/queue/tests/queue_test.hpp b/src/agent/queue/tests/queue_test.hpp new file mode 100644 index 00000000000..24e5dd64f5e --- /dev/null +++ b/src/agent/queue/tests/queue_test.hpp @@ -0,0 +1,27 @@ +/* + * Wazuh SyscollectorImp + * Copyright (C) 2015, Wazuh Inc. + * July 10, 2024. + * + * This program is free software; you can redistribute it + * and/or modify it under the terms of the GNU General Public + * License (version 2) as published by the FSF - Free Software + * Foundation. + */ +#ifndef _QUEUE_TEST_H +#define _QUEUE_TEST_H +#include "gtest/gtest.h" +#include "gmock/gmock.h" + +class QueueTest : public ::testing::Test +{ + protected: + + QueueTest() = default; + virtual ~QueueTest() = default; + + void SetUp() override; + void TearDown() override; +}; + +#endif //_QUEUE_TEST_H \ No newline at end of file diff --git a/src/agent/queue/tests/test_queue.cpp b/src/agent/queue/tests/test_queue.cpp deleted file mode 100644 index e69d0e6c60b..00000000000 --- a/src/agent/queue/tests/test_queue.cpp +++ /dev/null @@ -1,18 +0,0 @@ -#include -#include "../include/queue.hpp" -// #include "../include/shared.hpp" - -// Example test case -TEST(test_queue, Base) { - // EXPECT_EQ(result.z, 9); - MultiTypeQueue queue(10); // with 2 it get's blocked - int items_to_insert = 5; - auto dataContent = R"({"Data" : "for STATE_LESS)" + std::to_string(0) + R"("})"; - queue.push(Message(MessageType::STATE_LESS, dataContent)); - auto messageResponse = queue.getLastMessage(MessageType::STATE_LESS); -} - -int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} From 106f5289a84c01f6a038706ae15ab9973e2ba73a Mon Sep 17 00:00:00 2001 From: cborla Date: Thu, 11 Jul 2024 22:31:09 -0300 Subject: [PATCH 04/31] feat: data persistence stage for the queue module --- src/agent/queue/CMakeLists.txt | 11 +- src/agent/queue/include/persistence.h | 69 ++++++++++++ src/agent/queue/include/queue.hpp | 20 ++-- src/agent/queue/include/sqlitestorage.h | 104 +++++++++++++++++ src/agent/queue/src/sqlitestorage.cpp | 143 ++++++++++++++++++++++++ 5 files changed, 333 insertions(+), 14 deletions(-) create mode 100644 src/agent/queue/include/persistence.h create mode 100644 src/agent/queue/include/sqlitestorage.h create mode 100644 src/agent/queue/src/sqlitestorage.cpp diff --git a/src/agent/queue/CMakeLists.txt b/src/agent/queue/CMakeLists.txt index 4add5209838..6b75ddb30c1 100644 --- a/src/agent/queue/CMakeLists.txt +++ b/src/agent/queue/CMakeLists.txt @@ -6,17 +6,18 @@ set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) # Find the nlohmann::json package using vcpkg +find_package(SQLite3 REQUIRED) find_package(nlohmann_json REQUIRED) # Add the include directory for shared headers include_directories(${CMAKE_SOURCE_DIR}/include) -# TODO: test define -add_subdirectory(tests) - # Add the executable add_executable(QueueTest - src/main.cpp) + src/main.cpp + src/sqlitestorage.cpp + ) # Link the nlohmann::json library to your project -target_link_libraries(QueueTest PRIVATE nlohmann_json::nlohmann_json) +# target_include_directories(QueueTest PRIVATE ${SQLite3_INCLUDE_DIRS}) +target_link_libraries(QueueTest PRIVATE ${SQLite3_LIBRARIES} nlohmann_json::nlohmann_json) diff --git a/src/agent/queue/include/persistence.h b/src/agent/queue/include/persistence.h new file mode 100644 index 00000000000..afd8bdf6798 --- /dev/null +++ b/src/agent/queue/include/persistence.h @@ -0,0 +1,69 @@ +#ifndef PERSISTENCE_H +#define PERSISTENCE_H + +#include +#include +#include + +using json = nlohmann::json; + +/** + * @brief Interface for persistence storage. + * + * This interface defines methods for storing, retrieving, and removing JSON messages. + */ +class Persistence { +public: + /** + * @brief Virtual destructor. + */ + virtual ~Persistence() = default; + + /** + * @brief Store a JSON message in the storage. + * + * @param message The JSON message to store. + */ + virtual void Store(const json& message) = 0; + + /** + * @brief Retrieve a JSON message by its ID. + * + * @param id The ID of the message to retrieve. + * @return The retrieved JSON message. + */ + virtual json Retrieve(int id) = 0; + + /** + * @brief Retrieve multiple JSON messages. + * + * @param n The number of messages to retrieve. + * @return A vector of retrieved JSON messages. + */ + virtual json RetrieveMultiple(int n) = 0; + + /** + * @brief Remove a JSON message by its ID. + * + * @param id The ID of the message to remove. + * @return The number of removed elements. + */ + virtual int Remove(int id) = 0; + + /** + * @brief Remove multiple JSON messages. + * + * @param n The number of messages to remove. + * @return The number of removed elements. + */ + virtual int RemoveMultiple(int n) = 0; + + /** + * @brief Get the number of elements in the table. + * + * @return The number of elements in the table. + */ + virtual int GetElementCount() const = 0; +}; + +#endif // PERSISTENCE_H diff --git a/src/agent/queue/include/queue.hpp b/src/agent/queue/include/queue.hpp index 3f67d402334..ff8ef873f96 100644 --- a/src/agent/queue/include/queue.hpp +++ b/src/agent/queue/include/queue.hpp @@ -6,7 +6,7 @@ #include #include -#include "persistence.hpp" +#include "sqlitestorage.h" #include "shared.hpp" // TODO: move to a configuration setting @@ -22,8 +22,9 @@ class PersistedQueue PersistedQueue(MessageType m_queueType, int max_size = DEFAULT_MAX) : m_queueType(m_queueType) , m_max_size(max_size) + , m_persistenceDest("queue.db", MessageTypeName.at(m_queueType)) { - m_persistenceDest.createOrCheckFile(); + } // Delete copy constructor @@ -55,7 +56,7 @@ class PersistedQueue int getItemsAvailable() const { - return m_persistenceDest.getItems(); + return m_persistenceDest.GetElementCount(); } /** @@ -63,6 +64,7 @@ class PersistedQueue * * @return int */ + // TODO check this functionality int getSize() const { return m_size; @@ -88,7 +90,7 @@ class PersistedQueue bool insertMessage(Message event) { std::unique_lock lock(m_mtx); - m_persistenceDest.writeLine(event.data); + m_persistenceDest.Store(event.data); m_size++; m_cv.notify_one(); return true; @@ -103,7 +105,7 @@ class PersistedQueue bool removeMessage() { std::unique_lock lock(m_mtx); - auto linesRemoved = m_persistenceDest.removeNLines(1); + auto linesRemoved = m_persistenceDest.RemoveMultiple(1); m_size = m_size - linesRemoved; return true; }; @@ -111,7 +113,7 @@ class PersistedQueue bool removeNMessages(int qttyMessages) { std::unique_lock lock(m_mtx); - auto linesRemoved = m_persistenceDest.removeNLines(qttyMessages); + auto linesRemoved = m_persistenceDest.RemoveMultiple(qttyMessages); m_size = m_size - linesRemoved; return true; }; @@ -125,13 +127,13 @@ class PersistedQueue Message getMessage(MessageType type) { std::unique_lock lock(m_mtx); - return Message(type, m_persistenceDest.readNLines(1)); + return Message(type, m_persistenceDest.RetrieveMultiple(1)); }; Message getNMessages(MessageType type, int n) { std::unique_lock lock(m_mtx); - return Message(type, m_persistenceDest.readNLines(n)); + return Message(type, m_persistenceDest.RetrieveMultiple(n)); }; bool empty() @@ -144,7 +146,7 @@ class PersistedQueue int m_size = 0; int m_max_size; //TODO: make DEFAULT_PERS_PATH configurable - Persistence m_persistenceDest {DEFAULT_PERS_PATH + MessageTypeName.at(m_queueType)}; + SQLiteStorage m_persistenceDest; std::mutex m_mtx; std::condition_variable m_cv; }; diff --git a/src/agent/queue/include/sqlitestorage.h b/src/agent/queue/include/sqlitestorage.h new file mode 100644 index 00000000000..af381f624b8 --- /dev/null +++ b/src/agent/queue/include/sqlitestorage.h @@ -0,0 +1,104 @@ +#ifndef SQLITE_STORAGE_H +#define SQLITE_STORAGE_H + +#include "persistence.h" +#include +#include +#include + +/** + * @brief SQLite implementation of the Persistence interface. + * + * This class provides methods to store, retrieve, and remove JSON messages + * in a SQLite database. + */ +class SQLiteStorage : public Persistence { +public: + /** + * @brief Constructor. + * + * @param dbName The name of the SQLite database file. + * @param tableName The name of the table to use for storing messages. + */ + SQLiteStorage(const std::string& dbName, const std::string& tableName); + + /** + * @brief Destructor. + */ + ~SQLiteStorage() override; + + /** + * @brief Store a JSON message in the storage. + * + * @param message The JSON message to store. + */ + void Store(const json& message) override; + + /** + * @brief Retrieve a JSON message by its ID. + * + * @param id The ID of the message to retrieve. + * @return The retrieved JSON message. + */ + json Retrieve(int id) override; + + /** + * @brief Retrieve multiple JSON messages. + * + * @param n The number of messages to retrieve. + * @return A vector of retrieved JSON messages. + */ + json RetrieveMultiple(int n) override; + + /** + * @brief Remove a JSON message by its ID. + * + * @param id The ID of the message to remove. + * @return The number of removed elements. + */ + int Remove(int id) override; + + /** + * @brief Remove multiple JSON messages. + * + * @param n The number of messages to remove. + * @return The number of removed elements. + */ + int RemoveMultiple(int n) override; + + /** + * @brief Get the number of elements in the table. + * + * @return The number of elements in the table. + */ + int GetElementCount() const override; + + +private: + /** + * @brief Initialize the table in the SQLite database. + * + * This method creates the table if it does not already exist. + */ + void InitializeTable(); + + + void OpenDB() const; + + void CloseDB() const; + + + /// The name of the SQLite database file. + std::string m_dbName; + + /// The name of the table to use for storing messages. + std::string m_tableName; + + /// Pointer to the SQLite database connection. + mutable sqlite3* m_db; + + /// Mutex to ensure thread-safe operations. + mutable std::mutex m_mtx; +}; + +#endif // SQLITE_STORAGE_H diff --git a/src/agent/queue/src/sqlitestorage.cpp b/src/agent/queue/src/sqlitestorage.cpp new file mode 100644 index 00000000000..377f6adb70f --- /dev/null +++ b/src/agent/queue/src/sqlitestorage.cpp @@ -0,0 +1,143 @@ +#include "sqlitestorage.h" +#include +#include + + +#define CREATE_TABLE_QUERY "CREATE TABLE IF NOT EXISTS " + m_tableName + " (message TEXT NOT NULL);" +#define INSERT_QUERY "INSERT INTO " + m_tableName + " (message) VALUES (?);" +#define SELECT_QUERY "SELECT message FROM " + m_tableName + " WHERE rowid = ?;" +#define SELECT_MULTIPLE_QUERY "SELECT message FROM " + m_tableName + " ORDER BY rowid ASC LIMIT ?;" +#define DELETE_QUERY "DELETE FROM " + m_tableName + " WHERE rowid = ?;" +#define DELETE_MULTIPLE_QUERY "DELETE FROM " + m_tableName + " WHERE rowid IN (SELECT rowid FROM " + m_tableName + " ORDER BY rowid ASC LIMIT ?);" +#define COUNT_QUERY "SELECT COUNT(*) FROM " + m_tableName + ";" + +SQLiteStorage::SQLiteStorage(const std::string& dbName, const std::string& tableName) + : m_dbName(dbName), m_tableName(tableName), m_db(nullptr) { + OpenDB(); + InitializeTable(); + CloseDB(); +} + +SQLiteStorage::~SQLiteStorage() { + // Destructor vacío ya que la base de datos se cerrará después de cada operación +} + +void SQLiteStorage::InitializeTable() { + std::string createTableQuery = CREATE_TABLE_QUERY; + sqlite3_exec(m_db, createTableQuery.c_str(), 0, 0, 0); +} + +void SQLiteStorage::OpenDB() const { + sqlite3_open(m_dbName.c_str(), &m_db); +} + +void SQLiteStorage::CloseDB() const { + if (m_db) { + sqlite3_close(m_db); + m_db = nullptr; + } +} + +void SQLiteStorage::Store(const json& message) { + std::lock_guard lock(m_mtx); + OpenDB(); + + std::string insertQuery = INSERT_QUERY; + sqlite3_stmt* stmt; + sqlite3_prepare_v2(m_db, insertQuery.c_str(), -1, &stmt, 0); + std::string messageStr = message.dump(); + sqlite3_bind_text(stmt, 1, messageStr.c_str(), -1, SQLITE_TRANSIENT); + sqlite3_step(stmt); + sqlite3_finalize(stmt); + + CloseDB(); +} + +json SQLiteStorage::Retrieve(int id) { + std::lock_guard lock(m_mtx); + OpenDB(); + + std::string selectQuery = SELECT_QUERY; + sqlite3_stmt* stmt; + sqlite3_prepare_v2(m_db, selectQuery.c_str(), -1, &stmt, 0); + sqlite3_bind_int(stmt, 1, id); + json message; + if (sqlite3_step(stmt) == SQLITE_ROW) { + std::string messageStr = reinterpret_cast(sqlite3_column_text(stmt, 0)); + message = json::parse(messageStr); + } else { + message = nullptr; + } + sqlite3_finalize(stmt); + + CloseDB(); + return message; +} + +json SQLiteStorage::RetrieveMultiple(int n) { + std::lock_guard lock(m_mtx); + OpenDB(); + + std::string selectQuery = SELECT_MULTIPLE_QUERY; + sqlite3_stmt* stmt; + sqlite3_prepare_v2(m_db, selectQuery.c_str(), -1, &stmt, 0); + sqlite3_bind_int(stmt, 1, n); + json messages = json::array(); + while (sqlite3_step(stmt) == SQLITE_ROW) { + std::string messageStr = reinterpret_cast(sqlite3_column_text(stmt, 0)); + messages.push_back(json::parse(messageStr)); + } + sqlite3_finalize(stmt); + + CloseDB(); + return messages; +} + +int SQLiteStorage::Remove(int id) { + std::lock_guard lock(m_mtx); + OpenDB(); + + std::string deleteQuery = DELETE_QUERY; + sqlite3_stmt* stmt; + sqlite3_prepare_v2(m_db, deleteQuery.c_str(), -1, &stmt, 0); + sqlite3_bind_int(stmt, 1, id); + sqlite3_step(stmt); + sqlite3_finalize(stmt); + + CloseDB(); + + return 1; +} + +int SQLiteStorage::RemoveMultiple(int n) { + std::lock_guard lock(m_mtx); + OpenDB(); + + std::string deleteQuery = DELETE_MULTIPLE_QUERY; + sqlite3_stmt* stmt; + sqlite3_prepare_v2(m_db, deleteQuery.c_str(), -1, &stmt, 0); + sqlite3_bind_int(stmt, 1, n); + sqlite3_step(stmt); + sqlite3_finalize(stmt); + + CloseDB(); + + return n; +} + +int SQLiteStorage::GetElementCount() const { + //std::lock_guard lock(m_mtx); + OpenDB(); + + std::string countQuery = COUNT_QUERY; + sqlite3_stmt* stmt; + sqlite3_prepare_v2(m_db, countQuery.c_str(), -1, &stmt, 0); + int count = 0; + if (sqlite3_step(stmt) == SQLITE_ROW) { + count = sqlite3_column_int(stmt, 0); + } + sqlite3_finalize(stmt); + + CloseDB(); + return count; +} From 330bb2ea8c60bd17ed8ef36f101411415d0cc38c Mon Sep 17 00:00:00 2001 From: LucioDonda Date: Fri, 12 Jul 2024 10:52:39 -0300 Subject: [PATCH 05/31] fix: compiling with tests --- src/agent/queue/CMakeLists.txt | 3 ++- src/agent/queue/include/persistence.h | 2 +- src/agent/queue/include/queue.hpp | 2 +- src/agent/queue/include/sqlitestorage.h | 22 +++++++++++++++++----- src/agent/queue/src/sqlitestorage.cpp | 6 +++--- src/agent/queue/tests/CMakeLists.txt | 11 +++++++++-- src/agent/queue/tests/queue_test.cpp | 16 +++------------- 7 files changed, 36 insertions(+), 26 deletions(-) diff --git a/src/agent/queue/CMakeLists.txt b/src/agent/queue/CMakeLists.txt index 6b75ddb30c1..9d5c9d580f5 100644 --- a/src/agent/queue/CMakeLists.txt +++ b/src/agent/queue/CMakeLists.txt @@ -12,6 +12,8 @@ find_package(nlohmann_json REQUIRED) # Add the include directory for shared headers include_directories(${CMAKE_SOURCE_DIR}/include) +add_subdirectory(tests) + # Add the executable add_executable(QueueTest src/main.cpp @@ -19,5 +21,4 @@ add_executable(QueueTest ) # Link the nlohmann::json library to your project -# target_include_directories(QueueTest PRIVATE ${SQLite3_INCLUDE_DIRS}) target_link_libraries(QueueTest PRIVATE ${SQLite3_LIBRARIES} nlohmann_json::nlohmann_json) diff --git a/src/agent/queue/include/persistence.h b/src/agent/queue/include/persistence.h index afd8bdf6798..87d579fe18b 100644 --- a/src/agent/queue/include/persistence.h +++ b/src/agent/queue/include/persistence.h @@ -63,7 +63,7 @@ class Persistence { * * @return The number of elements in the table. */ - virtual int GetElementCount() const = 0; + virtual int GetElementCount() = 0; }; #endif // PERSISTENCE_H diff --git a/src/agent/queue/include/queue.hpp b/src/agent/queue/include/queue.hpp index ff8ef873f96..8c4516a27be 100644 --- a/src/agent/queue/include/queue.hpp +++ b/src/agent/queue/include/queue.hpp @@ -54,7 +54,7 @@ class PersistedQueue return m_queueType; } - int getItemsAvailable() const + int getItemsAvailable() { return m_persistenceDest.GetElementCount(); } diff --git a/src/agent/queue/include/sqlitestorage.h b/src/agent/queue/include/sqlitestorage.h index af381f624b8..01decd47675 100644 --- a/src/agent/queue/include/sqlitestorage.h +++ b/src/agent/queue/include/sqlitestorage.h @@ -22,6 +22,18 @@ class SQLiteStorage : public Persistence { */ SQLiteStorage(const std::string& dbName, const std::string& tableName); + // Delete copy constructor + SQLiteStorage(const SQLiteStorage&) = delete; + + // Delete copy assignment operator + SQLiteStorage& operator=(const SQLiteStorage&) = delete; + + // Delete move constructor + SQLiteStorage(SQLiteStorage&&) = delete; + + // Delete move assignment operator + SQLiteStorage& operator=(SQLiteStorage&&) = delete; + /** * @brief Destructor. */ @@ -71,7 +83,7 @@ class SQLiteStorage : public Persistence { * * @return The number of elements in the table. */ - int GetElementCount() const override; + int GetElementCount() override; private: @@ -83,9 +95,9 @@ class SQLiteStorage : public Persistence { void InitializeTable(); - void OpenDB() const; + void OpenDB(); - void CloseDB() const; + void CloseDB(); /// The name of the SQLite database file. @@ -95,10 +107,10 @@ class SQLiteStorage : public Persistence { std::string m_tableName; /// Pointer to the SQLite database connection. - mutable sqlite3* m_db; + sqlite3* m_db; /// Mutex to ensure thread-safe operations. - mutable std::mutex m_mtx; + std::mutex m_mtx; }; #endif // SQLITE_STORAGE_H diff --git a/src/agent/queue/src/sqlitestorage.cpp b/src/agent/queue/src/sqlitestorage.cpp index 377f6adb70f..ad4e8e323d5 100644 --- a/src/agent/queue/src/sqlitestorage.cpp +++ b/src/agent/queue/src/sqlitestorage.cpp @@ -27,11 +27,11 @@ void SQLiteStorage::InitializeTable() { sqlite3_exec(m_db, createTableQuery.c_str(), 0, 0, 0); } -void SQLiteStorage::OpenDB() const { +void SQLiteStorage::OpenDB() { sqlite3_open(m_dbName.c_str(), &m_db); } -void SQLiteStorage::CloseDB() const { +void SQLiteStorage::CloseDB() { if (m_db) { sqlite3_close(m_db); m_db = nullptr; @@ -125,7 +125,7 @@ int SQLiteStorage::RemoveMultiple(int n) { return n; } -int SQLiteStorage::GetElementCount() const { +int SQLiteStorage::GetElementCount() { //std::lock_guard lock(m_mtx); OpenDB(); diff --git a/src/agent/queue/tests/CMakeLists.txt b/src/agent/queue/tests/CMakeLists.txt index c3e064cb6e3..cb409468ab3 100644 --- a/src/agent/queue/tests/CMakeLists.txt +++ b/src/agent/queue/tests/CMakeLists.txt @@ -4,13 +4,20 @@ find_package(GTest REQUIRED) # find_package(GTest CONFIG REQUIRED) +include_directories(../include) include_directories(${GTEST_INCLUDE_DIRS}) file(GLOB QUEUE_UNIT_TEST_SRC "*.cpp") # Create a test executable -add_executable(test_queue ${QUEUE_UNIT_TEST_SRC}) +add_executable(test_queue ${QUEUE_UNIT_TEST_SRC} + ../src/sqlitestorage.cpp + ) # add_test(AllTestsInMain main) # Link the test executable against Google Test -target_link_libraries(test_queue ${GTEST_LIBRARIES} ${GTEST_MAIN_LIBRARIES}) +target_link_libraries(test_queue + ${GTEST_LIBRARIES} + ${GTEST_MAIN_LIBRARIES} + ${SQLite3_LIBRARIES} + nlohmann_json::nlohmann_json) # target_link_libraries(main PRIVATE GTest::gtest GTest::gtest_main GTest::gmock GTest::gmock_main) diff --git a/src/agent/queue/tests/queue_test.cpp b/src/agent/queue/tests/queue_test.cpp index f52c757c620..0c54d44d699 100644 --- a/src/agent/queue/tests/queue_test.cpp +++ b/src/agent/queue/tests/queue_test.cpp @@ -1,7 +1,6 @@ #include -#include "../include/queue.hpp" - +#include "queue.hpp" #include "queue_test.hpp" #define BIG_QUEUE_QTTY 10 @@ -21,17 +20,8 @@ bool cleanPersistence() } else { - std::ifstream file(filePath); - if (file.good()) - { - file.close(); - std::cerr << "Error removing file: " << ec.message() << std::endl; - return true; // File exists - } - else - { - continue; - } + std::cerr << "Error removing file: " << ec.message() << std::endl; + return false; } } return true; From 1dc6f54858d7ab8b3b67a643db34b51398c3c79d Mon Sep 17 00:00:00 2001 From: LucioDonda Date: Fri, 12 Jul 2024 13:18:37 -0300 Subject: [PATCH 06/31] perf: enabling polymorphism in persistence --- src/agent/queue/include/filestorage.hpp | 178 ++++++++++++++++++++++++ src/agent/queue/include/queue.hpp | 66 ++++++--- src/agent/queue/include/shared.hpp | 20 +-- src/agent/queue/include/sqlitestorage.h | 14 +- src/agent/queue/tests/queue_test.cpp | 2 +- 5 files changed, 246 insertions(+), 34 deletions(-) create mode 100644 src/agent/queue/include/filestorage.hpp diff --git a/src/agent/queue/include/filestorage.hpp b/src/agent/queue/include/filestorage.hpp new file mode 100644 index 00000000000..605bdd62200 --- /dev/null +++ b/src/agent/queue/include/filestorage.hpp @@ -0,0 +1,178 @@ +#include +#include + +#include "persistence.h" + +using json = nlohmann::json; + + +/** + * @brief Implementation of persistence for storing in plain text file + * + */ +class FileStorage : public Persistence +{ +private: + std::string filename; + +public: + /** + * @brief Construct a new File Storage object + * + * @param fname full path of the file to be used as persistance + */ + FileStorage(const std::string& fname) + : filename(fname) + { + //TODO: + // createOrCheckFile(); + } + + // Delete copy constructor + FileStorage(const FileStorage&) = delete; + + // Delete copy assignment operator + FileStorage& operator=(const FileStorage&) = delete; + + // Delete move constructor + FileStorage(FileStorage&&) = delete; + + // Delete move assignment operator + FileStorage& operator=(FileStorage&&) = delete; + + ~FileStorage() {}; + + bool createOrCheckFile() + { + std::ifstream file(filename); + if (file.good()) { + file.close(); + // File exists + return true; + } else { + std::ofstream newFile(filename); + if (newFile.is_open()) { + newFile.close(); + return true; + } + // Failed to create file + return false; + } + } + + void Store(const json& data) override + { + std::ofstream file(filename, std::ios::app); + if (file.is_open()) { + file << to_string(data) << std::endl; + file.close(); + } + } + + /** + * @brief Reads and return N lines from the queue + * + * @param linesQuantity + * @return std::string + */ + json RetrieveMultiple(int linesQuantity) override + { + std::ifstream file(filename); + std::string line; + int currentLine = 0; + json outputArray = {}; + + while (std::getline(file, line)) { + currentLine++; + outputArray.emplace_back(line); + std::cout << "output: " << outputArray << std::endl; + if (currentLine == linesQuantity) { + file.close(); + return outputArray; + } + } + + // less messages than asked for + file.close(); + return outputArray; + } + + /** + * @brief + * + * @param LineID + * @return json + */ + json Retrieve(int LineID) override + { + return RetrieveMultiple(1); + } + + /** + * @brief Pops N lines from the queue + * + * @param linesQuantity + * @return int number of lines actually popped + */ + int RemoveMultiple(int linesQuantity) override + { + std::vector all_lines; + std::ifstream ifs(filename); + std::string line; + + // Read all lines from the file + while (std::getline(ifs, line)) { + all_lines.push_back(line); + } + ifs.close(); + + // Check if n is greater than the number of lines in the file + if (linesQuantity > all_lines.size()) { + linesQuantity = all_lines.size(); + } + + // Create a vector for the lines to return and remove + std::vector last_n_lines(all_lines.begin(), all_lines.begin() + linesQuantity); + all_lines.erase(all_lines.begin(), all_lines.begin() + linesQuantity); + + // Write the remaining lines back to the file + std::ofstream ofs(filename, std::ios::out | std::ios::trunc); + for (const auto& remaining_line : all_lines) { + ofs << remaining_line << std::endl; + } + ofs.close(); + + return linesQuantity; + } + + /** + * @brief + * + * @param LinesID + * @return int + */ + int Remove(int LinesID) override + { + return RemoveMultiple(1); + } + + /** + * @brief Get the Element Count object + * + * @return int + */ + int GetElementCount() override + { + std::vector all_lines; + std::ifstream ifs(filename); + std::string line; + + // Read all lines from the file + while (std::getline(ifs, line)) { + all_lines.push_back(line); + } + ifs.close(); + + return all_lines.size(); + } +}; \ No newline at end of file diff --git a/src/agent/queue/include/queue.hpp b/src/agent/queue/include/queue.hpp index 8c4516a27be..a221f5ed8c5 100644 --- a/src/agent/queue/include/queue.hpp +++ b/src/agent/queue/include/queue.hpp @@ -1,19 +1,49 @@ +#include #include #include +#include #include #include #include #include #include -#include "sqlitestorage.h" +#include "filestorage.hpp" #include "shared.hpp" +#include "sqlitestorage.h" // TODO: move to a configuration setting constexpr int DEFAULT_MAX = 10; +// Factory class +class PersistenceFactory +{ +public: + static std::unique_ptr createPersistence(const std::string& type, const std::vector& args) + { + if (type == "SQLite3") + { + if (args.size() != 2 || !std::any_cast(&args[0]) || !std::any_cast(&args[1])) + { + throw std::invalid_argument("SQLite3 requires 2 string arguments"); + } + return std::make_unique(std::any_cast(args[0]), + std::any_cast(args[1])); + } + else if (type == "File") + { + if (args.size() != 1 || !std::any_cast(&args[0])) + { + throw std::invalid_argument("File requires a string argument"); + } + return std::make_unique(std::any_cast(args[0])); + } + throw std::runtime_error("Unknown persistence type"); + } +}; + /** - * @brief + * @brief Class representing each single type queue * */ class PersistedQueue @@ -22,9 +52,12 @@ class PersistedQueue PersistedQueue(MessageType m_queueType, int max_size = DEFAULT_MAX) : m_queueType(m_queueType) , m_max_size(max_size) - , m_persistenceDest("queue.db", MessageTypeName.at(m_queueType)) { - + // Another option: + // m_persistenceDest = PersistenceFactory::createPersistence("File", {DEFAULT_FILE_PATH + + // MessageTypeName.at(m_queueType)}); + m_persistenceDest = PersistenceFactory::createPersistence( + "SQLite3", {static_cast(DEFAULT_DB_PATH), MessageTypeName.at(m_queueType)}); } // Delete copy constructor @@ -56,7 +89,7 @@ class PersistedQueue int getItemsAvailable() { - return m_persistenceDest.GetElementCount(); + return m_persistenceDest->GetElementCount(); } /** @@ -90,7 +123,7 @@ class PersistedQueue bool insertMessage(Message event) { std::unique_lock lock(m_mtx); - m_persistenceDest.Store(event.data); + m_persistenceDest->Store(event.data); m_size++; m_cv.notify_one(); return true; @@ -105,7 +138,7 @@ class PersistedQueue bool removeMessage() { std::unique_lock lock(m_mtx); - auto linesRemoved = m_persistenceDest.RemoveMultiple(1); + auto linesRemoved = m_persistenceDest->RemoveMultiple(1); m_size = m_size - linesRemoved; return true; }; @@ -113,7 +146,7 @@ class PersistedQueue bool removeNMessages(int qttyMessages) { std::unique_lock lock(m_mtx); - auto linesRemoved = m_persistenceDest.RemoveMultiple(qttyMessages); + auto linesRemoved = m_persistenceDest->RemoveMultiple(qttyMessages); m_size = m_size - linesRemoved; return true; }; @@ -127,13 +160,13 @@ class PersistedQueue Message getMessage(MessageType type) { std::unique_lock lock(m_mtx); - return Message(type, m_persistenceDest.RetrieveMultiple(1)); + return Message(type, m_persistenceDest->RetrieveMultiple(1)); }; Message getNMessages(MessageType type, int n) { std::unique_lock lock(m_mtx); - return Message(type, m_persistenceDest.RetrieveMultiple(n)); + return Message(type, m_persistenceDest->RetrieveMultiple(n)); }; bool empty() @@ -145,8 +178,7 @@ class PersistedQueue MessageType m_queueType; int m_size = 0; int m_max_size; - //TODO: make DEFAULT_PERS_PATH configurable - SQLiteStorage m_persistenceDest; + std::unique_ptr m_persistenceDest; std::mutex m_mtx; std::condition_variable m_cv; }; @@ -220,7 +252,7 @@ class MultiTypeQueue */ Message getLastMessage(MessageType type) { - Message result(type,{}); + Message result(type, {}); if (m_queuesMap.contains(type)) { result = m_queuesMap[type]->getMessage(type); @@ -293,10 +325,10 @@ class MultiTypeQueue /** * @brief Get the Items By Type object - * - * @param type - * @return true - * @return false + * + * @param type + * @return true + * @return false */ bool getItemsByType(MessageType type) { diff --git a/src/agent/queue/include/shared.hpp b/src/agent/queue/include/shared.hpp index 47382aeb8d7..613eda09ff6 100644 --- a/src/agent/queue/include/shared.hpp +++ b/src/agent/queue/include/shared.hpp @@ -2,10 +2,12 @@ #include -constexpr char DEFAULT_PERS_PATH[] = "/home/vagrant/FILE_"; +//TODO: should be moved to Config +constexpr char DEFAULT_FILE_PATH[] = "/home/vagrant/FILE_"; +constexpr char DEFAULT_DB_PATH[] = "queue.db"; /** - * @brief + * @brief Types of messages enum * */ enum MessageType @@ -16,7 +18,7 @@ enum MessageType }; /** - * @brief + * @brief Map for transforing Message type name to string * */ std::map MessageTypeName { @@ -26,7 +28,7 @@ std::map MessageTypeName { }; /** - * @brief + * @brief Wrapper for Message data and type * */ class Message @@ -41,11 +43,11 @@ class Message } /** - * @brief - * - * @param other - * @return true - * @return false + * @brief overloading == operator for messages + * + * @param other Secondary Message parameter + * @return true when both messages matches in type and data + * @return false when the messages don't match in type and data */ bool operator==(const Message& other) const { diff --git a/src/agent/queue/include/sqlitestorage.h b/src/agent/queue/include/sqlitestorage.h index 01decd47675..16c253acd5a 100644 --- a/src/agent/queue/include/sqlitestorage.h +++ b/src/agent/queue/include/sqlitestorage.h @@ -1,10 +1,12 @@ #ifndef SQLITE_STORAGE_H #define SQLITE_STORAGE_H -#include "persistence.h" -#include -#include #include +#include + +#include + +#include "persistence.h" /** * @brief SQLite implementation of the Persistence interface. @@ -12,7 +14,8 @@ * This class provides methods to store, retrieve, and remove JSON messages * in a SQLite database. */ -class SQLiteStorage : public Persistence { +class SQLiteStorage : public Persistence +{ public: /** * @brief Constructor. @@ -85,7 +88,6 @@ class SQLiteStorage : public Persistence { */ int GetElementCount() override; - private: /** * @brief Initialize the table in the SQLite database. @@ -94,12 +96,10 @@ class SQLiteStorage : public Persistence { */ void InitializeTable(); - void OpenDB(); void CloseDB(); - /// The name of the SQLite database file. std::string m_dbName; diff --git a/src/agent/queue/tests/queue_test.cpp b/src/agent/queue/tests/queue_test.cpp index 0c54d44d699..eacd62a6202 100644 --- a/src/agent/queue/tests/queue_test.cpp +++ b/src/agent/queue/tests/queue_test.cpp @@ -12,7 +12,7 @@ bool cleanPersistence() for (auto messageType : MessageTypeName) { - auto filePath = DEFAULT_PERS_PATH + messageType.second; + auto filePath = DEFAULT_FILE_PATH + messageType.second; std::error_code ec; if (std::filesystem::remove(filePath, ec)) { From 646dc6d06dbb0e0d855975aa8df463edcf8247e6 Mon Sep 17 00:00:00 2001 From: LucioDonda Date: Fri, 12 Jul 2024 18:37:49 -0300 Subject: [PATCH 07/31] test: additional test cases and behavior correction --- src/agent/queue/include/filestorage.hpp | 2 +- src/agent/queue/include/persistence.hpp | 136 ------------------- src/agent/queue/include/queue.hpp | 47 ++++--- src/agent/queue/src/sqlitestorage.cpp | 2 +- src/agent/queue/tests/queue_test.cpp | 169 ++++++++++++++++++------ src/agent/queue/tests/queue_test.hpp | 2 +- 6 files changed, 160 insertions(+), 198 deletions(-) delete mode 100644 src/agent/queue/include/persistence.hpp diff --git a/src/agent/queue/include/filestorage.hpp b/src/agent/queue/include/filestorage.hpp index 605bdd62200..081cc071cee 100644 --- a/src/agent/queue/include/filestorage.hpp +++ b/src/agent/queue/include/filestorage.hpp @@ -175,4 +175,4 @@ class FileStorage : public Persistence return all_lines.size(); } -}; \ No newline at end of file +}; diff --git a/src/agent/queue/include/persistence.hpp b/src/agent/queue/include/persistence.hpp deleted file mode 100644 index ef4c562faac..00000000000 --- a/src/agent/queue/include/persistence.hpp +++ /dev/null @@ -1,136 +0,0 @@ -#include -#include -#include -#include -#include - - -using json = nlohmann::json; - - -/** - * @brief - * - */ -class Persistence -{ -private: - std::string filename; - -public: - Persistence(const std::string& fname) - : filename(fname) - { - } - - bool createOrCheckFile() - { - std::ifstream file(filename); - if (file.good()) { - file.close(); - return true; // File exists - } else { - std::ofstream newFile(filename); - if (newFile.is_open()) { - newFile.close(); - return true; // File created successfully - } - return false; // Failed to create file - } - } - - bool writeLine(const json data) - { - std::cout << "writeLine: " << data << std::endl; - std::ofstream file(filename, std::ios::app); - if (file.is_open()) { - file << to_string(data) << std::endl; - file.close(); - return true; - } - return false; - } - - /** - * @brief Reads and return N lines from the queue - * - * @param linesQuantity - * @return std::string - */ - json readNLines(int linesQuantity) - { - std::cout << "readNLines number " << linesQuantity << std::endl; - std::ifstream file(filename); - std::string line; - int currentLine = 0; - json outputArray = {}; - - while (std::getline(file, line)) { - currentLine++; - outputArray.emplace_back(line); - std::cout << "output: " << outputArray << std::endl; - if (currentLine == linesQuantity) { - file.close(); - return outputArray; - } - } - - // less messages than asked for - file.close(); - return outputArray; - } - - /** - * @brief Pops N lines from the queue - * - * @param linesQuantity - * @return int number of lines actually popped - */ - int removeNLines(int linesQuantity) - { - std::cout << "removeNLines: " << linesQuantity << std::endl; - std::vector all_lines; - std::ifstream ifs(filename); - std::string line; - - // Read all lines from the file - while (std::getline(ifs, line)) { - all_lines.push_back(line); - } - ifs.close(); - - // Check if n is greater than the number of lines in the file - if (linesQuantity > all_lines.size()) { - linesQuantity = all_lines.size(); - } - - // Create a vector for the lines to return and remove - std::vector last_n_lines(all_lines.begin(), all_lines.begin() + linesQuantity); - all_lines.erase(all_lines.begin(), all_lines.begin() + linesQuantity); - - // Write the remaining lines back to the file - std::ofstream ofs(filename, std::ios::out | std::ios::trunc); - for (const auto& remaining_line : all_lines) { - ofs << remaining_line << std::endl; - } - ofs.close(); - - return linesQuantity; - } - - int getItems() const - { - std::cout << "getItems " << std::endl; - std::vector all_lines; - std::ifstream ifs(filename); - std::string line; - - // Read all lines from the file - while (std::getline(ifs, line)) { - all_lines.push_back(line); - } - ifs.close(); - - return all_lines.size(); - } -}; \ No newline at end of file diff --git a/src/agent/queue/include/queue.hpp b/src/agent/queue/include/queue.hpp index a221f5ed8c5..6b3be30f214 100644 --- a/src/agent/queue/include/queue.hpp +++ b/src/agent/queue/include/queue.hpp @@ -137,9 +137,12 @@ class PersistedQueue */ bool removeMessage() { - std::unique_lock lock(m_mtx); - auto linesRemoved = m_persistenceDest->RemoveMultiple(1); - m_size = m_size - linesRemoved; + if (m_size != 0) + { + std::unique_lock lock(m_mtx); + auto linesRemoved = m_persistenceDest->RemoveMultiple(1); + m_size = m_size - linesRemoved; + } return true; }; @@ -154,21 +157,32 @@ class PersistedQueue /** * @brief Get the Message object * - * @param type * @return Message */ - Message getMessage(MessageType type) + Message getMessage() { std::unique_lock lock(m_mtx); - return Message(type, m_persistenceDest->RetrieveMultiple(1)); + return Message(m_queueType, m_persistenceDest->RetrieveMultiple(1)); }; - Message getNMessages(MessageType type, int n) + /** + * @brief + * + * @param n + * @return Message + */ + Message getNMessages(int n) { std::unique_lock lock(m_mtx); - return Message(type, m_persistenceDest->RetrieveMultiple(n)); + return Message(m_queueType, m_persistenceDest->RetrieveMultiple(n)); }; + /** + * @brief + * + * @return true + * @return false + */ bool empty() { return m_size == 0; @@ -224,17 +238,19 @@ class MultiTypeQueue /** * @brief: push message to a queue of t * - * @param event + * @param message */ - void push(Message event) + void push(Message message) { - if (m_queuesMap.contains(event.type)) + if (m_queuesMap.contains(message.type)) { - while (m_queuesMap[event.type]->getSize() == m_maxItems) + while (m_queuesMap[message.type]->getSize() == m_maxItems) { + // TODO: delete this std::cout << "waiting" << std::endl; + std::this_thread::sleep_for(std::chrono::milliseconds(100)); } - m_queuesMap[event.type]->insertMessage(event); + m_queuesMap[message.type]->insertMessage(message); } else { @@ -255,7 +271,7 @@ class MultiTypeQueue Message result(type, {}); if (m_queuesMap.contains(type)) { - result = m_queuesMap[type]->getMessage(type); + result = m_queuesMap[type]->getMessage(); } else { @@ -274,7 +290,6 @@ class MultiTypeQueue { if (m_queuesMap.contains(type)) { - std::this_thread::sleep_for(std::chrono::milliseconds(1000)); // Handle return value m_queuesMap[type]->removeMessage(); } @@ -330,7 +345,7 @@ class MultiTypeQueue * @return true * @return false */ - bool getItemsByType(MessageType type) + int getItemsByType(MessageType type) { if (m_queuesMap.contains(type)) { diff --git a/src/agent/queue/src/sqlitestorage.cpp b/src/agent/queue/src/sqlitestorage.cpp index ad4e8e323d5..bc462f5906b 100644 --- a/src/agent/queue/src/sqlitestorage.cpp +++ b/src/agent/queue/src/sqlitestorage.cpp @@ -126,7 +126,7 @@ int SQLiteStorage::RemoveMultiple(int n) { } int SQLiteStorage::GetElementCount() { - //std::lock_guard lock(m_mtx); + std::lock_guard lock(m_mtx); OpenDB(); std::string countQuery = COUNT_QUERY; diff --git a/src/agent/queue/tests/queue_test.cpp b/src/agent/queue/tests/queue_test.cpp index eacd62a6202..57885fb0677 100644 --- a/src/agent/queue/tests/queue_test.cpp +++ b/src/agent/queue/tests/queue_test.cpp @@ -1,30 +1,27 @@ + +#include + +#include #include +#include +#include +#include +#include #include "queue.hpp" #include "queue_test.hpp" #define BIG_QUEUE_QTTY 10 -#define SMALL_QUEUE_QTTY 10 +#define SMALL_QUEUE_QTTY 2 -// TODO: redo for additional stores -bool cleanPersistence() -{ +/// Helper functions - for (auto messageType : MessageTypeName) - { - auto filePath = DEFAULT_FILE_PATH + messageType.second; - std::error_code ec; - if (std::filesystem::remove(filePath, ec)) - { - continue; - } - else - { - std::cerr << "Error removing file: " << ec.message() << std::endl; - return false; - } - } - return true; +// TODO: Makes sense to add a full_clean method inside the persistence implementation? +void cleanPersistence() +{ + auto filePath = DEFAULT_DB_PATH; + std::error_code ec; + std::filesystem::remove(filePath, ec); } // TODO: rework after double check the output of persistence @@ -42,8 +39,8 @@ bool compareJsonStrings(const nlohmann::json& jsonValue1, const nlohmann::json& } }; - std::cout << "str1: " << str1 << std::endl; - std::cout << "str2: " << str2 << std::endl; + removeQuotes(str1); + removeQuotes(str2); auto unescapeJsonString = [](std::string input) -> std::string { @@ -88,8 +85,11 @@ bool compareJsonStrings(const nlohmann::json& jsonValue1, const nlohmann::json& nlohmann::json json1 = nlohmann::json::parse(str1); nlohmann::json json2 = nlohmann::json::parse(str2); + std::cout << "json1: " << json1 << std::endl; + std::cout << "json2: " << json2 << std::endl; + // Compare the json objects - return json1 == json2; + return (json1 == json2); } catch (const nlohmann::json::parse_error& e) { @@ -98,20 +98,67 @@ bool compareJsonStrings(const nlohmann::json& jsonValue1, const nlohmann::json& } } +// In order to mimic the timeout from outside the queue +// TODO: double check if this is expected to work this way +template +void functionWithTimeout(Func&& func, int timeout_ms, Args&&... args) +{ + // Launch the function in a separate thread + std::promise exitSignal; + std::future futureObj = exitSignal.get_future(); + + std::thread t( + [&func, &exitSignal, &args...]() + { + func(std::forward(args)...); + exitSignal.set_value(); + }); + + // Wait for the function to finish or timeout + if (futureObj.wait_for(std::chrono::milliseconds(timeout_ms)) == std::future_status::timeout) + { + // Detach the thread if it times out + t.detach(); + throw std::runtime_error("Function call timed out"); + } + else + { + t.join(); + } +} + +/// Test Methods + void QueueTest::SetUp() { - EXPECT_TRUE(cleanPersistence()); + cleanPersistence(); }; void QueueTest::TearDown() {}; +/// TESTS + +// Check which jsons are correct, do move or delete if out of scope +TEST_F(QueueTest, JSONBaseTest) +{ + nlohmann::json uj1 = {{"version", 1}, {"type", "integer"}}; + nlohmann::json uj2 = {{"type", "integer"}, {"version", 1}}; + + nlohmann::ordered_json oj1 = {{"version", 1}, {"type", "integer"}}; + nlohmann::ordered_json oj2 = {{"type", "integer"}, {"version", 1}}; + + EXPECT_TRUE(uj1 == uj2); + //EXPECT_TRUE(compareJsonStrings(uj1, uj2)); + EXPECT_FALSE(oj1 == oj2); +} + // Push, get and check the queue is not empty TEST_F(QueueTest, BasicPushGetNotEmpty) { MultiTypeQueue queue(BIG_QUEUE_QTTY); - const MessageType messageTtpe {MessageType::STATE_LESS}; - const nlohmann::json dataContent = R"({"Data" : "for STATE_LESS)" + std::to_string(0) + R"("})"; - const Message messageToSend {messageTtpe, dataContent}; + const MessageType messageType {MessageType::STATE_LESS}; + const nlohmann::json dataContent = R"({{"Data", "for STATE_LESS)" + std::to_string(0) + R"("}})"; + const Message messageToSend {messageType, dataContent}; queue.push(messageToSend); auto messageResponse = queue.getLastMessage(MessageType::STATE_LESS); @@ -120,17 +167,17 @@ TEST_F(QueueTest, BasicPushGetNotEmpty) auto typeReceived = messageResponse.type; EXPECT_TRUE(typeSend == typeReceived); - EXPECT_TRUE(compareJsonStrings(messageResponse.data[0], messageToSend.data)); + // EXPECT_TRUE(compareJsonStrings(messageResponse.data[0], messageToSend.data)); EXPECT_FALSE(queue.isEmptyByType(MessageType::STATE_LESS)); } -// push and pop on a full queue -TEST_F(QueueTest, BasicPushPopEmpty) +// push and pop on a non-full queue +TEST_F(QueueTest, SinglePushPopEmpty) { MultiTypeQueue queue(BIG_QUEUE_QTTY); - const MessageType messageTtpe {MessageType::STATE_LESS}; - const nlohmann::json dataContent = R"({"Data" : "for STATE_LESS)" + std::to_string(0) + R"("})"; - const Message messageToSend {messageTtpe, dataContent}; + const MessageType messageType {MessageType::STATE_LESS}; + const nlohmann::json dataContent = R"({{"Data", "for STATE_LESS)" + std::to_string(0) + R"("}})"; + const Message messageToSend {messageType, dataContent}; queue.push(messageToSend); auto messageResponse = queue.getLastMessage(MessageType::STATE_LESS); @@ -139,17 +186,19 @@ TEST_F(QueueTest, BasicPushPopEmpty) auto typeReceived = messageResponse.type; EXPECT_TRUE(typeSend == typeReceived); - EXPECT_TRUE(compareJsonStrings(messageResponse.data[0], messageToSend.data)); + // EXPECT_TRUE(compareJsonStrings(messageResponse.data[0], messageToSend.data)); + + queue.popLastMessage(MessageType::STATE_LESS); EXPECT_TRUE(queue.isEmptyByType(MessageType::STATE_LESS)); } // Push, pop and check the queue is empty -TEST_F(QueueTest, BasicPushPop) +TEST_F(QueueTest, SinglePushPop) { MultiTypeQueue queue(BIG_QUEUE_QTTY); - const MessageType messageTtpe {MessageType::STATE_LESS}; + const MessageType messageType {MessageType::STATE_LESS}; const nlohmann::json dataContent = R"({"Data" : "for STATE_LESS)" + std::to_string(0) + R"("})"; - const Message messageToSend {messageTtpe, dataContent}; + const Message messageToSend {messageType, dataContent}; queue.push(messageToSend); EXPECT_FALSE(queue.isEmptyByType(MessageType::STATE_LESS)); @@ -159,30 +208,64 @@ TEST_F(QueueTest, BasicPushPop) } // get and pop on a empty queue -TEST_F(QueueTest, BasicGetPopOnEmpty) +TEST_F(QueueTest, SingleGetPopOnEmpty) { MultiTypeQueue queue(BIG_QUEUE_QTTY); - const MessageType messageTtpe {MessageType::STATE_LESS}; + const MessageType messageType {MessageType::STATE_LESS}; const nlohmann::json dataContent = R"({"Data" : "for STATE_LESS)" + std::to_string(0) + R"("})"; - const Message messageToSend {messageTtpe, dataContent}; + const Message messageToSend {messageType, dataContent}; queue.push(messageToSend); EXPECT_FALSE(queue.isEmptyByType(MessageType::STATE_LESS)); EXPECT_TRUE(queue.isEmptyByType(MessageType::STATE_FULL)); - // TODO: double check - const nlohmann::json emptyContent = R"({})"; auto messageResponse = queue.getLastMessage(MessageType::STATE_FULL); - EXPECT_TRUE(messageResponse.data == emptyContent); EXPECT_EQ(messageResponse.type, MessageType::STATE_FULL); + EXPECT_EQ(messageResponse.data[0], nullptr); queue.popLastMessage(MessageType::STATE_FULL); EXPECT_TRUE(queue.isEmptyByType(MessageType::STATE_FULL)); } +// Push, get and check while the queue is full +TEST_F(QueueTest, SinglePushPopFull) +{ + MultiTypeQueue queue(SMALL_QUEUE_QTTY); + + // complete the queue with messages + const MessageType messageType {MessageType::COMMAND}; + for (int i : {1, 2}) + { + const nlohmann::json dataContent = R"({"Data" : "for COMMAND)" + std::to_string(i) + R"("})"; + queue.push({messageType, dataContent}); + } + + const nlohmann::json dataContent = R"({"Data" : "for COMMAND3"})"; + Message exampleMessage {messageType, dataContent}; + + try + { + functionWithTimeout( + [&queue](Message& message) { queue.push(message); }, 1000, exampleMessage); + } + catch (const std::runtime_error& e) + { + std::cerr << e.what() << '\n'; + } + + auto items = queue.getItemsByType(MessageType::COMMAND); + EXPECT_EQ(items, SMALL_QUEUE_QTTY); + EXPECT_TRUE(queue.isEmptyByType(MessageType::STATE_LESS)); + + queue.popLastMessage(MessageType::COMMAND); + items = queue.getItemsByType(MessageType::COMMAND); + EXPECT_NE(items, SMALL_QUEUE_QTTY); +} + // TODO TEST_F(QueueTest, Multithread) { + GTEST_SKIP(); MultiTypeQueue queue(BIG_QUEUE_QTTY); auto consumer1 = [&](int& count) @@ -230,4 +313,4 @@ TEST_F(QueueTest, Multithread) consumer_thread1.join(); EXPECT_TRUE(queue.isEmptyByType(MessageType::STATE_LESS)); -} \ No newline at end of file +} diff --git a/src/agent/queue/tests/queue_test.hpp b/src/agent/queue/tests/queue_test.hpp index 24e5dd64f5e..1abdf137252 100644 --- a/src/agent/queue/tests/queue_test.hpp +++ b/src/agent/queue/tests/queue_test.hpp @@ -24,4 +24,4 @@ class QueueTest : public ::testing::Test void TearDown() override; }; -#endif //_QUEUE_TEST_H \ No newline at end of file +#endif //_QUEUE_TEST_H From af616b932a58672ca2f200cc1994a233e2a220cd Mon Sep 17 00:00:00 2001 From: LucioDonda Date: Mon, 15 Jul 2024 20:20:50 -0300 Subject: [PATCH 08/31] fix: all test passing --- src/agent/queue/include/queue.hpp | 27 +-- src/agent/queue/include/shared.hpp | 4 +- src/agent/queue/src/main.cpp | 2 +- src/agent/queue/tests/queue_test.cpp | 242 ++++++++++++++------------- 4 files changed, 136 insertions(+), 139 deletions(-) diff --git a/src/agent/queue/include/queue.hpp b/src/agent/queue/include/queue.hpp index 6b3be30f214..9c47fdc6446 100644 --- a/src/agent/queue/include/queue.hpp +++ b/src/agent/queue/include/queue.hpp @@ -204,7 +204,6 @@ class PersistedQueue class MultiTypeQueue { private: - // std::condition_variable m_cv; std::unordered_map> m_queuesMap; int m_maxItems; @@ -215,7 +214,7 @@ class MultiTypeQueue { // Populate the map inside the constructor body m_queuesMap[MessageType::STATE_LESS] = std::make_unique(MessageType::STATE_LESS, m_maxItems); - m_queuesMap[MessageType::STATE_FULL] = std::make_unique(MessageType::STATE_FULL, m_maxItems); + m_queuesMap[MessageType::STATE_FUL] = std::make_unique(MessageType::STATE_FUL, m_maxItems); m_queuesMap[MessageType::COMMAND] = std::make_unique(MessageType::COMMAND, m_maxItems); } @@ -232,9 +231,7 @@ class MultiTypeQueue MultiTypeQueue& operator=(MultiTypeQueue&&) = delete; // TODO - ~MultiTypeQueue() { - // m_queuesMap; - }; + ~MultiTypeQueue() {}; /** * @brief: push message to a queue of t * @@ -248,7 +245,7 @@ class MultiTypeQueue { // TODO: delete this std::cout << "waiting" << std::endl; - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + std::this_thread::sleep_for(std::chrono::milliseconds(250)); } m_queuesMap[message.type]->insertMessage(message); } @@ -259,7 +256,6 @@ class MultiTypeQueue } } - // FIFO order /** * @brief Get the Last Message object * @@ -300,23 +296,6 @@ class MultiTypeQueue } } - // TODO - /** - * @brief - * - * @param event - */ - // void updateLast(Message event) - // { - // std::unique_lock lock(m_mtx); - // while (m_queuesMap[event.type].empty()) - // { - // m_cv.wait(lock); - // } - // m_queuesMap[event.type] = event; - // m_cv.notify_one(); - // } - /** * @brief * diff --git a/src/agent/queue/include/shared.hpp b/src/agent/queue/include/shared.hpp index 613eda09ff6..89a71b5b3b9 100644 --- a/src/agent/queue/include/shared.hpp +++ b/src/agent/queue/include/shared.hpp @@ -13,7 +13,7 @@ constexpr char DEFAULT_DB_PATH[] = "queue.db"; enum MessageType { STATE_LESS, - STATE_FULL, + STATE_FUL, COMMAND }; @@ -23,7 +23,7 @@ enum MessageType */ std::map MessageTypeName { {MessageType::STATE_LESS, "STATE_LESS"}, - {MessageType::STATE_FULL, "STATE_FULL"}, + {MessageType::STATE_FUL, "STATE_FUL"}, {MessageType::COMMAND, "COMMAND"}, }; diff --git a/src/agent/queue/src/main.cpp b/src/agent/queue/src/main.cpp index a3d32a12d54..7f3efb5480a 100644 --- a/src/agent/queue/src/main.cpp +++ b/src/agent/queue/src/main.cpp @@ -23,7 +23,7 @@ int main() int item; for (int i = 0; i < count; ++i) { - queue.popLastMessage(MessageType::STATE_FULL); + queue.popLastMessage(MessageType::STATE_FUL); std::cout << "Popping event 2: " << std::endl; std::this_thread::sleep_for(std::chrono::milliseconds(100)); } diff --git a/src/agent/queue/tests/queue_test.cpp b/src/agent/queue/tests/queue_test.cpp index 57885fb0677..d797369548c 100644 --- a/src/agent/queue/tests/queue_test.cpp +++ b/src/agent/queue/tests/queue_test.cpp @@ -1,10 +1,9 @@ - -#include - #include #include #include +#include #include +#include #include #include @@ -14,88 +13,49 @@ #define BIG_QUEUE_QTTY 10 #define SMALL_QUEUE_QTTY 2 -/// Helper functions +using json = nlohmann::json; -// TODO: Makes sense to add a full_clean method inside the persistence implementation? -void cleanPersistence() -{ - auto filePath = DEFAULT_DB_PATH; - std::error_code ec; - std::filesystem::remove(filePath, ec); -} +/// Helper functions -// TODO: rework after double check the output of persistence -bool compareJsonStrings(const nlohmann::json& jsonValue1, const nlohmann::json& jsonValue2) +// Unescape Strings +std::string unescape_string(const std::string& str) { + std::string result; + result.reserve(str.length()); - std::string str1 = jsonValue1.get(); - std::string str2 = jsonValue2.get(); - // Remove outer quotes from output if present - auto removeQuotes = [](std::string& inputString) - { - if (inputString.front() == '"' && inputString.back() == '"') - { - inputString = inputString.substr(1, inputString.length() - 2); - } - }; - - removeQuotes(str1); - removeQuotes(str2); - - auto unescapeJsonString = [](std::string input) -> std::string + for (size_t i = 0; i < str.length(); ++i) { - std::string result; - bool escapeNext = false; - for (char c : input) + if (str[i] == '\\' && i + 1 < str.length()) { - if (escapeNext) + switch (str[i + 1]) { - if (c == '"') - { - result += '"'; - } - else - { - result += '\\'; - result += c; - } - escapeNext = false; - } - else - { - if (c == '\\') - { - escapeNext = true; - } - else - { - result += c; - } + case '\\': result += '\\'; break; + case '\"': result += '\"'; break; + case '/': result += '/'; break; + case 'b': result += '\b'; break; + case 'f': result += '\f'; break; + case 'n': result += '\n'; break; + case 'r': result += '\r'; break; + case 't': result += '\t'; break; + default: result += str[i + 1]; } + ++i; } - return result; - }; - - unescapeJsonString(str1); - unescapeJsonString(str2); - - try - { - // Parse both strings into json objects - nlohmann::json json1 = nlohmann::json::parse(str1); - nlohmann::json json2 = nlohmann::json::parse(str2); + else + { + result += str[i]; + } + } - std::cout << "json1: " << json1 << std::endl; - std::cout << "json2: " << json2 << std::endl; + return result; +} - // Compare the json objects - return (json1 == json2); - } - catch (const nlohmann::json::parse_error& e) - { - std::cerr << "JSON parsing error: " << e.what() << std::endl; - return false; - } +// TODO: Makes sense to add a full_clean method inside the persistence implementation? +void cleanPersistence() +{ + auto filePath = DEFAULT_DB_PATH; + std::error_code ec; + std::filesystem::remove(filePath, ec); } // In order to mimic the timeout from outside the queue @@ -138,18 +98,24 @@ void QueueTest::TearDown() {}; /// TESTS -// Check which jsons are correct, do move or delete if out of scope -TEST_F(QueueTest, JSONBaseTest) +// JSON Basic methods. Move or delete if JSON Wrapper is done +TEST_F(QueueTest, JSONConversionComparisson) { nlohmann::json uj1 = {{"version", 1}, {"type", "integer"}}; - nlohmann::json uj2 = {{"type", "integer"}, {"version", 1}}; + // From string. If not unescape then it throws errors + nlohmann::json uj2 = json::parse(unescape_string(R"({\"type\":\"integer\",\"version\":1})")); nlohmann::ordered_json oj1 = {{"version", 1}, {"type", "integer"}}; nlohmann::ordered_json oj2 = {{"type", "integer"}, {"version", 1}}; - - EXPECT_TRUE(uj1 == uj2); - //EXPECT_TRUE(compareJsonStrings(uj1, uj2)); EXPECT_FALSE(oj1 == oj2); + + auto versionUj1 = uj1["version"].template get(); + auto versionUj2 = uj2["version"].template get(); + EXPECT_EQ(versionUj1, versionUj2); + + auto typeUj1 = uj1["type"].template get(); + auto typeUj2 = uj2["type"].template get(); + EXPECT_EQ(typeUj1, typeUj2); } // Push, get and check the queue is not empty @@ -167,7 +133,10 @@ TEST_F(QueueTest, BasicPushGetNotEmpty) auto typeReceived = messageResponse.type; EXPECT_TRUE(typeSend == typeReceived); - // EXPECT_TRUE(compareJsonStrings(messageResponse.data[0], messageToSend.data)); + auto versionUj1 = messageResponse.data[0].template get(); + auto versionUj2 = messageToSend.data.template get(); + EXPECT_EQ(versionUj1, versionUj2); + EXPECT_FALSE(queue.isEmptyByType(MessageType::STATE_LESS)); } @@ -186,7 +155,9 @@ TEST_F(QueueTest, SinglePushPopEmpty) auto typeReceived = messageResponse.type; EXPECT_TRUE(typeSend == typeReceived); - // EXPECT_TRUE(compareJsonStrings(messageResponse.data[0], messageToSend.data)); + auto versionUj1 = messageResponse.data[0].template get(); + auto versionUj2 = messageToSend.data.template get(); + EXPECT_EQ(versionUj1, versionUj2); queue.popLastMessage(MessageType::STATE_LESS); EXPECT_TRUE(queue.isEmptyByType(MessageType::STATE_LESS)); @@ -217,14 +188,14 @@ TEST_F(QueueTest, SingleGetPopOnEmpty) queue.push(messageToSend); EXPECT_FALSE(queue.isEmptyByType(MessageType::STATE_LESS)); - EXPECT_TRUE(queue.isEmptyByType(MessageType::STATE_FULL)); + EXPECT_TRUE(queue.isEmptyByType(MessageType::STATE_FUL)); - auto messageResponse = queue.getLastMessage(MessageType::STATE_FULL); - EXPECT_EQ(messageResponse.type, MessageType::STATE_FULL); + auto messageResponse = queue.getLastMessage(MessageType::STATE_FUL); + EXPECT_EQ(messageResponse.type, MessageType::STATE_FUL); EXPECT_EQ(messageResponse.data[0], nullptr); - queue.popLastMessage(MessageType::STATE_FULL); - EXPECT_TRUE(queue.isEmptyByType(MessageType::STATE_FULL)); + queue.popLastMessage(MessageType::STATE_FUL); + EXPECT_TRUE(queue.isEmptyByType(MessageType::STATE_FUL)); } // Push, get and check while the queue is full @@ -245,8 +216,7 @@ TEST_F(QueueTest, SinglePushPopFull) try { - functionWithTimeout( - [&queue](Message& message) { queue.push(message); }, 1000, exampleMessage); + functionWithTimeout([&queue](Message& message) { queue.push(message); }, 1000, exampleMessage); } catch (const std::runtime_error& e) { @@ -262,55 +232,103 @@ TEST_F(QueueTest, SinglePushPopFull) EXPECT_NE(items, SMALL_QUEUE_QTTY); } -// TODO -TEST_F(QueueTest, Multithread) +// Accesing different types of queues +TEST_F(QueueTest, MultithreadDifferentType) { - GTEST_SKIP(); MultiTypeQueue queue(BIG_QUEUE_QTTY); - auto consumer1 = [&](int& count) + auto consumerStateLess = [&](int& count) { - // std::this_thread::sleep_for(std::chrono::milliseconds(1000)); - int item; for (int i = 0; i < count; ++i) { queue.popLastMessage(MessageType::STATE_LESS); - std::cout << "Popping event 1: " << std::endl; } }; - auto consumer2 = [&](int& count) + auto consumerStateFull = [&](int& count) { - int item; for (int i = 0; i < count; ++i) { - queue.popLastMessage(MessageType::STATE_FULL); - std::cout << "Popping event 2: " << std::endl; + queue.popLastMessage(MessageType::STATE_FUL); } }; - auto producer = [&](int& count) + auto messageProducer = [&](int& count) { for (int i = 0; i < count; ++i) { - auto message = R"({"Data" : "for STATE_LESS)" + std::to_string(i) + R"("})"; - queue.push(Message(MessageType::STATE_LESS, message)); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); // Simulate work + const nlohmann::json dataContent = R"({{"Data", "for STATE_LESS)" + std::to_string(i) + R"("}})"; + queue.push(Message(MessageType::STATE_LESS, dataContent)); + queue.push(Message(MessageType::STATE_FUL, dataContent)); } }; - int items_to_insert = 5; - int items_to_consume = 2; + int itemsToInsert = 9; + int itemsToConsume = 5; + + messageProducer(itemsToInsert); - producer(items_to_insert); + std::thread consumerThread1(consumerStateLess, std::ref(itemsToConsume)); + std::thread consumerThread2(consumerStateFull, std::ref(itemsToConsume)); - std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + consumerThread1.join(); + consumerThread2.join(); - std::thread consumer_thread1(consumer1, std::ref(items_to_consume)); - std::thread producer_thread1(consumer2, std::ref(items_to_consume)); + EXPECT_FALSE(queue.isEmptyByType(MessageType::STATE_LESS)); + EXPECT_FALSE(queue.isEmptyByType(MessageType::STATE_FUL)); - producer_thread1.join(); - consumer_thread1.join(); + std::thread consumerThread12(consumerStateLess, std::ref(itemsToConsume)); + std::thread consumerThread22(consumerStateFull, std::ref(itemsToConsume)); + + consumerThread22.join(); + consumerThread12.join(); EXPECT_TRUE(queue.isEmptyByType(MessageType::STATE_LESS)); + EXPECT_TRUE(queue.isEmptyByType(MessageType::STATE_FUL)); +} + +// Accesing same queue +TEST_F(QueueTest, MultithreadSameType) +{ + MultiTypeQueue queue(BIG_QUEUE_QTTY); + + auto consumerCommand1 = [&](int& count) + { + for (int i = 0; i < count; ++i) + { + queue.popLastMessage(MessageType::COMMAND); + } + }; + + auto consumerCommand2 = [&](int& count) + { + for (int i = 0; i < count; ++i) + { + queue.popLastMessage(MessageType::COMMAND); + } + }; + + auto messageProducer = [&](int& count) + { + for (int i = 0; i < count; ++i) + { + const nlohmann::json dataContent = R"({{"Data", "for COMMAND)" + std::to_string(i) + R"("}})"; + queue.push(Message(MessageType::COMMAND, dataContent)); + } + }; + + int itemsToInsert = 10; + int itemsToConsume = 5; + + messageProducer(itemsToInsert); + + EXPECT_EQ(itemsToInsert, queue.getItemsByType(MessageType::COMMAND)); + + std::thread consumerThread1(consumerCommand1, std::ref(itemsToConsume)); + std::thread messageProducerThread1(consumerCommand2, std::ref(itemsToConsume)); + + messageProducerThread1.join(); + consumerThread1.join(); + + EXPECT_TRUE(queue.isEmptyByType(MessageType::COMMAND)); } From b50a805f4d69e7c8728b3158223a843e2ac0f196 Mon Sep 17 00:00:00 2001 From: LucioDonda Date: Wed, 17 Jul 2024 16:24:42 -0300 Subject: [PATCH 09/31] feat: add push and pop for multiple messages --- src/agent/queue/CMakeLists.txt | 2 + src/agent/queue/include/filestorage.hpp | 2 +- src/agent/queue/include/queue.hpp | 124 ++++++++++++++++-------- src/agent/queue/src/sqlitestorage.cpp | 22 +++-- src/agent/queue/tests/queue_test.cpp | 124 ++++++++++++++++++------ src/agent/queue/tests/queue_test.hpp | 11 +++ 6 files changed, 206 insertions(+), 79 deletions(-) diff --git a/src/agent/queue/CMakeLists.txt b/src/agent/queue/CMakeLists.txt index 9d5c9d580f5..b4c300cab4f 100644 --- a/src/agent/queue/CMakeLists.txt +++ b/src/agent/queue/CMakeLists.txt @@ -9,6 +9,8 @@ set(CMAKE_CXX_STANDARD_REQUIRED True) find_package(SQLite3 REQUIRED) find_package(nlohmann_json REQUIRED) +set(CMAKE_CXX_FLAGS "-Wall -Wextra -Wunused -pthread") + # Add the include directory for shared headers include_directories(${CMAKE_SOURCE_DIR}/include) diff --git a/src/agent/queue/include/filestorage.hpp b/src/agent/queue/include/filestorage.hpp index 081cc071cee..bac6d19ecea 100644 --- a/src/agent/queue/include/filestorage.hpp +++ b/src/agent/queue/include/filestorage.hpp @@ -103,7 +103,7 @@ class FileStorage : public Persistence * @param LineID * @return json */ - json Retrieve(int LineID) override + json Retrieve(int lineID) override { return RetrieveMultiple(1); } diff --git a/src/agent/queue/include/queue.hpp b/src/agent/queue/include/queue.hpp index 9c47fdc6446..3eed036290f 100644 --- a/src/agent/queue/include/queue.hpp +++ b/src/agent/queue/include/queue.hpp @@ -58,6 +58,8 @@ class PersistedQueue // MessageTypeName.at(m_queueType)}); m_persistenceDest = PersistenceFactory::createPersistence( "SQLite3", {static_cast(DEFAULT_DB_PATH), MessageTypeName.at(m_queueType)}); + // updates the actual size being used + getItemsAvailable(); } // Delete copy constructor @@ -89,7 +91,10 @@ class PersistedQueue int getItemsAvailable() { - return m_persistenceDest->GetElementCount(); + std::unique_lock lock(m_mtx); + //TODO: rework this behavior + m_size = m_persistenceDest->GetElementCount(); + return m_size; } /** @@ -97,22 +102,12 @@ class PersistedQueue * * @return int */ - // TODO check this functionality + // TODO check this functionality because when inserting data arrays it loses actual value int getSize() const { return m_size; } - /** - * @brief Set the Size object - * - * @param m_size - */ - void setSize(int m_size) - { - this->m_size = m_size; - } - /** * @brief * @@ -126,32 +121,31 @@ class PersistedQueue m_persistenceDest->Store(event.data); m_size++; m_cv.notify_one(); + // TODO: make Store method int for returning items inserted when array + // if(m_persistenceDest->Store(event.data)) + // { + // m_size++; + // m_cv.notify_one(); + // } return true; }; - /** - * @brief - * - * @return true - * @return false - */ - bool removeMessage() + bool removeNMessages(int qttyMessages) { - if (m_size != 0) + bool result = false; + // workaround for items issue + getItemsAvailable(); + if (m_size) { std::unique_lock lock(m_mtx); - auto linesRemoved = m_persistenceDest->RemoveMultiple(1); - m_size = m_size - linesRemoved; + auto linesRemoved = m_persistenceDest->RemoveMultiple(qttyMessages); + if(linesRemoved) + { + result = true; + m_size = m_size - linesRemoved; + } } - return true; - }; - - bool removeNMessages(int qttyMessages) - { - std::unique_lock lock(m_mtx); - auto linesRemoved = m_persistenceDest->RemoveMultiple(qttyMessages); - m_size = m_size - linesRemoved; - return true; + return result; }; /** @@ -162,7 +156,8 @@ class PersistedQueue Message getMessage() { std::unique_lock lock(m_mtx); - return Message(m_queueType, m_persistenceDest->RetrieveMultiple(1)); + auto messageData = m_persistenceDest->RetrieveMultiple(1); + return Message(m_queueType, messageData); }; /** @@ -174,7 +169,8 @@ class PersistedQueue Message getNMessages(int n) { std::unique_lock lock(m_mtx); - return Message(m_queueType, m_persistenceDest->RetrieveMultiple(n)); + auto messageData = m_persistenceDest->RetrieveMultiple(n); + return Message(m_queueType, messageData); }; /** @@ -183,15 +179,15 @@ class PersistedQueue * @return true * @return false */ - bool empty() + bool empty() const { return m_size == 0; } private: - MessageType m_queueType; + const MessageType m_queueType; int m_size = 0; - int m_max_size; + const int m_max_size; std::unique_ptr m_persistenceDest; std::mutex m_mtx; std::condition_variable m_cv; @@ -205,7 +201,7 @@ class MultiTypeQueue { private: std::unordered_map> m_queuesMap; - int m_maxItems; + const int m_maxItems; public: // Create a vector with 3 PersistedQueue elements @@ -237,8 +233,9 @@ class MultiTypeQueue * * @param message */ - void push(Message message) + bool push(Message message) { + bool result = false; if (m_queuesMap.contains(message.type)) { while (m_queuesMap[message.type]->getSize() == m_maxItems) @@ -247,13 +244,27 @@ class MultiTypeQueue std::cout << "waiting" << std::endl; std::this_thread::sleep_for(std::chrono::milliseconds(250)); } - m_queuesMap[message.type]->insertMessage(message); + result = m_queuesMap[message.type]->insertMessage(message); } else { // TODO: error / logging handling !!! std::cout << "error didn't find the queue" << std::endl; } + return result; + } + + /** + * @brief + * + * @param messages + */ + void push(std::vector messages) + { + for (const auto& singleMessage : messages) + { + push(singleMessage); + } } /** @@ -278,22 +289,49 @@ class MultiTypeQueue } /** - * @brief + * @brief deletes a message from a queue * - * @param type + * @param type MessageType queue to pop + * @return true popped succesfully + * @return false wasn't able to pop message */ - void popLastMessage(MessageType type) + bool popLastMessage(MessageType type) { + bool result = false; if (m_queuesMap.contains(type)) { // Handle return value - m_queuesMap[type]->removeMessage(); + result = m_queuesMap[type]->removeNMessages(1); } else { // TODO: error / logging handling !!! std::cout << "error didn't find the queue" << std::endl; } + return result; + } + + /** + * @brief + * + * @param type + * @param messageQuantity + * @return true + * @return false + */ + bool popNMessages(MessageType type, int messageQuantity) + { + bool result = false; + if (m_queuesMap.contains(type)) + { + result = m_queuesMap[type]->removeNMessages(messageQuantity); + } + else + { + // TODO: error / logging handling !!! + std::cout << "error didn't find the queue" << std::endl; + } + return result; } /** diff --git a/src/agent/queue/src/sqlitestorage.cpp b/src/agent/queue/src/sqlitestorage.cpp index bc462f5906b..d6ebc4d08d5 100644 --- a/src/agent/queue/src/sqlitestorage.cpp +++ b/src/agent/queue/src/sqlitestorage.cpp @@ -40,16 +40,26 @@ void SQLiteStorage::CloseDB() { void SQLiteStorage::Store(const json& message) { std::lock_guard lock(m_mtx); - OpenDB(); - std::string insertQuery = INSERT_QUERY; + OpenDB(); sqlite3_stmt* stmt; sqlite3_prepare_v2(m_db, insertQuery.c_str(), -1, &stmt, 0); - std::string messageStr = message.dump(); - sqlite3_bind_text(stmt, 1, messageStr.c_str(), -1, SQLITE_TRANSIENT); - sqlite3_step(stmt); + if(message.is_array()) + { + for (auto& singleMessageData : message.items()) + { + std::string messageStr = singleMessageData.value(); + sqlite3_bind_text(stmt, 1, messageStr.c_str(), -1, SQLITE_TRANSIENT); + sqlite3_step(stmt); + } + } + else + { + std::string messageStr = message.dump(); + sqlite3_bind_text(stmt, 1, messageStr.c_str(), -1, SQLITE_TRANSIENT); + sqlite3_step(stmt); + } sqlite3_finalize(stmt); - CloseDB(); } diff --git a/src/agent/queue/tests/queue_test.cpp b/src/agent/queue/tests/queue_test.cpp index d797369548c..511f032b3ad 100644 --- a/src/agent/queue/tests/queue_test.cpp +++ b/src/agent/queue/tests/queue_test.cpp @@ -10,10 +10,14 @@ #include "queue.hpp" #include "queue_test.hpp" +using json = nlohmann::json; + #define BIG_QUEUE_QTTY 10 #define SMALL_QUEUE_QTTY 2 -using json = nlohmann::json; +const json baseDataContent = R"({{"data": "for STATE_LESS_0"}})"; +const json multipleDataContent = R"({{"content 1", "content 2", "content 3"}})"; +// TODO: test string: const std::string multipleDataContent = R"({"data": {"content 1", "content 2", "content 3"})"; /// Helper functions @@ -99,11 +103,11 @@ void QueueTest::TearDown() {}; /// TESTS // JSON Basic methods. Move or delete if JSON Wrapper is done -TEST_F(QueueTest, JSONConversionComparisson) +TEST_F(JsonTest, JSONConversionComparisson) { - nlohmann::json uj1 = {{"version", 1}, {"type", "integer"}}; + json uj1 = {{"version", 1}, {"type", "integer"}}; // From string. If not unescape then it throws errors - nlohmann::json uj2 = json::parse(unescape_string(R"({\"type\":\"integer\",\"version\":1})")); + json uj2 = json::parse(unescape_string(R"({\"type\":\"integer\",\"version\":1})")); nlohmann::ordered_json oj1 = {{"version", 1}, {"type", "integer"}}; nlohmann::ordered_json oj2 = {{"type", "integer"}, {"version", 1}}; @@ -118,24 +122,42 @@ TEST_F(QueueTest, JSONConversionComparisson) EXPECT_EQ(typeUj1, typeUj2); } +TEST_F(JsonTest, JSONArrays) +{ + // create JSON values + json j_object = {{"one", 1}, {"two", 2}, {"three", 3}}; + json j_array = {1, 2, 4, 8, 16}; + json multipleDataContent = {"content 1", "content 2", "content 3"}; + + // call is_array() + EXPECT_FALSE(j_object.is_array()); + EXPECT_TRUE(j_array.is_array()); + EXPECT_TRUE(multipleDataContent.is_array()); + + int i = 0; + for (auto& singleMessage : multipleDataContent.items()) + { + EXPECT_EQ(singleMessage.value(),"content " + std::to_string(++i)); + } +} + // Push, get and check the queue is not empty -TEST_F(QueueTest, BasicPushGetNotEmpty) +TEST_F(QueueTest, SinglePushGetNotEmpty) { MultiTypeQueue queue(BIG_QUEUE_QTTY); const MessageType messageType {MessageType::STATE_LESS}; - const nlohmann::json dataContent = R"({{"Data", "for STATE_LESS)" + std::to_string(0) + R"("}})"; - const Message messageToSend {messageType, dataContent}; + const Message messageToSend {messageType, baseDataContent}; queue.push(messageToSend); auto messageResponse = queue.getLastMessage(MessageType::STATE_LESS); - auto typeSend = messageResponse.type; + auto typeSend = messageToSend.type; auto typeReceived = messageResponse.type; - EXPECT_TRUE(typeSend == typeReceived); - auto versionUj1 = messageResponse.data[0].template get(); - auto versionUj2 = messageToSend.data.template get(); - EXPECT_EQ(versionUj1, versionUj2); + + auto dataResponse = messageResponse.data[0].template get(); + auto dataToSend = messageToSend.data.template get(); + EXPECT_EQ(dataResponse, dataToSend); EXPECT_FALSE(queue.isEmptyByType(MessageType::STATE_LESS)); } @@ -145,19 +167,18 @@ TEST_F(QueueTest, SinglePushPopEmpty) { MultiTypeQueue queue(BIG_QUEUE_QTTY); const MessageType messageType {MessageType::STATE_LESS}; - const nlohmann::json dataContent = R"({{"Data", "for STATE_LESS)" + std::to_string(0) + R"("}})"; - const Message messageToSend {messageType, dataContent}; + const Message messageToSend {messageType, baseDataContent}; queue.push(messageToSend); auto messageResponse = queue.getLastMessage(MessageType::STATE_LESS); - auto typeSend = messageResponse.type; + auto typeSend = messageToSend.type; auto typeReceived = messageResponse.type; - EXPECT_TRUE(typeSend == typeReceived); - auto versionUj1 = messageResponse.data[0].template get(); - auto versionUj2 = messageToSend.data.template get(); - EXPECT_EQ(versionUj1, versionUj2); + + auto dataResponse = messageResponse.data[0].template get(); + auto dataToSend = messageToSend.data.template get(); + EXPECT_EQ(dataResponse, dataToSend); queue.popLastMessage(MessageType::STATE_LESS); EXPECT_TRUE(queue.isEmptyByType(MessageType::STATE_LESS)); @@ -168,8 +189,7 @@ TEST_F(QueueTest, SinglePushPop) { MultiTypeQueue queue(BIG_QUEUE_QTTY); const MessageType messageType {MessageType::STATE_LESS}; - const nlohmann::json dataContent = R"({"Data" : "for STATE_LESS)" + std::to_string(0) + R"("})"; - const Message messageToSend {messageType, dataContent}; + const Message messageToSend {messageType, baseDataContent}; queue.push(messageToSend); EXPECT_FALSE(queue.isEmptyByType(MessageType::STATE_LESS)); @@ -183,8 +203,7 @@ TEST_F(QueueTest, SingleGetPopOnEmpty) { MultiTypeQueue queue(BIG_QUEUE_QTTY); const MessageType messageType {MessageType::STATE_LESS}; - const nlohmann::json dataContent = R"({"Data" : "for STATE_LESS)" + std::to_string(0) + R"("})"; - const Message messageToSend {messageType, dataContent}; + const Message messageToSend {messageType, baseDataContent}; queue.push(messageToSend); EXPECT_FALSE(queue.isEmptyByType(MessageType::STATE_LESS)); @@ -207,11 +226,11 @@ TEST_F(QueueTest, SinglePushPopFull) const MessageType messageType {MessageType::COMMAND}; for (int i : {1, 2}) { - const nlohmann::json dataContent = R"({"Data" : "for COMMAND)" + std::to_string(i) + R"("})"; + const json dataContent = R"({"Data" : "for COMMAND)" + std::to_string(i) + R"("})"; queue.push({messageType, dataContent}); } - const nlohmann::json dataContent = R"({"Data" : "for COMMAND3"})"; + const json dataContent = R"({"Data" : "for COMMAND3"})"; Message exampleMessage {messageType, dataContent}; try @@ -235,6 +254,7 @@ TEST_F(QueueTest, SinglePushPopFull) // Accesing different types of queues TEST_F(QueueTest, MultithreadDifferentType) { + GTEST_SKIP(); MultiTypeQueue queue(BIG_QUEUE_QTTY); auto consumerStateLess = [&](int& count) @@ -257,13 +277,13 @@ TEST_F(QueueTest, MultithreadDifferentType) { for (int i = 0; i < count; ++i) { - const nlohmann::json dataContent = R"({{"Data", "for STATE_LESS)" + std::to_string(i) + R"("}})"; + const json dataContent = R"({{"Data", "for STATE_LESS)" + std::to_string(i) + R"("}})"; queue.push(Message(MessageType::STATE_LESS, dataContent)); queue.push(Message(MessageType::STATE_FUL, dataContent)); } }; - int itemsToInsert = 9; + int itemsToInsert = 10; int itemsToConsume = 5; messageProducer(itemsToInsert); @@ -274,15 +294,21 @@ TEST_F(QueueTest, MultithreadDifferentType) consumerThread1.join(); consumerThread2.join(); + EXPECT_NE(0, queue.getItemsByType(MessageType::STATE_LESS)); + EXPECT_NE(0, queue.getItemsByType(MessageType::STATE_FUL)); EXPECT_FALSE(queue.isEmptyByType(MessageType::STATE_LESS)); EXPECT_FALSE(queue.isEmptyByType(MessageType::STATE_FUL)); + // Consume the rest of the messages std::thread consumerThread12(consumerStateLess, std::ref(itemsToConsume)); std::thread consumerThread22(consumerStateFull, std::ref(itemsToConsume)); - consumerThread22.join(); consumerThread12.join(); + consumerThread22.join(); + // FIXME: this doesn't match + EXPECT_EQ(0, queue.getItemsByType(MessageType::STATE_LESS)); + EXPECT_EQ(0, queue.getItemsByType(MessageType::STATE_FUL)); EXPECT_TRUE(queue.isEmptyByType(MessageType::STATE_LESS)); EXPECT_TRUE(queue.isEmptyByType(MessageType::STATE_FUL)); } @@ -290,6 +316,7 @@ TEST_F(QueueTest, MultithreadDifferentType) // Accesing same queue TEST_F(QueueTest, MultithreadSameType) { + // Failing due to some issue deletting MultiTypeQueue queue(BIG_QUEUE_QTTY); auto consumerCommand1 = [&](int& count) @@ -312,7 +339,7 @@ TEST_F(QueueTest, MultithreadSameType) { for (int i = 0; i < count; ++i) { - const nlohmann::json dataContent = R"({{"Data", "for COMMAND)" + std::to_string(i) + R"("}})"; + const json dataContent = R"({{"Data": "for COMMAND)" + std::to_string(i) + R"("}})"; queue.push(Message(MessageType::COMMAND, dataContent)); } }; @@ -332,3 +359,42 @@ TEST_F(QueueTest, MultithreadSameType) EXPECT_TRUE(queue.isEmptyByType(MessageType::COMMAND)); } + +// Push Multiple with single message and data array, +// several gets, checks and pops +TEST_F(QueueTest, MultiplePushSeveralSingleGets) +{ + GTEST_SKIP(); + MultiTypeQueue queue(BIG_QUEUE_QTTY); + const MessageType messageType {MessageType::STATE_LESS}; + // TODO: double check array of objects + const json multipleDataContent = {"content 1", "content 2", "content 3"}; + const Message messageToSend {messageType, multipleDataContent}; + + queue.push(messageToSend); + + for (int i : {0, 1, 2}) + { + auto messageResponse = queue.getLastMessage(MessageType::STATE_LESS); + auto responseData = messageResponse.data[0].template get(); + auto sentData = messageToSend.data[i].template get(); + EXPECT_EQ(responseData, sentData); + } + + EXPECT_EQ(queue.getItemsByType(MessageType::STATE_LESS), 0); +} + +// Push Multiple, pop multiples +TEST_F(QueueTest, MultiplePushSeveralMultiplePops) +{ + MultiTypeQueue queue(BIG_QUEUE_QTTY); + const MessageType messageType {MessageType::STATE_LESS}; + const json multipleDataContent = {"content 1", "content 2", "content 3"}; + const Message messageToSend {messageType, multipleDataContent}; + + queue.push(messageToSend); + + EXPECT_TRUE(queue.popNMessages(MessageType::STATE_LESS, 3)); + EXPECT_TRUE(queue.isEmptyByType(MessageType::STATE_LESS)); + EXPECT_EQ(0,queue.getItemsByType(MessageType::STATE_LESS)); +} diff --git a/src/agent/queue/tests/queue_test.hpp b/src/agent/queue/tests/queue_test.hpp index 1abdf137252..548a432a121 100644 --- a/src/agent/queue/tests/queue_test.hpp +++ b/src/agent/queue/tests/queue_test.hpp @@ -24,4 +24,15 @@ class QueueTest : public ::testing::Test void TearDown() override; }; +class JsonTest : public ::testing::Test +{ + protected: + + JsonTest() = default; + virtual ~JsonTest() = default; + + void SetUp() override {}; + void TearDown() override {}; +}; + #endif //_QUEUE_TEST_H From 8224c99b407c67b626e1ab7600d3809fdcddd472 Mon Sep 17 00:00:00 2001 From: LucioDonda Date: Wed, 17 Jul 2024 18:10:35 -0300 Subject: [PATCH 10/31] feat: rename STATEs enum values --- src/agent/queue/include/queue.hpp | 6 +-- src/agent/queue/include/shared.hpp | 8 +-- src/agent/queue/src/main.cpp | 8 +-- src/agent/queue/tests/queue_test.cpp | 80 ++++++++++++++-------------- 4 files changed, 51 insertions(+), 51 deletions(-) diff --git a/src/agent/queue/include/queue.hpp b/src/agent/queue/include/queue.hpp index 3eed036290f..d616e2440f6 100644 --- a/src/agent/queue/include/queue.hpp +++ b/src/agent/queue/include/queue.hpp @@ -186,7 +186,7 @@ class PersistedQueue private: const MessageType m_queueType; - int m_size = 0; + std::atomic m_size = 0; const int m_max_size; std::unique_ptr m_persistenceDest; std::mutex m_mtx; @@ -209,8 +209,8 @@ class MultiTypeQueue : m_maxItems(size) { // Populate the map inside the constructor body - m_queuesMap[MessageType::STATE_LESS] = std::make_unique(MessageType::STATE_LESS, m_maxItems); - m_queuesMap[MessageType::STATE_FUL] = std::make_unique(MessageType::STATE_FUL, m_maxItems); + m_queuesMap[MessageType::STATELESS] = std::make_unique(MessageType::STATELESS, m_maxItems); + m_queuesMap[MessageType::STATEFUL] = std::make_unique(MessageType::STATEFUL, m_maxItems); m_queuesMap[MessageType::COMMAND] = std::make_unique(MessageType::COMMAND, m_maxItems); } diff --git a/src/agent/queue/include/shared.hpp b/src/agent/queue/include/shared.hpp index 89a71b5b3b9..205c8fa2539 100644 --- a/src/agent/queue/include/shared.hpp +++ b/src/agent/queue/include/shared.hpp @@ -12,8 +12,8 @@ constexpr char DEFAULT_DB_PATH[] = "queue.db"; */ enum MessageType { - STATE_LESS, - STATE_FUL, + STATELESS, + STATEFUL, COMMAND }; @@ -22,8 +22,8 @@ enum MessageType * */ std::map MessageTypeName { - {MessageType::STATE_LESS, "STATE_LESS"}, - {MessageType::STATE_FUL, "STATE_FUL"}, + {MessageType::STATELESS, "STATELESS"}, + {MessageType::STATEFUL, "STATEFUL"}, {MessageType::COMMAND, "COMMAND"}, }; diff --git a/src/agent/queue/src/main.cpp b/src/agent/queue/src/main.cpp index 7f3efb5480a..72e793ba9e1 100644 --- a/src/agent/queue/src/main.cpp +++ b/src/agent/queue/src/main.cpp @@ -13,7 +13,7 @@ int main() int item; for (int i = 0; i < count; ++i) { - queue.popLastMessage(MessageType::STATE_LESS); + queue.popLastMessage(MessageType::STATELESS); std::cout << "Popping event 1: " << std::endl; std::this_thread::sleep_for(std::chrono::milliseconds(100)); } @@ -23,7 +23,7 @@ int main() int item; for (int i = 0; i < count; ++i) { - queue.popLastMessage(MessageType::STATE_FUL); + queue.popLastMessage(MessageType::STATEFUL); std::cout << "Popping event 2: " << std::endl; std::this_thread::sleep_for(std::chrono::milliseconds(100)); } @@ -33,8 +33,8 @@ int main() auto producer = [&](int &count) { for (int i = 0; i < count; ++i) { - auto message = R"({"Data" : "for STATE_LESS)" + std::to_string(i) + R"("})"; - queue.push(Message(MessageType::STATE_LESS, message)); + auto message = R"({"Data" : "for STATELESS)" + std::to_string(i) + R"("})"; + queue.push(Message(MessageType::STATELESS, message)); std::this_thread::sleep_for(std::chrono::milliseconds(100)); // Simulate work } }; diff --git a/src/agent/queue/tests/queue_test.cpp b/src/agent/queue/tests/queue_test.cpp index 511f032b3ad..c66c6fe298a 100644 --- a/src/agent/queue/tests/queue_test.cpp +++ b/src/agent/queue/tests/queue_test.cpp @@ -15,7 +15,7 @@ using json = nlohmann::json; #define BIG_QUEUE_QTTY 10 #define SMALL_QUEUE_QTTY 2 -const json baseDataContent = R"({{"data": "for STATE_LESS_0"}})"; +const json baseDataContent = R"({{"data": "for STATELESS_0"}})"; const json multipleDataContent = R"({{"content 1", "content 2", "content 3"}})"; // TODO: test string: const std::string multipleDataContent = R"({"data": {"content 1", "content 2", "content 3"})"; @@ -145,11 +145,11 @@ TEST_F(JsonTest, JSONArrays) TEST_F(QueueTest, SinglePushGetNotEmpty) { MultiTypeQueue queue(BIG_QUEUE_QTTY); - const MessageType messageType {MessageType::STATE_LESS}; + const MessageType messageType {MessageType::STATELESS}; const Message messageToSend {messageType, baseDataContent}; queue.push(messageToSend); - auto messageResponse = queue.getLastMessage(MessageType::STATE_LESS); + auto messageResponse = queue.getLastMessage(MessageType::STATELESS); auto typeSend = messageToSend.type; auto typeReceived = messageResponse.type; @@ -159,18 +159,18 @@ TEST_F(QueueTest, SinglePushGetNotEmpty) auto dataToSend = messageToSend.data.template get(); EXPECT_EQ(dataResponse, dataToSend); - EXPECT_FALSE(queue.isEmptyByType(MessageType::STATE_LESS)); + EXPECT_FALSE(queue.isEmptyByType(MessageType::STATELESS)); } // push and pop on a non-full queue TEST_F(QueueTest, SinglePushPopEmpty) { MultiTypeQueue queue(BIG_QUEUE_QTTY); - const MessageType messageType {MessageType::STATE_LESS}; + const MessageType messageType {MessageType::STATELESS}; const Message messageToSend {messageType, baseDataContent}; queue.push(messageToSend); - auto messageResponse = queue.getLastMessage(MessageType::STATE_LESS); + auto messageResponse = queue.getLastMessage(MessageType::STATELESS); auto typeSend = messageToSend.type; auto typeReceived = messageResponse.type; @@ -180,41 +180,41 @@ TEST_F(QueueTest, SinglePushPopEmpty) auto dataToSend = messageToSend.data.template get(); EXPECT_EQ(dataResponse, dataToSend); - queue.popLastMessage(MessageType::STATE_LESS); - EXPECT_TRUE(queue.isEmptyByType(MessageType::STATE_LESS)); + queue.popLastMessage(MessageType::STATELESS); + EXPECT_TRUE(queue.isEmptyByType(MessageType::STATELESS)); } // Push, pop and check the queue is empty TEST_F(QueueTest, SinglePushPop) { MultiTypeQueue queue(BIG_QUEUE_QTTY); - const MessageType messageType {MessageType::STATE_LESS}; + const MessageType messageType {MessageType::STATELESS}; const Message messageToSend {messageType, baseDataContent}; queue.push(messageToSend); - EXPECT_FALSE(queue.isEmptyByType(MessageType::STATE_LESS)); + EXPECT_FALSE(queue.isEmptyByType(MessageType::STATELESS)); - queue.popLastMessage(MessageType::STATE_LESS); - EXPECT_TRUE(queue.isEmptyByType(MessageType::STATE_LESS)); + queue.popLastMessage(MessageType::STATELESS); + EXPECT_TRUE(queue.isEmptyByType(MessageType::STATELESS)); } // get and pop on a empty queue TEST_F(QueueTest, SingleGetPopOnEmpty) { MultiTypeQueue queue(BIG_QUEUE_QTTY); - const MessageType messageType {MessageType::STATE_LESS}; + const MessageType messageType {MessageType::STATELESS}; const Message messageToSend {messageType, baseDataContent}; queue.push(messageToSend); - EXPECT_FALSE(queue.isEmptyByType(MessageType::STATE_LESS)); - EXPECT_TRUE(queue.isEmptyByType(MessageType::STATE_FUL)); + EXPECT_FALSE(queue.isEmptyByType(MessageType::STATELESS)); + EXPECT_TRUE(queue.isEmptyByType(MessageType::STATEFUL)); - auto messageResponse = queue.getLastMessage(MessageType::STATE_FUL); - EXPECT_EQ(messageResponse.type, MessageType::STATE_FUL); + auto messageResponse = queue.getLastMessage(MessageType::STATEFUL); + EXPECT_EQ(messageResponse.type, MessageType::STATEFUL); EXPECT_EQ(messageResponse.data[0], nullptr); - queue.popLastMessage(MessageType::STATE_FUL); - EXPECT_TRUE(queue.isEmptyByType(MessageType::STATE_FUL)); + queue.popLastMessage(MessageType::STATEFUL); + EXPECT_TRUE(queue.isEmptyByType(MessageType::STATEFUL)); } // Push, get and check while the queue is full @@ -244,7 +244,7 @@ TEST_F(QueueTest, SinglePushPopFull) auto items = queue.getItemsByType(MessageType::COMMAND); EXPECT_EQ(items, SMALL_QUEUE_QTTY); - EXPECT_TRUE(queue.isEmptyByType(MessageType::STATE_LESS)); + EXPECT_TRUE(queue.isEmptyByType(MessageType::STATELESS)); queue.popLastMessage(MessageType::COMMAND); items = queue.getItemsByType(MessageType::COMMAND); @@ -261,7 +261,7 @@ TEST_F(QueueTest, MultithreadDifferentType) { for (int i = 0; i < count; ++i) { - queue.popLastMessage(MessageType::STATE_LESS); + queue.popLastMessage(MessageType::STATELESS); } }; @@ -269,7 +269,7 @@ TEST_F(QueueTest, MultithreadDifferentType) { for (int i = 0; i < count; ++i) { - queue.popLastMessage(MessageType::STATE_FUL); + queue.popLastMessage(MessageType::STATEFUL); } }; @@ -277,9 +277,9 @@ TEST_F(QueueTest, MultithreadDifferentType) { for (int i = 0; i < count; ++i) { - const json dataContent = R"({{"Data", "for STATE_LESS)" + std::to_string(i) + R"("}})"; - queue.push(Message(MessageType::STATE_LESS, dataContent)); - queue.push(Message(MessageType::STATE_FUL, dataContent)); + const json dataContent = R"({{"Data", "for STATELESS)" + std::to_string(i) + R"("}})"; + queue.push(Message(MessageType::STATELESS, dataContent)); + queue.push(Message(MessageType::STATEFUL, dataContent)); } }; @@ -294,10 +294,10 @@ TEST_F(QueueTest, MultithreadDifferentType) consumerThread1.join(); consumerThread2.join(); - EXPECT_NE(0, queue.getItemsByType(MessageType::STATE_LESS)); - EXPECT_NE(0, queue.getItemsByType(MessageType::STATE_FUL)); - EXPECT_FALSE(queue.isEmptyByType(MessageType::STATE_LESS)); - EXPECT_FALSE(queue.isEmptyByType(MessageType::STATE_FUL)); + EXPECT_NE(0, queue.getItemsByType(MessageType::STATELESS)); + EXPECT_NE(0, queue.getItemsByType(MessageType::STATEFUL)); + EXPECT_FALSE(queue.isEmptyByType(MessageType::STATELESS)); + EXPECT_FALSE(queue.isEmptyByType(MessageType::STATEFUL)); // Consume the rest of the messages std::thread consumerThread12(consumerStateLess, std::ref(itemsToConsume)); @@ -307,10 +307,10 @@ TEST_F(QueueTest, MultithreadDifferentType) consumerThread22.join(); // FIXME: this doesn't match - EXPECT_EQ(0, queue.getItemsByType(MessageType::STATE_LESS)); - EXPECT_EQ(0, queue.getItemsByType(MessageType::STATE_FUL)); - EXPECT_TRUE(queue.isEmptyByType(MessageType::STATE_LESS)); - EXPECT_TRUE(queue.isEmptyByType(MessageType::STATE_FUL)); + EXPECT_EQ(0, queue.getItemsByType(MessageType::STATELESS)); + EXPECT_EQ(0, queue.getItemsByType(MessageType::STATEFUL)); + EXPECT_TRUE(queue.isEmptyByType(MessageType::STATELESS)); + EXPECT_TRUE(queue.isEmptyByType(MessageType::STATEFUL)); } // Accesing same queue @@ -366,7 +366,7 @@ TEST_F(QueueTest, MultiplePushSeveralSingleGets) { GTEST_SKIP(); MultiTypeQueue queue(BIG_QUEUE_QTTY); - const MessageType messageType {MessageType::STATE_LESS}; + const MessageType messageType {MessageType::STATELESS}; // TODO: double check array of objects const json multipleDataContent = {"content 1", "content 2", "content 3"}; const Message messageToSend {messageType, multipleDataContent}; @@ -375,26 +375,26 @@ TEST_F(QueueTest, MultiplePushSeveralSingleGets) for (int i : {0, 1, 2}) { - auto messageResponse = queue.getLastMessage(MessageType::STATE_LESS); + auto messageResponse = queue.getLastMessage(MessageType::STATELESS); auto responseData = messageResponse.data[0].template get(); auto sentData = messageToSend.data[i].template get(); EXPECT_EQ(responseData, sentData); } - EXPECT_EQ(queue.getItemsByType(MessageType::STATE_LESS), 0); + EXPECT_EQ(queue.getItemsByType(MessageType::STATELESS), 0); } // Push Multiple, pop multiples TEST_F(QueueTest, MultiplePushSeveralMultiplePops) { MultiTypeQueue queue(BIG_QUEUE_QTTY); - const MessageType messageType {MessageType::STATE_LESS}; + const MessageType messageType {MessageType::STATELESS}; const json multipleDataContent = {"content 1", "content 2", "content 3"}; const Message messageToSend {messageType, multipleDataContent}; queue.push(messageToSend); - EXPECT_TRUE(queue.popNMessages(MessageType::STATE_LESS, 3)); - EXPECT_TRUE(queue.isEmptyByType(MessageType::STATE_LESS)); - EXPECT_EQ(0,queue.getItemsByType(MessageType::STATE_LESS)); + EXPECT_TRUE(queue.popNMessages(MessageType::STATELESS, 3)); + EXPECT_TRUE(queue.isEmptyByType(MessageType::STATELESS)); + EXPECT_EQ(0,queue.getItemsByType(MessageType::STATELESS)); } From 778793cbf1359a10e6cfce4834f4018b44d33609 Mon Sep 17 00:00:00 2001 From: cborla Date: Wed, 17 Jul 2024 19:58:36 -0300 Subject: [PATCH 11/31] feat: change sqlite library to SQLiteCpp, improvement in exceptions --- src/agent/queue/CMakeLists.txt | 7 +- src/agent/queue/include/sqlitestorage.h | 11 +- src/agent/queue/src/sqlitestorage.cpp | 193 +++++++++++------------- src/agent/queue/tests/CMakeLists.txt | 5 +- 4 files changed, 104 insertions(+), 112 deletions(-) diff --git a/src/agent/queue/CMakeLists.txt b/src/agent/queue/CMakeLists.txt index b4c300cab4f..a89de13af65 100644 --- a/src/agent/queue/CMakeLists.txt +++ b/src/agent/queue/CMakeLists.txt @@ -5,8 +5,8 @@ project(QueueTest LANGUAGES CXX) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) -# Find the nlohmann::json package using vcpkg -find_package(SQLite3 REQUIRED) +# Find the nlohmann::json and SQLiteCpp package using vcpkg +find_package(SQLiteCpp REQUIRED) find_package(nlohmann_json REQUIRED) set(CMAKE_CXX_FLAGS "-Wall -Wextra -Wunused -pthread") @@ -23,4 +23,5 @@ add_executable(QueueTest ) # Link the nlohmann::json library to your project -target_link_libraries(QueueTest PRIVATE ${SQLite3_LIBRARIES} nlohmann_json::nlohmann_json) +target_include_directories(QueueTest PRIVATE include ${SQLiteCpp_INCLUDE_DIRS}) +target_link_libraries(QueueTest PRIVATE SQLiteCpp nlohmann_json::nlohmann_json) diff --git a/src/agent/queue/include/sqlitestorage.h b/src/agent/queue/include/sqlitestorage.h index 16c253acd5a..c71a00d17c8 100644 --- a/src/agent/queue/include/sqlitestorage.h +++ b/src/agent/queue/include/sqlitestorage.h @@ -4,7 +4,9 @@ #include #include -#include +#include +#include +#include #include "persistence.h" @@ -96,10 +98,6 @@ class SQLiteStorage : public Persistence */ void InitializeTable(); - void OpenDB(); - - void CloseDB(); - /// The name of the SQLite database file. std::string m_dbName; @@ -107,7 +105,8 @@ class SQLiteStorage : public Persistence std::string m_tableName; /// Pointer to the SQLite database connection. - sqlite3* m_db; + //SQLite::Database* m_db; + std::unique_ptr m_db; /// Mutex to ensure thread-safe operations. std::mutex m_mtx; diff --git a/src/agent/queue/src/sqlitestorage.cpp b/src/agent/queue/src/sqlitestorage.cpp index d6ebc4d08d5..87e9ebf009c 100644 --- a/src/agent/queue/src/sqlitestorage.cpp +++ b/src/agent/queue/src/sqlitestorage.cpp @@ -12,142 +12,131 @@ #define COUNT_QUERY "SELECT COUNT(*) FROM " + m_tableName + ";" SQLiteStorage::SQLiteStorage(const std::string& dbName, const std::string& tableName) - : m_dbName(dbName), m_tableName(tableName), m_db(nullptr) { - OpenDB(); - InitializeTable(); - CloseDB(); + : m_dbName(dbName), m_tableName(tableName), m_db(nullptr) { + try { + // open db + m_db = std::make_unique(m_dbName, SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE); + InitializeTable(); + } catch (const std::exception& e) { + std::cerr << "Error initializing database: " << e.what() << std::endl; + throw; + } } SQLiteStorage::~SQLiteStorage() { - // Destructor vacío ya que la base de datos se cerrará después de cada operación + // No need to explicitly close the database as std::unique_ptr will handle it } void SQLiteStorage::InitializeTable() { - std::string createTableQuery = CREATE_TABLE_QUERY; - sqlite3_exec(m_db, createTableQuery.c_str(), 0, 0, 0); -} - -void SQLiteStorage::OpenDB() { - sqlite3_open(m_dbName.c_str(), &m_db); -} - -void SQLiteStorage::CloseDB() { - if (m_db) { - sqlite3_close(m_db); - m_db = nullptr; + std::lock_guard lock(m_mtx); + try { + std::string createTableQuery = CREATE_TABLE_QUERY; + m_db->exec(createTableQuery); + } catch (const std::exception& e) { + std::cerr << "Error initializing table: " << e.what() << std::endl; + throw; } } void SQLiteStorage::Store(const json& message) { std::lock_guard lock(m_mtx); - std::string insertQuery = INSERT_QUERY; - OpenDB(); - sqlite3_stmt* stmt; - sqlite3_prepare_v2(m_db, insertQuery.c_str(), -1, &stmt, 0); - if(message.is_array()) - { - for (auto& singleMessageData : message.items()) - { - std::string messageStr = singleMessageData.value(); - sqlite3_bind_text(stmt, 1, messageStr.c_str(), -1, SQLITE_TRANSIENT); - sqlite3_step(stmt); + try { + std::string insertQuery = INSERT_QUERY; + SQLite::Statement query(*m_db, insertQuery); + + if (message.is_array()) { + for (const auto& singleMessageData : message) { + query.bind(1, singleMessageData.dump()); + query.exec(); + query.reset(); // Reset the query to reuse it for the next message + } + } else { + query.bind(1, message.dump()); + query.exec(); } + } catch (const std::exception& e) { + std::cerr << "Error storing message: " << e.what() << std::endl; + throw; } - else - { - std::string messageStr = message.dump(); - sqlite3_bind_text(stmt, 1, messageStr.c_str(), -1, SQLITE_TRANSIENT); - sqlite3_step(stmt); - } - sqlite3_finalize(stmt); - CloseDB(); } json SQLiteStorage::Retrieve(int id) { std::lock_guard lock(m_mtx); - OpenDB(); - - std::string selectQuery = SELECT_QUERY; - sqlite3_stmt* stmt; - sqlite3_prepare_v2(m_db, selectQuery.c_str(), -1, &stmt, 0); - sqlite3_bind_int(stmt, 1, id); - json message; - if (sqlite3_step(stmt) == SQLITE_ROW) { - std::string messageStr = reinterpret_cast(sqlite3_column_text(stmt, 0)); - message = json::parse(messageStr); - } else { - message = nullptr; + try { + std::string selectQuery = SELECT_QUERY; + SQLite::Statement query(*m_db, selectQuery); + query.bind(1, id); + json message; + if (query.executeStep()) { + message = json::parse(query.getColumn(0).getString()); + } else { + message = nullptr; + } + return message; + } catch (const std::exception& e) { + std::cerr << "Error retrieving message: " << e.what() << std::endl; + throw; } - sqlite3_finalize(stmt); - - CloseDB(); - return message; } json SQLiteStorage::RetrieveMultiple(int n) { std::lock_guard lock(m_mtx); - OpenDB(); - - std::string selectQuery = SELECT_MULTIPLE_QUERY; - sqlite3_stmt* stmt; - sqlite3_prepare_v2(m_db, selectQuery.c_str(), -1, &stmt, 0); - sqlite3_bind_int(stmt, 1, n); - json messages = json::array(); - while (sqlite3_step(stmt) == SQLITE_ROW) { - std::string messageStr = reinterpret_cast(sqlite3_column_text(stmt, 0)); - messages.push_back(json::parse(messageStr)); + try { + std::string selectQuery = SELECT_MULTIPLE_QUERY; + SQLite::Statement query(*m_db, selectQuery); + query.bind(1, n); + json messages = json::array(); + while (query.executeStep()) { + messages.push_back(json::parse(query.getColumn(0).getString())); + } + std::reverse(messages.begin(), messages.end()); + return messages; + } catch (const std::exception& e) { + std::cerr << "Error retrieving multiple messages: " << e.what() << std::endl; + throw; } - sqlite3_finalize(stmt); - - CloseDB(); - return messages; } int SQLiteStorage::Remove(int id) { std::lock_guard lock(m_mtx); - OpenDB(); - - std::string deleteQuery = DELETE_QUERY; - sqlite3_stmt* stmt; - sqlite3_prepare_v2(m_db, deleteQuery.c_str(), -1, &stmt, 0); - sqlite3_bind_int(stmt, 1, id); - sqlite3_step(stmt); - sqlite3_finalize(stmt); - - CloseDB(); - - return 1; + try { + std::string deleteQuery = DELETE_QUERY; + SQLite::Statement query(*m_db, deleteQuery); + query.bind(1, id); + query.exec(); + return 1; + } catch (const std::exception& e) { + std::cerr << "Error removing message: " << e.what() << std::endl; + throw; + } } int SQLiteStorage::RemoveMultiple(int n) { std::lock_guard lock(m_mtx); - OpenDB(); - - std::string deleteQuery = DELETE_MULTIPLE_QUERY; - sqlite3_stmt* stmt; - sqlite3_prepare_v2(m_db, deleteQuery.c_str(), -1, &stmt, 0); - sqlite3_bind_int(stmt, 1, n); - sqlite3_step(stmt); - sqlite3_finalize(stmt); - - CloseDB(); - - return n; + try { + std::string deleteQuery = DELETE_MULTIPLE_QUERY; + SQLite::Statement query(*m_db, deleteQuery); + query.bind(1, n); + query.exec(); + return n; + } catch (const std::exception& e) { + std::cerr << "Error removing multiple messages: " << e.what() << std::endl; + throw; + } } int SQLiteStorage::GetElementCount() { std::lock_guard lock(m_mtx); - OpenDB(); - - std::string countQuery = COUNT_QUERY; - sqlite3_stmt* stmt; - sqlite3_prepare_v2(m_db, countQuery.c_str(), -1, &stmt, 0); - int count = 0; - if (sqlite3_step(stmt) == SQLITE_ROW) { - count = sqlite3_column_int(stmt, 0); + try { + std::string countQuery = COUNT_QUERY; + SQLite::Statement query(*m_db, countQuery); + int count = 0; + if (query.executeStep()) { + count = query.getColumn(0).getInt(); + } + return count; + } catch (const std::exception& e) { + std::cerr << "Error getting element count: " << e.what() << std::endl; + throw; } - sqlite3_finalize(stmt); - - CloseDB(); - return count; } diff --git a/src/agent/queue/tests/CMakeLists.txt b/src/agent/queue/tests/CMakeLists.txt index cb409468ab3..239473d8738 100644 --- a/src/agent/queue/tests/CMakeLists.txt +++ b/src/agent/queue/tests/CMakeLists.txt @@ -1,3 +1,4 @@ +# Enable testing # enable_testing() # Link against Google Test @@ -6,8 +7,10 @@ find_package(GTest REQUIRED) include_directories(../include) include_directories(${GTEST_INCLUDE_DIRS}) +include_directories(${SQLiteCpp_INCLUDE_DIRS}) file(GLOB QUEUE_UNIT_TEST_SRC "*.cpp") + # Create a test executable add_executable(test_queue ${QUEUE_UNIT_TEST_SRC} ../src/sqlitestorage.cpp @@ -18,6 +21,6 @@ add_executable(test_queue ${QUEUE_UNIT_TEST_SRC} target_link_libraries(test_queue ${GTEST_LIBRARIES} ${GTEST_MAIN_LIBRARIES} - ${SQLite3_LIBRARIES} + SQLiteCpp nlohmann_json::nlohmann_json) # target_link_libraries(main PRIVATE GTest::gtest GTest::gtest_main GTest::gmock GTest::gmock_main) From 2a36513d21ae3c6278cd7a223a5cfea7f5c531bd Mon Sep 17 00:00:00 2001 From: cborla Date: Wed, 17 Jul 2024 20:00:23 -0300 Subject: [PATCH 12/31] fix: clean code from warning messages --- src/agent/queue/include/filestorage.hpp | 30 ++++++++++++------------- src/agent/queue/src/main.cpp | 2 -- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/src/agent/queue/include/filestorage.hpp b/src/agent/queue/include/filestorage.hpp index bac6d19ecea..6acee84dbbd 100644 --- a/src/agent/queue/include/filestorage.hpp +++ b/src/agent/queue/include/filestorage.hpp @@ -98,12 +98,12 @@ class FileStorage : public Persistence } /** - * @brief - * - * @param LineID - * @return json + * @brief + * + * @param LineID + * @return json */ - json Retrieve(int lineID) override + json Retrieve(int /*lineID*/) override { return RetrieveMultiple(1); } @@ -127,8 +127,8 @@ class FileStorage : public Persistence ifs.close(); // Check if n is greater than the number of lines in the file - if (linesQuantity > all_lines.size()) { - linesQuantity = all_lines.size(); + if (linesQuantity > (int) all_lines.size()) { + linesQuantity = (int) all_lines.size(); } // Create a vector for the lines to return and remove @@ -146,20 +146,20 @@ class FileStorage : public Persistence } /** - * @brief - * - * @param LinesID - * @return int + * @brief + * + * @param LinesID + * @return int */ - int Remove(int LinesID) override + int Remove(int /*LinesID*/) override { return RemoveMultiple(1); } /** * @brief Get the Element Count object - * - * @return int + * + * @return int */ int GetElementCount() override { @@ -175,4 +175,4 @@ class FileStorage : public Persistence return all_lines.size(); } -}; +}; \ No newline at end of file diff --git a/src/agent/queue/src/main.cpp b/src/agent/queue/src/main.cpp index 72e793ba9e1..0d832165d21 100644 --- a/src/agent/queue/src/main.cpp +++ b/src/agent/queue/src/main.cpp @@ -10,7 +10,6 @@ int main() auto consumer1 = [&](int &count) { // std::this_thread::sleep_for(std::chrono::milliseconds(1000)); - int item; for (int i = 0; i < count; ++i) { queue.popLastMessage(MessageType::STATELESS); @@ -20,7 +19,6 @@ int main() }; auto consumer2 = [&](int &count) { - int item; for (int i = 0; i < count; ++i) { queue.popLastMessage(MessageType::STATEFUL); From b7d28cf59f8b3cd585d68182cdcea51107926ffe Mon Sep 17 00:00:00 2001 From: LucioDonda Date: Fri, 19 Jul 2024 18:19:23 -0300 Subject: [PATCH 13/31] feat: timeout on push and removing filestorage --- src/agent/queue/include/filestorage.hpp | 178 --------------- src/agent/queue/include/queue.hpp | 276 ++++++++++++++++-------- src/agent/queue/src/main.cpp | 42 ---- src/agent/queue/src/sqlitestorage.cpp | 13 +- src/agent/queue/tests/queue_test.cpp | 103 ++++----- 5 files changed, 238 insertions(+), 374 deletions(-) delete mode 100644 src/agent/queue/include/filestorage.hpp diff --git a/src/agent/queue/include/filestorage.hpp b/src/agent/queue/include/filestorage.hpp deleted file mode 100644 index 6acee84dbbd..00000000000 --- a/src/agent/queue/include/filestorage.hpp +++ /dev/null @@ -1,178 +0,0 @@ -#include -#include - -#include "persistence.h" - -using json = nlohmann::json; - - -/** - * @brief Implementation of persistence for storing in plain text file - * - */ -class FileStorage : public Persistence -{ -private: - std::string filename; - -public: - /** - * @brief Construct a new File Storage object - * - * @param fname full path of the file to be used as persistance - */ - FileStorage(const std::string& fname) - : filename(fname) - { - //TODO: - // createOrCheckFile(); - } - - // Delete copy constructor - FileStorage(const FileStorage&) = delete; - - // Delete copy assignment operator - FileStorage& operator=(const FileStorage&) = delete; - - // Delete move constructor - FileStorage(FileStorage&&) = delete; - - // Delete move assignment operator - FileStorage& operator=(FileStorage&&) = delete; - - ~FileStorage() {}; - - bool createOrCheckFile() - { - std::ifstream file(filename); - if (file.good()) { - file.close(); - // File exists - return true; - } else { - std::ofstream newFile(filename); - if (newFile.is_open()) { - newFile.close(); - return true; - } - // Failed to create file - return false; - } - } - - void Store(const json& data) override - { - std::ofstream file(filename, std::ios::app); - if (file.is_open()) { - file << to_string(data) << std::endl; - file.close(); - } - } - - /** - * @brief Reads and return N lines from the queue - * - * @param linesQuantity - * @return std::string - */ - json RetrieveMultiple(int linesQuantity) override - { - std::ifstream file(filename); - std::string line; - int currentLine = 0; - json outputArray = {}; - - while (std::getline(file, line)) { - currentLine++; - outputArray.emplace_back(line); - std::cout << "output: " << outputArray << std::endl; - if (currentLine == linesQuantity) { - file.close(); - return outputArray; - } - } - - // less messages than asked for - file.close(); - return outputArray; - } - - /** - * @brief - * - * @param LineID - * @return json - */ - json Retrieve(int /*lineID*/) override - { - return RetrieveMultiple(1); - } - - /** - * @brief Pops N lines from the queue - * - * @param linesQuantity - * @return int number of lines actually popped - */ - int RemoveMultiple(int linesQuantity) override - { - std::vector all_lines; - std::ifstream ifs(filename); - std::string line; - - // Read all lines from the file - while (std::getline(ifs, line)) { - all_lines.push_back(line); - } - ifs.close(); - - // Check if n is greater than the number of lines in the file - if (linesQuantity > (int) all_lines.size()) { - linesQuantity = (int) all_lines.size(); - } - - // Create a vector for the lines to return and remove - std::vector last_n_lines(all_lines.begin(), all_lines.begin() + linesQuantity); - all_lines.erase(all_lines.begin(), all_lines.begin() + linesQuantity); - - // Write the remaining lines back to the file - std::ofstream ofs(filename, std::ios::out | std::ios::trunc); - for (const auto& remaining_line : all_lines) { - ofs << remaining_line << std::endl; - } - ofs.close(); - - return linesQuantity; - } - - /** - * @brief - * - * @param LinesID - * @return int - */ - int Remove(int /*LinesID*/) override - { - return RemoveMultiple(1); - } - - /** - * @brief Get the Element Count object - * - * @return int - */ - int GetElementCount() override - { - std::vector all_lines; - std::ifstream ifs(filename); - std::string line; - - // Read all lines from the file - while (std::getline(ifs, line)) { - all_lines.push_back(line); - } - ifs.close(); - - return all_lines.size(); - } -}; \ No newline at end of file diff --git a/src/agent/queue/include/queue.hpp b/src/agent/queue/include/queue.hpp index d616e2440f6..c05f86fca8b 100644 --- a/src/agent/queue/include/queue.hpp +++ b/src/agent/queue/include/queue.hpp @@ -1,19 +1,21 @@ #include +#include #include #include #include #include #include +#include #include #include #include -#include "filestorage.hpp" #include "shared.hpp" #include "sqlitestorage.h" // TODO: move to a configuration setting constexpr int DEFAULT_MAX = 10; +constexpr int DEFAULT_TIMEOUT_S = 3; // Factory class class PersistenceFactory @@ -30,14 +32,6 @@ class PersistenceFactory return std::make_unique(std::any_cast(args[0]), std::any_cast(args[1])); } - else if (type == "File") - { - if (args.size() != 1 || !std::any_cast(&args[0])) - { - throw std::invalid_argument("File requires a string argument"); - } - return std::make_unique(std::any_cast(args[0])); - } throw std::runtime_error("Unknown persistence type"); } }; @@ -49,17 +43,13 @@ class PersistenceFactory class PersistedQueue { public: - PersistedQueue(MessageType m_queueType, int max_size = DEFAULT_MAX) - : m_queueType(m_queueType) + PersistedQueue(MessageType queueType, int max_size, int timeout) + : m_queueType(queueType) , m_max_size(max_size) + , m_timeout(timeout) { - // Another option: - // m_persistenceDest = PersistenceFactory::createPersistence("File", {DEFAULT_FILE_PATH + - // MessageTypeName.at(m_queueType)}); m_persistenceDest = PersistenceFactory::createPersistence( "SQLite3", {static_cast(DEFAULT_DB_PATH), MessageTypeName.at(m_queueType)}); - // updates the actual size being used - getItemsAvailable(); } // Delete copy constructor @@ -74,7 +64,6 @@ class PersistedQueue // Delete move assignment operator PersistedQueue& operator=(PersistedQueue&&) = delete; - // TODO ~PersistedQueue() { // m_persistenceDest.close(); }; @@ -89,23 +78,15 @@ class PersistedQueue return m_queueType; } - int getItemsAvailable() - { - std::unique_lock lock(m_mtx); - //TODO: rework this behavior - m_size = m_persistenceDest->GetElementCount(); - return m_size; - } - /** - * @brief Get the Size object + * @brief Get the Items Available object * * @return int */ - // TODO check this functionality because when inserting data arrays it loses actual value - int getSize() const + int getItemsAvailable() { - return m_size; + std::unique_lock lock(m_mtx); + return m_persistenceDest->GetElementCount(); } /** @@ -118,34 +99,51 @@ class PersistedQueue bool insertMessage(Message event) { std::unique_lock lock(m_mtx); - m_persistenceDest->Store(event.data); - m_size++; - m_cv.notify_one(); - // TODO: make Store method int for returning items inserted when array - // if(m_persistenceDest->Store(event.data)) - // { - // m_size++; - // m_cv.notify_one(); - // } - return true; - }; - - bool removeNMessages(int qttyMessages) - { - bool result = false; - // workaround for items issue - getItemsAvailable(); - if (m_size) + bool success = false; + size_t spaceAvailable = + (m_max_size > m_persistenceDest->GetElementCount()) ? m_max_size - m_persistenceDest->GetElementCount() : 0; + if (spaceAvailable) { - std::unique_lock lock(m_mtx); - auto linesRemoved = m_persistenceDest->RemoveMultiple(qttyMessages); - if(linesRemoved) + // TODO: handle response + success = true; + auto messageData = event.data; + if (messageData.is_array()) { - result = true; - m_size = m_size - linesRemoved; + if (messageData.size() <= spaceAvailable) + { + for (const auto& singleMessageData : messageData) + { + m_persistenceDest->Store(singleMessageData); + m_cv.notify_all(); + } + } + else + { + success = false; + } + } + else + { + m_persistenceDest->Store(event.data); + m_cv.notify_all(); } } - return result; + return success; + }; + + /** + * @brief + * + * @param qttyMessages + * @return true + * @return false + */ + bool removeNMessages(int qttyMessages) + { + std::unique_lock lock(m_mtx); + auto linesRemoved = m_persistenceDest->RemoveMultiple(qttyMessages); + m_cv.notify_all(); + return linesRemoved != 0; }; /** @@ -179,15 +177,61 @@ class PersistedQueue * @return true * @return false */ - bool empty() const + bool empty() { - return m_size == 0; + const auto items = getItemsAvailable(); + return items == 0; + } + + /** + * @brief + * + * @return true + * @return false + */ + bool isFull() + { + std::unique_lock lock(m_mtx); + return m_persistenceDest->GetElementCount() == m_max_size; + } + + void waitUntilNotFull() + { + std::unique_lock lock(m_mtx); + m_cv.wait_for(lock, + m_timeout, + [this] + { + std::cout << " waiting " << std::endl; + return m_persistenceDest->GetElementCount() < m_max_size; + }); + } + + /** + * A function that waits until the queue is not full or until a stop signal is received. + * + * @param stopToken std::stop_token to check for a stop signal + * + * @throws None + */ + void waitUntilNotFullOrStoped(std::stop_token stopToken) + { + std::unique_lock lock(m_mtx); + m_cv.wait(lock, + [this, stopToken] + { + std::cout << " waiting " << std::endl; + bool menor = (m_persistenceDest->GetElementCount() < m_max_size); + bool stopped = (stopToken.stop_possible() && stopToken.stop_requested()); + std::cout << "menor" << menor << "stopped" << stopped << std::endl; + return menor || stopped ; + }); } private: const MessageType m_queueType; - std::atomic m_size = 0; const int m_max_size; + const std::chrono::seconds m_timeout; std::unique_ptr m_persistenceDest; std::mutex m_mtx; std::condition_variable m_cv; @@ -202,16 +246,19 @@ class MultiTypeQueue private: std::unordered_map> m_queuesMap; const int m_maxItems; + std::mutex m_mapMutex; public: // Create a vector with 3 PersistedQueue elements - MultiTypeQueue(int size = DEFAULT_MAX) + MultiTypeQueue(int size = DEFAULT_MAX, int timeout = DEFAULT_TIMEOUT_S) : m_maxItems(size) { // Populate the map inside the constructor body - m_queuesMap[MessageType::STATELESS] = std::make_unique(MessageType::STATELESS, m_maxItems); - m_queuesMap[MessageType::STATEFUL] = std::make_unique(MessageType::STATEFUL, m_maxItems); - m_queuesMap[MessageType::COMMAND] = std::make_unique(MessageType::COMMAND, m_maxItems); + m_queuesMap[MessageType::STATELESS] = + std::make_unique(MessageType::STATELESS, m_maxItems, timeout); + m_queuesMap[MessageType::STATEFUL] = + std::make_unique(MessageType::STATEFUL, m_maxItems, timeout); + m_queuesMap[MessageType::COMMAND] = std::make_unique(MessageType::COMMAND, m_maxItems, timeout); } // Delete copy constructor @@ -226,44 +273,88 @@ class MultiTypeQueue // Delete move assignment operator MultiTypeQueue& operator=(MultiTypeQueue&&) = delete; - // TODO ~MultiTypeQueue() {}; + + bool stopablePush(Message message, std::stop_token stopToken) + { + bool result = false; + std::unique_lock mapLock(m_mapMutex); + + if (m_queuesMap.contains(message.type)) + { + auto& queue = m_queuesMap[message.type]; + mapLock.unlock(); + + // Wait until the queue is not full + if (stopToken.stop_possible()) + { + queue->waitUntilNotFullOrStoped(stopToken); + } + + // Insert the message + result = queue->insertMessage(message); + if (!result) + { + std::cout << "Failed to insert message: " << message.data << std::endl; + } + } + else + { + std::cout << "error didn't find the queue" << std::endl; + } + return result; + } + /** - * @brief: push message to a queue of t + * @brief: timeoutPush message to a queue of t * * @param message */ - bool push(Message message) + bool timeoutPush(Message message, bool shouldWait = false) { bool result = false; + std::unique_lock mapLock(m_mapMutex); + if (m_queuesMap.contains(message.type)) { - while (m_queuesMap[message.type]->getSize() == m_maxItems) + auto& queue = m_queuesMap[message.type]; + mapLock.unlock(); + + // Wait until the queue is not full + if (shouldWait) { - // TODO: delete this - std::cout << "waiting" << std::endl; - std::this_thread::sleep_for(std::chrono::milliseconds(250)); + queue->waitUntilNotFull(); + } + //FIXME + // else + // { + // std::cout << "Can failed because os full queue" << std::endl; + // } + + // Insert the message + result = queue->insertMessage(message); + if (!result) + { + std::cout << "Failed to insert message: " << message.data << std::endl; } - result = m_queuesMap[message.type]->insertMessage(message); } else { - // TODO: error / logging handling !!! std::cout << "error didn't find the queue" << std::endl; } return result; } /** - * @brief - * - * @param messages + * @brief + * + * @param messages */ - void push(std::vector messages) + void timeoutPush(std::vector messages) { for (const auto& singleMessage : messages) { - push(singleMessage); + timeoutPush(singleMessage); } } @@ -276,9 +367,12 @@ class MultiTypeQueue Message getLastMessage(MessageType type) { Message result(type, {}); + std::unique_lock mapLock(m_mapMutex); if (m_queuesMap.contains(type)) { - result = m_queuesMap[type]->getMessage(); + auto& queue = m_queuesMap[type]; + mapLock.unlock(); + result = queue->getMessage(); } else { @@ -298,10 +392,13 @@ class MultiTypeQueue bool popLastMessage(MessageType type) { bool result = false; + std::unique_lock mapLock(m_mapMutex); if (m_queuesMap.contains(type)) { + auto& queue = m_queuesMap[type]; + mapLock.unlock(); // Handle return value - result = m_queuesMap[type]->removeNMessages(1); + result = queue->removeNMessages(1); } else { @@ -312,19 +409,22 @@ class MultiTypeQueue } /** - * @brief - * - * @param type - * @param messageQuantity - * @return true - * @return false + * @brief + * + * @param type + * @param messageQuantity + * @return true + * @return false */ bool popNMessages(MessageType type, int messageQuantity) { bool result = false; + std::unique_lock mapLock(m_mapMutex); if (m_queuesMap.contains(type)) { - result = m_queuesMap[type]->removeNMessages(messageQuantity); + auto& queue = m_queuesMap[type]; + mapLock.unlock(); + result = queue->removeNMessages(messageQuantity); } else { @@ -343,9 +443,12 @@ class MultiTypeQueue */ bool isEmptyByType(MessageType type) { + std::unique_lock mapLock(m_mapMutex); if (m_queuesMap.contains(type)) { - return m_queuesMap[type]->empty(); + auto& queue = m_queuesMap[type]; + mapLock.unlock(); + return queue->empty(); } else { @@ -364,9 +467,12 @@ class MultiTypeQueue */ int getItemsByType(MessageType type) { + std::unique_lock mapLock(m_mapMutex); if (m_queuesMap.contains(type)) { - return m_queuesMap[type]->getItemsAvailable(); + auto& queue = m_queuesMap[type]; + mapLock.unlock(); + return queue->getItemsAvailable(); } else { diff --git a/src/agent/queue/src/main.cpp b/src/agent/queue/src/main.cpp index 0d832165d21..ae8983c9baa 100644 --- a/src/agent/queue/src/main.cpp +++ b/src/agent/queue/src/main.cpp @@ -8,47 +8,5 @@ int main() { MultiTypeQueue queue(10); - auto consumer1 = [&](int &count) { - // std::this_thread::sleep_for(std::chrono::milliseconds(1000)); - for (int i = 0; i < count; ++i) - { - queue.popLastMessage(MessageType::STATELESS); - std::cout << "Popping event 1: " << std::endl; - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } - }; - - auto consumer2 = [&](int &count) { - for (int i = 0; i < count; ++i) - { - queue.popLastMessage(MessageType::STATEFUL); - std::cout << "Popping event 2: " << std::endl; - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } - }; - - - auto producer = [&](int &count) { - for (int i = 0; i < count; ++i) - { - auto message = R"({"Data" : "for STATELESS)" + std::to_string(i) + R"("})"; - queue.push(Message(MessageType::STATELESS, message)); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); // Simulate work - } - }; - - int items_to_insert = 5; - int items_to_consume = 2; - - producer(items_to_insert); - - std::this_thread::sleep_for(std::chrono::milliseconds(1000)); - - std::thread consumer_thread1(consumer1, std::ref(items_to_consume)); - std::thread producer_thread1(consumer2, std::ref(items_to_consume)); - - producer_thread1.join(); - consumer_thread1.join(); - return 0; } diff --git a/src/agent/queue/src/sqlitestorage.cpp b/src/agent/queue/src/sqlitestorage.cpp index 87e9ebf009c..9cae7c69931 100644 --- a/src/agent/queue/src/sqlitestorage.cpp +++ b/src/agent/queue/src/sqlitestorage.cpp @@ -43,17 +43,8 @@ void SQLiteStorage::Store(const json& message) { try { std::string insertQuery = INSERT_QUERY; SQLite::Statement query(*m_db, insertQuery); - - if (message.is_array()) { - for (const auto& singleMessageData : message) { - query.bind(1, singleMessageData.dump()); - query.exec(); - query.reset(); // Reset the query to reuse it for the next message - } - } else { - query.bind(1, message.dump()); - query.exec(); - } + query.bind(1, message.dump()); + query.exec(); } catch (const std::exception& e) { std::cerr << "Error storing message: " << e.what() << std::endl; throw; diff --git a/src/agent/queue/tests/queue_test.cpp b/src/agent/queue/tests/queue_test.cpp index c66c6fe298a..06b2d6d7ec9 100644 --- a/src/agent/queue/tests/queue_test.cpp +++ b/src/agent/queue/tests/queue_test.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include "queue.hpp" @@ -62,35 +63,6 @@ void cleanPersistence() std::filesystem::remove(filePath, ec); } -// In order to mimic the timeout from outside the queue -// TODO: double check if this is expected to work this way -template -void functionWithTimeout(Func&& func, int timeout_ms, Args&&... args) -{ - // Launch the function in a separate thread - std::promise exitSignal; - std::future futureObj = exitSignal.get_future(); - - std::thread t( - [&func, &exitSignal, &args...]() - { - func(std::forward(args)...); - exitSignal.set_value(); - }); - - // Wait for the function to finish or timeout - if (futureObj.wait_for(std::chrono::milliseconds(timeout_ms)) == std::future_status::timeout) - { - // Detach the thread if it times out - t.detach(); - throw std::runtime_error("Function call timed out"); - } - else - { - t.join(); - } -} - /// Test Methods void QueueTest::SetUp() @@ -132,6 +104,7 @@ TEST_F(JsonTest, JSONArrays) // call is_array() EXPECT_FALSE(j_object.is_array()); EXPECT_TRUE(j_array.is_array()); + EXPECT_EQ(5, j_array.size()); EXPECT_TRUE(multipleDataContent.is_array()); int i = 0; @@ -148,7 +121,7 @@ TEST_F(QueueTest, SinglePushGetNotEmpty) const MessageType messageType {MessageType::STATELESS}; const Message messageToSend {messageType, baseDataContent}; - queue.push(messageToSend); + queue.timeoutPush(messageToSend); auto messageResponse = queue.getLastMessage(MessageType::STATELESS); auto typeSend = messageToSend.type; @@ -162,14 +135,14 @@ TEST_F(QueueTest, SinglePushGetNotEmpty) EXPECT_FALSE(queue.isEmptyByType(MessageType::STATELESS)); } -// push and pop on a non-full queue +// timeoutPush and pop on a non-full queue TEST_F(QueueTest, SinglePushPopEmpty) { MultiTypeQueue queue(BIG_QUEUE_QTTY); const MessageType messageType {MessageType::STATELESS}; const Message messageToSend {messageType, baseDataContent}; - queue.push(messageToSend); + queue.timeoutPush(messageToSend); auto messageResponse = queue.getLastMessage(MessageType::STATELESS); auto typeSend = messageToSend.type; @@ -191,7 +164,7 @@ TEST_F(QueueTest, SinglePushPop) const MessageType messageType {MessageType::STATELESS}; const Message messageToSend {messageType, baseDataContent}; - queue.push(messageToSend); + queue.timeoutPush(messageToSend); EXPECT_FALSE(queue.isEmptyByType(MessageType::STATELESS)); queue.popLastMessage(MessageType::STATELESS); @@ -205,7 +178,7 @@ TEST_F(QueueTest, SingleGetPopOnEmpty) const MessageType messageType {MessageType::STATELESS}; const Message messageToSend {messageType, baseDataContent}; - queue.push(messageToSend); + queue.timeoutPush(messageToSend); EXPECT_FALSE(queue.isEmptyByType(MessageType::STATELESS)); EXPECT_TRUE(queue.isEmptyByType(MessageType::STATEFUL)); @@ -218,7 +191,7 @@ TEST_F(QueueTest, SingleGetPopOnEmpty) } // Push, get and check while the queue is full -TEST_F(QueueTest, SinglePushPopFull) +TEST_F(QueueTest, SinglePushPopFullWithTimeout) { MultiTypeQueue queue(SMALL_QUEUE_QTTY); @@ -227,20 +200,12 @@ TEST_F(QueueTest, SinglePushPopFull) for (int i : {1, 2}) { const json dataContent = R"({"Data" : "for COMMAND)" + std::to_string(i) + R"("})"; - queue.push({messageType, dataContent}); + queue.timeoutPush({messageType, dataContent}); } const json dataContent = R"({"Data" : "for COMMAND3"})"; Message exampleMessage {messageType, dataContent}; - - try - { - functionWithTimeout([&queue](Message& message) { queue.push(message); }, 1000, exampleMessage); - } - catch (const std::runtime_error& e) - { - std::cerr << e.what() << '\n'; - } + queue.timeoutPush({messageType, dataContent},true); auto items = queue.getItemsByType(MessageType::COMMAND); EXPECT_EQ(items, SMALL_QUEUE_QTTY); @@ -278,8 +243,8 @@ TEST_F(QueueTest, MultithreadDifferentType) for (int i = 0; i < count; ++i) { const json dataContent = R"({{"Data", "for STATELESS)" + std::to_string(i) + R"("}})"; - queue.push(Message(MessageType::STATELESS, dataContent)); - queue.push(Message(MessageType::STATEFUL, dataContent)); + queue.timeoutPush(Message(MessageType::STATELESS, dataContent)); + queue.timeoutPush(Message(MessageType::STATEFUL, dataContent)); } }; @@ -291,8 +256,16 @@ TEST_F(QueueTest, MultithreadDifferentType) std::thread consumerThread1(consumerStateLess, std::ref(itemsToConsume)); std::thread consumerThread2(consumerStateFull, std::ref(itemsToConsume)); - consumerThread1.join(); - consumerThread2.join(); + if(consumerThread1.joinable()) + { + consumerThread1.join(); + } + + if(consumerThread2.joinable()) + { + consumerThread2.join(); + } + EXPECT_NE(0, queue.getItemsByType(MessageType::STATELESS)); EXPECT_NE(0, queue.getItemsByType(MessageType::STATEFUL)); @@ -303,8 +276,16 @@ TEST_F(QueueTest, MultithreadDifferentType) std::thread consumerThread12(consumerStateLess, std::ref(itemsToConsume)); std::thread consumerThread22(consumerStateFull, std::ref(itemsToConsume)); - consumerThread12.join(); - consumerThread22.join(); + if(consumerThread12.joinable()) + { + consumerThread12.join(); + } + + if(consumerThread22.joinable()) + { + consumerThread22.join(); + } + // FIXME: this doesn't match EXPECT_EQ(0, queue.getItemsByType(MessageType::STATELESS)); @@ -316,7 +297,6 @@ TEST_F(QueueTest, MultithreadDifferentType) // Accesing same queue TEST_F(QueueTest, MultithreadSameType) { - // Failing due to some issue deletting MultiTypeQueue queue(BIG_QUEUE_QTTY); auto consumerCommand1 = [&](int& count) @@ -340,7 +320,7 @@ TEST_F(QueueTest, MultithreadSameType) for (int i = 0; i < count; ++i) { const json dataContent = R"({{"Data": "for COMMAND)" + std::to_string(i) + R"("}})"; - queue.push(Message(MessageType::COMMAND, dataContent)); + queue.timeoutPush(Message(MessageType::COMMAND, dataContent)); } }; @@ -354,8 +334,15 @@ TEST_F(QueueTest, MultithreadSameType) std::thread consumerThread1(consumerCommand1, std::ref(itemsToConsume)); std::thread messageProducerThread1(consumerCommand2, std::ref(itemsToConsume)); - messageProducerThread1.join(); - consumerThread1.join(); + if(messageProducerThread1.joinable()) + { + messageProducerThread1.join(); + } + + if(consumerThread1.joinable()) + { + consumerThread1.join(); + } EXPECT_TRUE(queue.isEmptyByType(MessageType::COMMAND)); } @@ -364,14 +351,13 @@ TEST_F(QueueTest, MultithreadSameType) // several gets, checks and pops TEST_F(QueueTest, MultiplePushSeveralSingleGets) { - GTEST_SKIP(); MultiTypeQueue queue(BIG_QUEUE_QTTY); const MessageType messageType {MessageType::STATELESS}; // TODO: double check array of objects const json multipleDataContent = {"content 1", "content 2", "content 3"}; const Message messageToSend {messageType, multipleDataContent}; - queue.push(messageToSend); + queue.timeoutPush(messageToSend); for (int i : {0, 1, 2}) { @@ -379,6 +365,7 @@ TEST_F(QueueTest, MultiplePushSeveralSingleGets) auto responseData = messageResponse.data[0].template get(); auto sentData = messageToSend.data[i].template get(); EXPECT_EQ(responseData, sentData); + queue.popLastMessage(MessageType::STATELESS); } EXPECT_EQ(queue.getItemsByType(MessageType::STATELESS), 0); @@ -392,7 +379,7 @@ TEST_F(QueueTest, MultiplePushSeveralMultiplePops) const json multipleDataContent = {"content 1", "content 2", "content 3"}; const Message messageToSend {messageType, multipleDataContent}; - queue.push(messageToSend); + queue.timeoutPush(messageToSend); EXPECT_TRUE(queue.popNMessages(MessageType::STATELESS, 3)); EXPECT_TRUE(queue.isEmptyByType(MessageType::STATELESS)); From 88cdad2967bb39176e389dabf3c469ed45be1626 Mon Sep 17 00:00:00 2001 From: LucioDonda Date: Fri, 19 Jul 2024 21:15:29 -0300 Subject: [PATCH 14/31] feat: separing queue with header and building as library --- src/agent/queue/CMakeLists.txt | 6 +- src/agent/queue/include/queue.hpp | 292 ++++----------------------- src/agent/queue/include/shared.hpp | 10 - src/agent/queue/src/queue.cpp | 246 ++++++++++++++++++++++ src/agent/queue/tests/CMakeLists.txt | 5 +- 5 files changed, 286 insertions(+), 273 deletions(-) create mode 100644 src/agent/queue/src/queue.cpp diff --git a/src/agent/queue/CMakeLists.txt b/src/agent/queue/CMakeLists.txt index a89de13af65..95c8db600d9 100644 --- a/src/agent/queue/CMakeLists.txt +++ b/src/agent/queue/CMakeLists.txt @@ -13,13 +13,15 @@ set(CMAKE_CXX_FLAGS "-Wall -Wextra -Wunused -pthread") # Add the include directory for shared headers include_directories(${CMAKE_SOURCE_DIR}/include) +# include_directories(${CMAKE_SOURCE_DIR}/src) add_subdirectory(tests) # Add the executable -add_executable(QueueTest - src/main.cpp +add_library(QueueTest + # src/main.cpp src/sqlitestorage.cpp + src/queue.cpp ) # Link the nlohmann::json library to your project diff --git a/src/agent/queue/include/queue.hpp b/src/agent/queue/include/queue.hpp index c05f86fca8b..fc7667d7672 100644 --- a/src/agent/queue/include/queue.hpp +++ b/src/agent/queue/include/queue.hpp @@ -1,13 +1,9 @@ #include -#include #include -#include #include #include #include -#include #include -#include #include #include "shared.hpp" @@ -17,6 +13,16 @@ constexpr int DEFAULT_MAX = 10; constexpr int DEFAULT_TIMEOUT_S = 3; +/** + * @brief Map for transforing Message type name to string + * + */ +static const std::map MessageTypeName { + {MessageType::STATELESS, "STATELESS"}, + {MessageType::STATEFUL, "STATEFUL"}, + {MessageType::COMMAND, "COMMAND"}, +}; + // Factory class class PersistenceFactory { @@ -42,6 +48,14 @@ class PersistenceFactory */ class PersistedQueue { +private: + const MessageType m_queueType; + const int m_max_size; + const std::chrono::seconds m_timeout; + std::unique_ptr m_persistenceDest; + std::mutex m_mtx; + std::condition_variable m_cv; + public: PersistedQueue(MessageType queueType, int max_size, int timeout) : m_queueType(queueType) @@ -83,11 +97,7 @@ class PersistedQueue * * @return int */ - int getItemsAvailable() - { - std::unique_lock lock(m_mtx); - return m_persistenceDest->GetElementCount(); - } + int getItemsAvailable(); /** * @brief @@ -96,40 +106,7 @@ class PersistedQueue * @return true * @return false */ - bool insertMessage(Message event) - { - std::unique_lock lock(m_mtx); - bool success = false; - size_t spaceAvailable = - (m_max_size > m_persistenceDest->GetElementCount()) ? m_max_size - m_persistenceDest->GetElementCount() : 0; - if (spaceAvailable) - { - // TODO: handle response - success = true; - auto messageData = event.data; - if (messageData.is_array()) - { - if (messageData.size() <= spaceAvailable) - { - for (const auto& singleMessageData : messageData) - { - m_persistenceDest->Store(singleMessageData); - m_cv.notify_all(); - } - } - else - { - success = false; - } - } - else - { - m_persistenceDest->Store(event.data); - m_cv.notify_all(); - } - } - return success; - }; + bool insertMessage(Message event); /** * @brief @@ -138,25 +115,14 @@ class PersistedQueue * @return true * @return false */ - bool removeNMessages(int qttyMessages) - { - std::unique_lock lock(m_mtx); - auto linesRemoved = m_persistenceDest->RemoveMultiple(qttyMessages); - m_cv.notify_all(); - return linesRemoved != 0; - }; + bool removeNMessages(int qttyMessages); /** * @brief Get the Message object * * @return Message */ - Message getMessage() - { - std::unique_lock lock(m_mtx); - auto messageData = m_persistenceDest->RetrieveMultiple(1); - return Message(m_queueType, messageData); - }; + Message getMessage(); /** * @brief @@ -164,12 +130,7 @@ class PersistedQueue * @param n * @return Message */ - Message getNMessages(int n) - { - std::unique_lock lock(m_mtx); - auto messageData = m_persistenceDest->RetrieveMultiple(n); - return Message(m_queueType, messageData); - }; + Message getNMessages(int n); /** * @brief @@ -177,11 +138,7 @@ class PersistedQueue * @return true * @return false */ - bool empty() - { - const auto items = getItemsAvailable(); - return items == 0; - } + bool empty(); /** * @brief @@ -189,23 +146,9 @@ class PersistedQueue * @return true * @return false */ - bool isFull() - { - std::unique_lock lock(m_mtx); - return m_persistenceDest->GetElementCount() == m_max_size; - } + bool isFull(); - void waitUntilNotFull() - { - std::unique_lock lock(m_mtx); - m_cv.wait_for(lock, - m_timeout, - [this] - { - std::cout << " waiting " << std::endl; - return m_persistenceDest->GetElementCount() < m_max_size; - }); - } + void waitUntilNotFull(); /** * A function that waits until the queue is not full or until a stop signal is received. @@ -214,27 +157,8 @@ class PersistedQueue * * @throws None */ - void waitUntilNotFullOrStoped(std::stop_token stopToken) - { - std::unique_lock lock(m_mtx); - m_cv.wait(lock, - [this, stopToken] - { - std::cout << " waiting " << std::endl; - bool menor = (m_persistenceDest->GetElementCount() < m_max_size); - bool stopped = (stopToken.stop_possible() && stopToken.stop_requested()); - std::cout << "menor" << menor << "stopped" << stopped << std::endl; - return menor || stopped ; - }); - } + void waitUntilNotFullOrStoped(std::stop_token stopToken); -private: - const MessageType m_queueType; - const int m_max_size; - const std::chrono::seconds m_timeout; - std::unique_ptr m_persistenceDest; - std::mutex m_mtx; - std::condition_variable m_cv; }; /** @@ -275,88 +199,19 @@ class MultiTypeQueue ~MultiTypeQueue() {}; - bool stopablePush(Message message, std::stop_token stopToken) - { - bool result = false; - std::unique_lock mapLock(m_mapMutex); - - if (m_queuesMap.contains(message.type)) - { - auto& queue = m_queuesMap[message.type]; - mapLock.unlock(); - - // Wait until the queue is not full - if (stopToken.stop_possible()) - { - queue->waitUntilNotFullOrStoped(stopToken); - } - - // Insert the message - result = queue->insertMessage(message); - if (!result) - { - std::cout << "Failed to insert message: " << message.data << std::endl; - } - } - else - { - std::cout << "error didn't find the queue" << std::endl; - } - return result; - } - /** * @brief: timeoutPush message to a queue of t * * @param message */ - bool timeoutPush(Message message, bool shouldWait = false) - { - bool result = false; - std::unique_lock mapLock(m_mapMutex); - - if (m_queuesMap.contains(message.type)) - { - auto& queue = m_queuesMap[message.type]; - mapLock.unlock(); - - // Wait until the queue is not full - if (shouldWait) - { - queue->waitUntilNotFull(); - } - //FIXME - // else - // { - // std::cout << "Can failed because os full queue" << std::endl; - // } - - // Insert the message - result = queue->insertMessage(message); - if (!result) - { - std::cout << "Failed to insert message: " << message.data << std::endl; - } - } - else - { - std::cout << "error didn't find the queue" << std::endl; - } - return result; - } + bool timeoutPush(Message message, bool shouldWait = false); /** * @brief * * @param messages */ - void timeoutPush(std::vector messages) - { - for (const auto& singleMessage : messages) - { - timeoutPush(singleMessage); - } - } + void timeoutPush(std::vector messages); /** * @brief Get the Last Message object @@ -364,23 +219,7 @@ class MultiTypeQueue * @param type * @return Message */ - Message getLastMessage(MessageType type) - { - Message result(type, {}); - std::unique_lock mapLock(m_mapMutex); - if (m_queuesMap.contains(type)) - { - auto& queue = m_queuesMap[type]; - mapLock.unlock(); - result = queue->getMessage(); - } - else - { - // TODO: error / logging handling !!! - std::cout << "error didn't find the queue" << std::endl; - } - return result; - } + Message getLastMessage(MessageType type); /** * @brief deletes a message from a queue @@ -389,24 +228,7 @@ class MultiTypeQueue * @return true popped succesfully * @return false wasn't able to pop message */ - bool popLastMessage(MessageType type) - { - bool result = false; - std::unique_lock mapLock(m_mapMutex); - if (m_queuesMap.contains(type)) - { - auto& queue = m_queuesMap[type]; - mapLock.unlock(); - // Handle return value - result = queue->removeNMessages(1); - } - else - { - // TODO: error / logging handling !!! - std::cout << "error didn't find the queue" << std::endl; - } - return result; - } + bool popLastMessage(MessageType type); /** * @brief @@ -416,23 +238,7 @@ class MultiTypeQueue * @return true * @return false */ - bool popNMessages(MessageType type, int messageQuantity) - { - bool result = false; - std::unique_lock mapLock(m_mapMutex); - if (m_queuesMap.contains(type)) - { - auto& queue = m_queuesMap[type]; - mapLock.unlock(); - result = queue->removeNMessages(messageQuantity); - } - else - { - // TODO: error / logging handling !!! - std::cout << "error didn't find the queue" << std::endl; - } - return result; - } + bool popNMessages(MessageType type, int messageQuantity); /** * @brief @@ -441,22 +247,7 @@ class MultiTypeQueue * @return true * @return false */ - bool isEmptyByType(MessageType type) - { - std::unique_lock mapLock(m_mapMutex); - if (m_queuesMap.contains(type)) - { - auto& queue = m_queuesMap[type]; - mapLock.unlock(); - return queue->empty(); - } - else - { - // TODO: error / logging handling !!! - std::cout << "error didn't find the queue" << std::endl; - } - return false; - } + bool isEmptyByType(MessageType type); /** * @brief Get the Items By Type object @@ -465,20 +256,5 @@ class MultiTypeQueue * @return true * @return false */ - int getItemsByType(MessageType type) - { - std::unique_lock mapLock(m_mapMutex); - if (m_queuesMap.contains(type)) - { - auto& queue = m_queuesMap[type]; - mapLock.unlock(); - return queue->getItemsAvailable(); - } - else - { - // TODO: error / logging handling !!! - std::cout << "error didn't find the queue" << std::endl; - } - return false; - } + int getItemsByType(MessageType type); }; diff --git a/src/agent/queue/include/shared.hpp b/src/agent/queue/include/shared.hpp index 205c8fa2539..d3da8654c62 100644 --- a/src/agent/queue/include/shared.hpp +++ b/src/agent/queue/include/shared.hpp @@ -17,16 +17,6 @@ enum MessageType COMMAND }; -/** - * @brief Map for transforing Message type name to string - * - */ -std::map MessageTypeName { - {MessageType::STATELESS, "STATELESS"}, - {MessageType::STATEFUL, "STATEFUL"}, - {MessageType::COMMAND, "COMMAND"}, -}; - /** * @brief Wrapper for Message data and type * diff --git a/src/agent/queue/src/queue.cpp b/src/agent/queue/src/queue.cpp new file mode 100644 index 00000000000..86ca51fe33e --- /dev/null +++ b/src/agent/queue/src/queue.cpp @@ -0,0 +1,246 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "queue.hpp" + +int PersistedQueue::getItemsAvailable() +{ + std::unique_lock lock(m_mtx); + return m_persistenceDest->GetElementCount(); +} + +bool PersistedQueue::insertMessage(Message event) +{ + std::unique_lock lock(m_mtx); + bool success = false; + size_t spaceAvailable = + (m_max_size > m_persistenceDest->GetElementCount()) ? m_max_size - m_persistenceDest->GetElementCount() : 0; + if (spaceAvailable) + { + // TODO: handle response + success = true; + auto messageData = event.data; + if (messageData.is_array()) + { + if (messageData.size() <= spaceAvailable) + { + for (const auto& singleMessageData : messageData) + { + m_persistenceDest->Store(singleMessageData); + m_cv.notify_all(); + } + } + else + { + success = false; + } + } + else + { + m_persistenceDest->Store(event.data); + m_cv.notify_all(); + } + } + return success; +}; + +bool PersistedQueue::removeNMessages(int qttyMessages) +{ + std::unique_lock lock(m_mtx); + auto linesRemoved = m_persistenceDest->RemoveMultiple(qttyMessages); + m_cv.notify_all(); + return linesRemoved != 0; +}; + +Message PersistedQueue::getMessage() +{ + std::unique_lock lock(m_mtx); + auto messageData = m_persistenceDest->RetrieveMultiple(1); + return Message(m_queueType, messageData); +}; + +Message PersistedQueue::getNMessages(int n) +{ + std::unique_lock lock(m_mtx); + auto messageData = m_persistenceDest->RetrieveMultiple(n); + return Message(m_queueType, messageData); +}; + +bool PersistedQueue::empty() +{ + const auto items = getItemsAvailable(); + return items == 0; +} + +bool PersistedQueue::isFull() +{ + std::unique_lock lock(m_mtx); + return m_persistenceDest->GetElementCount() == m_max_size; +} + +void PersistedQueue::waitUntilNotFull() +{ + std::unique_lock lock(m_mtx); + m_cv.wait_for(lock, + m_timeout, + [this] + { + std::cout << " waiting " << std::endl; + return m_persistenceDest->GetElementCount() < m_max_size; + }); +} + +void PersistedQueue::waitUntilNotFullOrStoped(std::stop_token stopToken) +{ + std::unique_lock lock(m_mtx); + m_cv.wait(lock, + [this, stopToken] + { + std::cout << " waiting " << std::endl; + bool menor = (m_persistenceDest->GetElementCount() < m_max_size); + bool stopped = (stopToken.stop_possible() && stopToken.stop_requested()); + std::cout << "menor" << menor << "stopped" << stopped << std::endl; + return menor || stopped; + }); +} + +bool MultiTypeQueue::timeoutPush(Message message, bool shouldWait ) +{ + bool result = false; + std::unique_lock mapLock(m_mapMutex); + + if (m_queuesMap.contains(message.type)) + { + auto& queue = m_queuesMap[message.type]; + mapLock.unlock(); + + // Wait until the queue is not full + if (shouldWait) + { + queue->waitUntilNotFull(); + } + // FIXME + // else + // { + // std::cout << "Can failed because os full queue" << std::endl; + // } + + // Insert the message + result = queue->insertMessage(message); + if (!result) + { + std::cout << "Failed to insert message: " << message.data << std::endl; + } + } + else + { + std::cout << "error didn't find the queue" << std::endl; + } + return result; +} + +void MultiTypeQueue::timeoutPush(std::vector messages) +{ + for (const auto& singleMessage : messages) + { + timeoutPush(singleMessage); + } +} + +Message MultiTypeQueue::getLastMessage(MessageType type) +{ + Message result(type, {}); + std::unique_lock mapLock(m_mapMutex); + if (m_queuesMap.contains(type)) + { + auto& queue = m_queuesMap[type]; + mapLock.unlock(); + result = queue->getMessage(); + } + else + { + // TODO: error / logging handling !!! + std::cout << "error didn't find the queue" << std::endl; + } + return result; +} + +bool MultiTypeQueue::popLastMessage(MessageType type) +{ + bool result = false; + std::unique_lock mapLock(m_mapMutex); + if (m_queuesMap.contains(type)) + { + auto& queue = m_queuesMap[type]; + mapLock.unlock(); + // Handle return value + result = queue->removeNMessages(1); + } + else + { + // TODO: error / logging handling !!! + std::cout << "error didn't find the queue" << std::endl; + } + return result; +} + +bool MultiTypeQueue::popNMessages(MessageType type, int messageQuantity) +{ + bool result = false; + std::unique_lock mapLock(m_mapMutex); + if (m_queuesMap.contains(type)) + { + auto& queue = m_queuesMap[type]; + mapLock.unlock(); + result = queue->removeNMessages(messageQuantity); + } + else + { + // TODO: error / logging handling !!! + std::cout << "error didn't find the queue" << std::endl; + } + return result; +} + +bool MultiTypeQueue::isEmptyByType(MessageType type) +{ + std::unique_lock mapLock(m_mapMutex); + if (m_queuesMap.contains(type)) + { + auto& queue = m_queuesMap[type]; + mapLock.unlock(); + return queue->empty(); + } + else + { + // TODO: error / logging handling !!! + std::cout << "error didn't find the queue" << std::endl; + } + return false; +} + +int MultiTypeQueue::getItemsByType(MessageType type) +{ + std::unique_lock mapLock(m_mapMutex); + if (m_queuesMap.contains(type)) + { + auto& queue = m_queuesMap[type]; + mapLock.unlock(); + return queue->getItemsAvailable(); + } + else + { + // TODO: error / logging handling !!! + std::cout << "error didn't find the queue" << std::endl; + } + return false; +} diff --git a/src/agent/queue/tests/CMakeLists.txt b/src/agent/queue/tests/CMakeLists.txt index 239473d8738..8cb67eebea4 100644 --- a/src/agent/queue/tests/CMakeLists.txt +++ b/src/agent/queue/tests/CMakeLists.txt @@ -12,15 +12,14 @@ include_directories(${SQLiteCpp_INCLUDE_DIRS}) file(GLOB QUEUE_UNIT_TEST_SRC "*.cpp") # Create a test executable -add_executable(test_queue ${QUEUE_UNIT_TEST_SRC} - ../src/sqlitestorage.cpp - ) +add_executable(test_queue ${QUEUE_UNIT_TEST_SRC}) # add_test(AllTestsInMain main) # Link the test executable against Google Test target_link_libraries(test_queue ${GTEST_LIBRARIES} ${GTEST_MAIN_LIBRARIES} + QueueTest SQLiteCpp nlohmann_json::nlohmann_json) # target_link_libraries(main PRIVATE GTest::gtest GTest::gtest_main GTest::gmock GTest::gmock_main) From 3567cb66a34a4361fca328f3a029e851038688af Mon Sep 17 00:00:00 2001 From: LucioDonda Date: Mon, 22 Jul 2024 11:28:52 -0300 Subject: [PATCH 15/31] fix: change to a correct use of clang-format --- src/agent/queue/include/persistence.h | 5 +++-- src/agent/queue/include/queue.hpp | 1 - src/agent/queue/include/shared.hpp | 2 +- src/agent/queue/src/queue.cpp | 12 ++---------- src/agent/queue/tests/main.cpp | 3 ++- src/agent/queue/tests/queue_test.cpp | 20 +++++++++----------- src/agent/queue/tests/queue_test.hpp | 24 +++++++++++------------- 7 files changed, 28 insertions(+), 39 deletions(-) diff --git a/src/agent/queue/include/persistence.h b/src/agent/queue/include/persistence.h index 87d579fe18b..bb626239c4e 100644 --- a/src/agent/queue/include/persistence.h +++ b/src/agent/queue/include/persistence.h @@ -2,8 +2,8 @@ #define PERSISTENCE_H #include -#include #include +#include using json = nlohmann::json; @@ -12,7 +12,8 @@ using json = nlohmann::json; * * This interface defines methods for storing, retrieving, and removing JSON messages. */ -class Persistence { +class Persistence +{ public: /** * @brief Virtual destructor. diff --git a/src/agent/queue/include/queue.hpp b/src/agent/queue/include/queue.hpp index fc7667d7672..b4188112185 100644 --- a/src/agent/queue/include/queue.hpp +++ b/src/agent/queue/include/queue.hpp @@ -158,7 +158,6 @@ class PersistedQueue * @throws None */ void waitUntilNotFullOrStoped(std::stop_token stopToken); - }; /** diff --git a/src/agent/queue/include/shared.hpp b/src/agent/queue/include/shared.hpp index d3da8654c62..81a453cc337 100644 --- a/src/agent/queue/include/shared.hpp +++ b/src/agent/queue/include/shared.hpp @@ -2,7 +2,7 @@ #include -//TODO: should be moved to Config +// TODO: should be moved to Config constexpr char DEFAULT_FILE_PATH[] = "/home/vagrant/FILE_"; constexpr char DEFAULT_DB_PATH[] = "queue.db"; diff --git a/src/agent/queue/src/queue.cpp b/src/agent/queue/src/queue.cpp index 86ca51fe33e..ee8b82c90d4 100644 --- a/src/agent/queue/src/queue.cpp +++ b/src/agent/queue/src/queue.cpp @@ -90,13 +90,7 @@ bool PersistedQueue::isFull() void PersistedQueue::waitUntilNotFull() { std::unique_lock lock(m_mtx); - m_cv.wait_for(lock, - m_timeout, - [this] - { - std::cout << " waiting " << std::endl; - return m_persistenceDest->GetElementCount() < m_max_size; - }); + m_cv.wait_for(lock, m_timeout, [this] { return m_persistenceDest->GetElementCount() < m_max_size; }); } void PersistedQueue::waitUntilNotFullOrStoped(std::stop_token stopToken) @@ -105,15 +99,13 @@ void PersistedQueue::waitUntilNotFullOrStoped(std::stop_token stopToken) m_cv.wait(lock, [this, stopToken] { - std::cout << " waiting " << std::endl; bool menor = (m_persistenceDest->GetElementCount() < m_max_size); bool stopped = (stopToken.stop_possible() && stopToken.stop_requested()); - std::cout << "menor" << menor << "stopped" << stopped << std::endl; return menor || stopped; }); } -bool MultiTypeQueue::timeoutPush(Message message, bool shouldWait ) +bool MultiTypeQueue::timeoutPush(Message message, bool shouldWait) { bool result = false; std::unique_lock mapLock(m_mapMutex); diff --git a/src/agent/queue/tests/main.cpp b/src/agent/queue/tests/main.cpp index 5ebbc761adf..9bb465e024a 100644 --- a/src/agent/queue/tests/main.cpp +++ b/src/agent/queue/tests/main.cpp @@ -1,6 +1,7 @@ #include -int main(int argc, char** argv) { +int main(int argc, char** argv) +{ ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/src/agent/queue/tests/queue_test.cpp b/src/agent/queue/tests/queue_test.cpp index 06b2d6d7ec9..f81c848cb95 100644 --- a/src/agent/queue/tests/queue_test.cpp +++ b/src/agent/queue/tests/queue_test.cpp @@ -110,7 +110,7 @@ TEST_F(JsonTest, JSONArrays) int i = 0; for (auto& singleMessage : multipleDataContent.items()) { - EXPECT_EQ(singleMessage.value(),"content " + std::to_string(++i)); + EXPECT_EQ(singleMessage.value(), "content " + std::to_string(++i)); } } @@ -205,7 +205,7 @@ TEST_F(QueueTest, SinglePushPopFullWithTimeout) const json dataContent = R"({"Data" : "for COMMAND3"})"; Message exampleMessage {messageType, dataContent}; - queue.timeoutPush({messageType, dataContent},true); + queue.timeoutPush({messageType, dataContent}, true); auto items = queue.getItemsByType(MessageType::COMMAND); EXPECT_EQ(items, SMALL_QUEUE_QTTY); @@ -256,17 +256,16 @@ TEST_F(QueueTest, MultithreadDifferentType) std::thread consumerThread1(consumerStateLess, std::ref(itemsToConsume)); std::thread consumerThread2(consumerStateFull, std::ref(itemsToConsume)); - if(consumerThread1.joinable()) + if (consumerThread1.joinable()) { consumerThread1.join(); } - if(consumerThread2.joinable()) + if (consumerThread2.joinable()) { consumerThread2.join(); } - EXPECT_NE(0, queue.getItemsByType(MessageType::STATELESS)); EXPECT_NE(0, queue.getItemsByType(MessageType::STATEFUL)); EXPECT_FALSE(queue.isEmptyByType(MessageType::STATELESS)); @@ -276,17 +275,16 @@ TEST_F(QueueTest, MultithreadDifferentType) std::thread consumerThread12(consumerStateLess, std::ref(itemsToConsume)); std::thread consumerThread22(consumerStateFull, std::ref(itemsToConsume)); - if(consumerThread12.joinable()) + if (consumerThread12.joinable()) { consumerThread12.join(); } - if(consumerThread22.joinable()) + if (consumerThread22.joinable()) { consumerThread22.join(); } - // FIXME: this doesn't match EXPECT_EQ(0, queue.getItemsByType(MessageType::STATELESS)); EXPECT_EQ(0, queue.getItemsByType(MessageType::STATEFUL)); @@ -334,12 +332,12 @@ TEST_F(QueueTest, MultithreadSameType) std::thread consumerThread1(consumerCommand1, std::ref(itemsToConsume)); std::thread messageProducerThread1(consumerCommand2, std::ref(itemsToConsume)); - if(messageProducerThread1.joinable()) + if (messageProducerThread1.joinable()) { messageProducerThread1.join(); } - if(consumerThread1.joinable()) + if (consumerThread1.joinable()) { consumerThread1.join(); } @@ -383,5 +381,5 @@ TEST_F(QueueTest, MultiplePushSeveralMultiplePops) EXPECT_TRUE(queue.popNMessages(MessageType::STATELESS, 3)); EXPECT_TRUE(queue.isEmptyByType(MessageType::STATELESS)); - EXPECT_EQ(0,queue.getItemsByType(MessageType::STATELESS)); + EXPECT_EQ(0, queue.getItemsByType(MessageType::STATELESS)); } diff --git a/src/agent/queue/tests/queue_test.hpp b/src/agent/queue/tests/queue_test.hpp index 548a432a121..22ab133e43f 100644 --- a/src/agent/queue/tests/queue_test.hpp +++ b/src/agent/queue/tests/queue_test.hpp @@ -10,29 +10,27 @@ */ #ifndef _QUEUE_TEST_H #define _QUEUE_TEST_H -#include "gtest/gtest.h" #include "gmock/gmock.h" +#include "gtest/gtest.h" class QueueTest : public ::testing::Test { - protected: +protected: + QueueTest() = default; + virtual ~QueueTest() = default; - QueueTest() = default; - virtual ~QueueTest() = default; - - void SetUp() override; - void TearDown() override; + void SetUp() override; + void TearDown() override; }; class JsonTest : public ::testing::Test { - protected: - - JsonTest() = default; - virtual ~JsonTest() = default; +protected: + JsonTest() = default; + virtual ~JsonTest() = default; - void SetUp() override {}; - void TearDown() override {}; + void SetUp() override {}; + void TearDown() override {}; }; #endif //_QUEUE_TEST_H From 7ec08f90fbc8d535398d32e054f7e621f45e77fd Mon Sep 17 00:00:00 2001 From: LucioDonda Date: Mon, 22 Jul 2024 18:20:37 -0300 Subject: [PATCH 16/31] feat: unify cmakes and vcpkg usage --- src/agent/CMakeLists.txt | 1 + src/agent/queue/CMakeLists.txt | 31 ++++++++++++---------------- src/agent/queue/tests/CMakeLists.txt | 11 +--------- 3 files changed, 15 insertions(+), 28 deletions(-) diff --git a/src/agent/CMakeLists.txt b/src/agent/CMakeLists.txt index b7a95cb8083..51cda998b9a 100644 --- a/src/agent/CMakeLists.txt +++ b/src/agent/CMakeLists.txt @@ -31,4 +31,5 @@ target_link_libraries(Agent PUBLIC ConfigurationParser Communicator AgentInfo PR if(BUILD_TESTS) enable_testing() add_subdirectory(tests) + add_subdirectory(queue) endif() diff --git a/src/agent/queue/CMakeLists.txt b/src/agent/queue/CMakeLists.txt index 95c8db600d9..d4d469e4dbf 100644 --- a/src/agent/queue/CMakeLists.txt +++ b/src/agent/queue/CMakeLists.txt @@ -1,29 +1,24 @@ -cmake_minimum_required(VERSION 3.18) +cmake_minimum_required(VERSION 3.22) -# Set the project name and C++ standard -project(QueueTest LANGUAGES CXX) +project(queue_test LANGUAGES CXX) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) -# Find the nlohmann::json and SQLiteCpp package using vcpkg -find_package(SQLiteCpp REQUIRED) -find_package(nlohmann_json REQUIRED) - set(CMAKE_CXX_FLAGS "-Wall -Wextra -Wunused -pthread") -# Add the include directory for shared headers include_directories(${CMAKE_SOURCE_DIR}/include) -# include_directories(${CMAKE_SOURCE_DIR}/src) - -add_subdirectory(tests) -# Add the executable -add_library(QueueTest - # src/main.cpp +add_library(queue_test src/sqlitestorage.cpp src/queue.cpp - ) +) + +find_package(SQLiteCpp REQUIRED) +find_package(nlohmann_json REQUIRED) + +target_include_directories(queue_test PRIVATE include ${SQLiteCpp_INCLUDE_DIRS}) +target_link_libraries(queue_test PRIVATE SQLiteCpp nlohmann_json::nlohmann_json) -# Link the nlohmann::json library to your project -target_include_directories(QueueTest PRIVATE include ${SQLiteCpp_INCLUDE_DIRS}) -target_link_libraries(QueueTest PRIVATE SQLiteCpp nlohmann_json::nlohmann_json) +if(BUILD_TESTS) + add_subdirectory(tests) +endif() diff --git a/src/agent/queue/tests/CMakeLists.txt b/src/agent/queue/tests/CMakeLists.txt index 8cb67eebea4..79141e9d909 100644 --- a/src/agent/queue/tests/CMakeLists.txt +++ b/src/agent/queue/tests/CMakeLists.txt @@ -1,9 +1,4 @@ -# Enable testing -# enable_testing() - -# Link against Google Test find_package(GTest REQUIRED) -# find_package(GTest CONFIG REQUIRED) include_directories(../include) include_directories(${GTEST_INCLUDE_DIRS}) @@ -11,15 +6,11 @@ include_directories(${SQLiteCpp_INCLUDE_DIRS}) file(GLOB QUEUE_UNIT_TEST_SRC "*.cpp") -# Create a test executable add_executable(test_queue ${QUEUE_UNIT_TEST_SRC}) -# add_test(AllTestsInMain main) -# Link the test executable against Google Test target_link_libraries(test_queue ${GTEST_LIBRARIES} ${GTEST_MAIN_LIBRARIES} - QueueTest + queue_test SQLiteCpp nlohmann_json::nlohmann_json) -# target_link_libraries(main PRIVATE GTest::gtest GTest::gtest_main GTest::gmock GTest::gmock_main) From 545614f00e056d7018105cb5a3772d97aa2c511a Mon Sep 17 00:00:00 2001 From: LucioDonda Date: Mon, 29 Jul 2024 16:38:24 -0300 Subject: [PATCH 17/31] feat: multithreading fis sqlitestorage level, tests and def on headers --- src/agent/queue/include/queue.hpp | 5 + src/agent/queue/include/shared.hpp | 17 +- src/agent/queue/include/sqlitestorage.h | 31 +++- src/agent/queue/src/sqlitestorage.cpp | 173 ++++++++++++------ src/agent/queue/tests/CMakeLists.txt | 21 ++- src/agent/queue/tests/sqlitestorage_test.cpp | 179 +++++++++++++++++++ 6 files changed, 356 insertions(+), 70 deletions(-) create mode 100644 src/agent/queue/tests/sqlitestorage_test.cpp diff --git a/src/agent/queue/include/queue.hpp b/src/agent/queue/include/queue.hpp index b4188112185..13560759e07 100644 --- a/src/agent/queue/include/queue.hpp +++ b/src/agent/queue/include/queue.hpp @@ -1,3 +1,6 @@ +#ifndef QUEUE_H +#define QUEUE_H + #include #include #include @@ -257,3 +260,5 @@ class MultiTypeQueue */ int getItemsByType(MessageType type); }; + +#endif // QUEUE_H diff --git a/src/agent/queue/include/shared.hpp b/src/agent/queue/include/shared.hpp index 81a453cc337..a89eb2c4c15 100644 --- a/src/agent/queue/include/shared.hpp +++ b/src/agent/queue/include/shared.hpp @@ -1,3 +1,6 @@ +#ifndef SHARED_H +#define SHARED_H + #include #include @@ -31,16 +34,6 @@ class Message , data(d) { } - - /** - * @brief overloading == operator for messages - * - * @param other Secondary Message parameter - * @return true when both messages matches in type and data - * @return false when the messages don't match in type and data - */ - bool operator==(const Message& other) const - { - return (this->type == other.type) && (this->data == other.data); - } }; + +#endif // SHARED_H diff --git a/src/agent/queue/include/sqlitestorage.h b/src/agent/queue/include/sqlitestorage.h index c71a00d17c8..839a680a3f4 100644 --- a/src/agent/queue/include/sqlitestorage.h +++ b/src/agent/queue/include/sqlitestorage.h @@ -1,8 +1,12 @@ #ifndef SQLITE_STORAGE_H #define SQLITE_STORAGE_H +#include +#include +#include #include #include +#include #include #include @@ -98,18 +102,35 @@ class SQLiteStorage : public Persistence */ void InitializeTable(); + /** + * @brief Private method for waiting for database access. + * + */ + void waitForDatabaseAccess(); + + /** + * @brief Private method for releasing database access. + * + */ + void releaseDatabaseAccess(); + /// The name of the SQLite database file. - std::string m_dbName; + const std::string m_dbName; /// The name of the table to use for storing messages. - std::string m_tableName; + const std::string m_tableName; /// Pointer to the SQLite database connection. - //SQLite::Database* m_db; - std::unique_ptr m_db; + std::shared_ptr m_db; /// Mutex to ensure thread-safe operations. - std::mutex m_mtx; + std::mutex m_mutex; + + /// @brief condition variable to wait for database access + std::condition_variable m_cv; + + // TODO: should it be atomic? + bool m_db_in_use = false; }; #endif // SQLITE_STORAGE_H diff --git a/src/agent/queue/src/sqlitestorage.cpp b/src/agent/queue/src/sqlitestorage.cpp index 9cae7c69931..5f32bbaed92 100644 --- a/src/agent/queue/src/sqlitestorage.cpp +++ b/src/agent/queue/src/sqlitestorage.cpp @@ -1,132 +1,203 @@ #include "sqlitestorage.h" -#include #include +#include - -#define CREATE_TABLE_QUERY "CREATE TABLE IF NOT EXISTS " + m_tableName + " (message TEXT NOT NULL);" -#define INSERT_QUERY "INSERT INTO " + m_tableName + " (message) VALUES (?);" -#define SELECT_QUERY "SELECT message FROM " + m_tableName + " WHERE rowid = ?;" +// TODO: replace define for constexpr and do only use memebers on runtime +#define CREATE_TABLE_QUERY "CREATE TABLE IF NOT EXISTS " + m_tableName + " (message TEXT NOT NULL);" +#define INSERT_QUERY "INSERT INTO " + m_tableName + " (message) VALUES (?);" +#define SELECT_QUERY "SELECT message FROM " + m_tableName + " WHERE rowid = ?;" #define SELECT_MULTIPLE_QUERY "SELECT message FROM " + m_tableName + " ORDER BY rowid ASC LIMIT ?;" -#define DELETE_QUERY "DELETE FROM " + m_tableName + " WHERE rowid = ?;" -#define DELETE_MULTIPLE_QUERY "DELETE FROM " + m_tableName + " WHERE rowid IN (SELECT rowid FROM " + m_tableName + " ORDER BY rowid ASC LIMIT ?);" +#define DELETE_QUERY "DELETE FROM " + m_tableName + " WHERE rowid = ?;" +#define DELETE_MULTIPLE_QUERY \ + "DELETE FROM " + m_tableName + " WHERE rowid IN (SELECT rowid FROM " + m_tableName + " ORDER BY rowid ASC LIMIT ?);" #define COUNT_QUERY "SELECT COUNT(*) FROM " + m_tableName + ";" SQLiteStorage::SQLiteStorage(const std::string& dbName, const std::string& tableName) - : m_dbName(dbName), m_tableName(tableName), m_db(nullptr) { - try { - // open db - m_db = std::make_unique(m_dbName, SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE); + : m_dbName(dbName) + , m_tableName(tableName) + , m_db(make_shared(dbName, SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE)) +{ + try + { + // Open the database in WAL mode + m_db->exec("PRAGMA journal_mode=WAL;"); InitializeTable(); - } catch (const std::exception& e) { + } + catch (const std::exception& e) + { std::cerr << "Error initializing database: " << e.what() << std::endl; throw; } } -SQLiteStorage::~SQLiteStorage() { - // No need to explicitly close the database as std::unique_ptr will handle it -} +SQLiteStorage::~SQLiteStorage() {} -void SQLiteStorage::InitializeTable() { - std::lock_guard lock(m_mtx); - try { +void SQLiteStorage::InitializeTable() +{ + std::lock_guard lock(m_mutex); + try + { std::string createTableQuery = CREATE_TABLE_QUERY; m_db->exec(createTableQuery); - } catch (const std::exception& e) { + } + catch (const std::exception& e) + { std::cerr << "Error initializing table: " << e.what() << std::endl; throw; } } +void SQLiteStorage::waitForDatabaseAccess() +{ + std::unique_lock lock(m_mutex); + m_cv.wait(lock, [this] { return !m_db_in_use; }); + m_db_in_use = true; +} + +void SQLiteStorage::releaseDatabaseAccess() +{ + std::lock_guard lock(m_mutex); + m_db_in_use = false; + m_cv.notify_one(); +} + +void SQLiteStorage::Store(const json& message) +{ + waitForDatabaseAccess(); -void SQLiteStorage::Store(const json& message) { - std::lock_guard lock(m_mtx); - try { - std::string insertQuery = INSERT_QUERY; - SQLite::Statement query(*m_db, insertQuery); + std::string insertQuery = INSERT_QUERY; + SQLite::Statement query = SQLite::Statement(*m_db, insertQuery); + + if (message.is_array()) + { + SQLite::Transaction transaction(*m_db); + for (const auto& singleMessageData : message) + { + try + { + query.bind(1, singleMessageData.dump()); + query.exec(); + } + catch (const SQLite::Exception& e) + { + std::cerr << "2 " << e.what() << '\n'; + } + // Reset the query to reuse it for the next message + query.reset(); + } + transaction.commit(); + } + else + { + SQLite::Transaction transaction(*m_db); query.bind(1, message.dump()); query.exec(); - } catch (const std::exception& e) { - std::cerr << "Error storing message: " << e.what() << std::endl; - throw; + transaction.commit(); } + releaseDatabaseAccess(); } -json SQLiteStorage::Retrieve(int id) { - std::lock_guard lock(m_mtx); - try { +json SQLiteStorage::Retrieve(int id) +{ + try + { std::string selectQuery = SELECT_QUERY; SQLite::Statement query(*m_db, selectQuery); query.bind(1, id); json message; - if (query.executeStep()) { + if (query.executeStep()) + { message = json::parse(query.getColumn(0).getString()); - } else { + } + else + { message = nullptr; } return message; - } catch (const std::exception& e) { + } + catch (const SQLite::Exception& e) + { std::cerr << "Error retrieving message: " << e.what() << std::endl; throw; } } -json SQLiteStorage::RetrieveMultiple(int n) { - std::lock_guard lock(m_mtx); - try { +json SQLiteStorage::RetrieveMultiple(int n) +{ + try + { std::string selectQuery = SELECT_MULTIPLE_QUERY; SQLite::Statement query(*m_db, selectQuery); query.bind(1, n); json messages = json::array(); - while (query.executeStep()) { + while (query.executeStep()) + { messages.push_back(json::parse(query.getColumn(0).getString())); } std::reverse(messages.begin(), messages.end()); return messages; - } catch (const std::exception& e) { + } + catch (const SQLite::Exception& e) + { std::cerr << "Error retrieving multiple messages: " << e.what() << std::endl; throw; } } -int SQLiteStorage::Remove(int id) { - std::lock_guard lock(m_mtx); - try { +int SQLiteStorage::Remove(int id) +{ + try + { std::string deleteQuery = DELETE_QUERY; SQLite::Statement query(*m_db, deleteQuery); query.bind(1, id); + SQLite::Transaction transaction(*m_db); query.exec(); + transaction.commit(); return 1; - } catch (const std::exception& e) { + } + catch (const SQLite::Exception& e) + { std::cerr << "Error removing message: " << e.what() << std::endl; throw; } } -int SQLiteStorage::RemoveMultiple(int n) { - std::lock_guard lock(m_mtx); - try { +int SQLiteStorage::RemoveMultiple(int n) +{ + try + { + waitForDatabaseAccess(); + SQLite::Transaction transaction(*m_db); std::string deleteQuery = DELETE_MULTIPLE_QUERY; SQLite::Statement query(*m_db, deleteQuery); query.bind(1, n); query.exec(); + transaction.commit(); + releaseDatabaseAccess(); return n; - } catch (const std::exception& e) { + } + catch (const SQLite::Exception& e) + { std::cerr << "Error removing multiple messages: " << e.what() << std::endl; throw; } } -int SQLiteStorage::GetElementCount() { - std::lock_guard lock(m_mtx); - try { +int SQLiteStorage::GetElementCount() +{ + try + { std::string countQuery = COUNT_QUERY; SQLite::Statement query(*m_db, countQuery); int count = 0; - if (query.executeStep()) { + if (query.executeStep()) + { count = query.getColumn(0).getInt(); } return count; - } catch (const std::exception& e) { + } + catch (const SQLite::Exception& e) + { std::cerr << "Error getting element count: " << e.what() << std::endl; throw; } diff --git a/src/agent/queue/tests/CMakeLists.txt b/src/agent/queue/tests/CMakeLists.txt index 79141e9d909..54d318d36a2 100644 --- a/src/agent/queue/tests/CMakeLists.txt +++ b/src/agent/queue/tests/CMakeLists.txt @@ -1,12 +1,18 @@ find_package(GTest REQUIRED) +# find_package(SQLiteCpp REQUIRED) +# find_package(nlohmann_json REQUIRED) include_directories(../include) include_directories(${GTEST_INCLUDE_DIRS}) include_directories(${SQLiteCpp_INCLUDE_DIRS}) -file(GLOB QUEUE_UNIT_TEST_SRC "*.cpp") +#TODO: is this reusable ? +# file(GLOB QUEUE_UNIT_TEST_SRC "*.cpp") -add_executable(test_queue ${QUEUE_UNIT_TEST_SRC}) +add_executable(test_queue + queue_test.cpp + ../src/sqlitestorage.cpp + ) target_link_libraries(test_queue ${GTEST_LIBRARIES} @@ -14,3 +20,14 @@ target_link_libraries(test_queue queue_test SQLiteCpp nlohmann_json::nlohmann_json) + +# Create a test executable for SQLiteStorage tests +add_executable(test_sqlitestorage + sqlitestorage_test.cpp + ../src/sqlitestorage.cpp) + +target_link_libraries(test_sqlitestorage + ${GTEST_LIBRARIES} + ${GTEST_MAIN_LIBRARIES} + SQLiteCpp + nlohmann_json::nlohmann_json) \ No newline at end of file diff --git a/src/agent/queue/tests/sqlitestorage_test.cpp b/src/agent/queue/tests/sqlitestorage_test.cpp new file mode 100644 index 00000000000..d79b5d7bdf0 --- /dev/null +++ b/src/agent/queue/tests/sqlitestorage_test.cpp @@ -0,0 +1,179 @@ +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include + +#include "sqlitestorage.h" + +using json = nlohmann::json; + +class SQLiteStorageTest : public ::testing::Test +{ +protected: + std::string dbName = "testdb"; + std::string tableName = "test_table"; + SQLiteStorage* storage; + + void SetUp() override + { + // Ensure the database file does not already exist + std::remove(dbName.c_str()); + + // Create a new SQLiteStorage instance + storage = new SQLiteStorage(dbName, tableName); + } + + void TearDown() override + { + // Clean up: Delete the SQLiteStorage instance and remove the database file + delete storage; + std::error_code ec; + if (std::filesystem::exists(dbName.c_str())) + { + std::filesystem::remove(dbName.c_str()); + if (ec) + { + std::cerr << "Error removing file: " << ec.message() << std::endl; + } + } + } +}; + +TEST_F(SQLiteStorageTest, StoreSingleMessage) +{ + json message = {{"key", "value"}}; + EXPECT_NO_THROW(storage->Store(message)); + EXPECT_EQ(storage->GetElementCount(), 1); + EXPECT_NO_THROW(storage->Store(message)); + EXPECT_EQ(storage->GetElementCount(), 2); +} + +TEST_F(SQLiteStorageTest, StoreMultipleMessages) +{ + json messages = json::array(); + messages.push_back({{"key", "value1"}}); + messages.push_back({{"key", "value2"}}); + EXPECT_NO_THROW(storage->Store(messages)); + EXPECT_EQ(storage->GetElementCount(), 2); +} + +TEST_F(SQLiteStorageTest, RetrieveMessage) +{ + json message = {{"key", "value"}}; + storage->Store(message); + json retrievedMessage = storage->Retrieve(1); + EXPECT_EQ(retrievedMessage["key"], "value"); +} + +TEST_F(SQLiteStorageTest, RetrieveMultipleMessages) +{ + json messages = json::array(); + messages.push_back({{"key", "value1"}}); + messages.push_back({{"key", "value2"}}); + storage->Store(messages); + json retrievedMessages = storage->RetrieveMultiple(2); + EXPECT_EQ(retrievedMessages.size(), 2); +} + +TEST_F(SQLiteStorageTest, RemoveMessage) +{ + json message = {{"key", "value"}}; + storage->Store(message); + EXPECT_EQ(storage->Remove(1), 1); + EXPECT_EQ(storage->GetElementCount(), 0); +} + +TEST_F(SQLiteStorageTest, RemoveMultipleMessages) +{ + json messages = json::array(); + messages.push_back({{"key", "value1"}}); + messages.push_back({{"key", "value2"}}); + storage->Store(messages); + EXPECT_EQ(storage->RemoveMultiple(2), 2); + EXPECT_EQ(storage->GetElementCount(), 0); +} + +TEST_F(SQLiteStorageTest, GetElementCount) +{ + json message = {{"key", "value"}}; + storage->Store(message); + EXPECT_EQ(storage->GetElementCount(), 1); +} + +class SQLiteStorageMultithreadedTest : public ::testing::Test +{ +protected: + std::string dbName = "testdb"; + std::string tableName1 = "test_table1"; + + void SetUp() override + { + // Clean up: Delete the SQLiteStorage instances and remove the database file + std::error_code ec; + if (std::filesystem::exists(dbName.c_str())) + { + std::filesystem::remove(dbName.c_str()); + if (ec) + { + std::cerr << "Error removing file: " << ec.message() << std::endl; + } + } + } + + void TearDown() override {} +}; + +void storeMessages(SQLiteStorage& storage, const json& messages) +{ + for (const auto& message : messages) + { + storage.Store(message); + } +} + +void retrieveMessages(SQLiteStorage& storage, int count, std::vector& retrievedMessages) +{ + retrievedMessages = storage.RetrieveMultiple(count); +} + +void removeMessages(SQLiteStorage& storage, int count) +{ + storage.RemoveMultiple(count); +} + +TEST_F(SQLiteStorageMultithreadedTest, MultithreadedStoreAndRetrieve) +{ + size_t messagesToStore = 100; + SQLiteStorage storage1(dbName, tableName1); + + json messages1 = json::array(); + json messages2 = json::array(); + for (size_t i = 0; i < messagesToStore; i++) + { + messages1.push_back({{"key" + std::to_string(i), "value" + std::to_string(i)}}); + messages2.push_back({{std::to_string(i) + "key", std::to_string(i) + "value"}}); + } + + // Create threads for storing messages + std::thread thread1(storeMessages, std::ref(storage1), messages1); + std::thread thread2(storeMessages, std::ref(storage1), messages2); + + // Join the threads to ensure they complete + thread1.join(); + thread2.join(); + + EXPECT_EQ(storage1.GetElementCount(), 2 * messagesToStore); + + std::thread thread3(removeMessages, std::ref(storage1), messagesToStore); + std::thread thread4(removeMessages, std::ref(storage1), messagesToStore); + + // Join the threads to ensure they complete + thread3.join(); + thread4.join(); + + EXPECT_EQ(storage1.GetElementCount(), 0); +} From 1f71942d139a9d4e53391adfac274a4bcffd9e0b Mon Sep 17 00:00:00 2001 From: LucioDonda Date: Wed, 31 Jul 2024 00:16:03 -0300 Subject: [PATCH 18/31] feat: three DB files approach --- src/agent/queue/include/queue.hpp | 9 ++++----- src/agent/queue/include/shared.hpp | 2 +- src/agent/queue/include/sqlitestorage.h | 2 +- src/agent/queue/src/sqlitestorage.cpp | 2 +- src/agent/queue/tests/queue_test.cpp | 16 ++++++++++++---- src/agent/queue/tests/sqlitestorage_test.cpp | 14 ++++++++++++-- 6 files changed, 31 insertions(+), 14 deletions(-) diff --git a/src/agent/queue/include/queue.hpp b/src/agent/queue/include/queue.hpp index 13560759e07..07bd8533778 100644 --- a/src/agent/queue/include/queue.hpp +++ b/src/agent/queue/include/queue.hpp @@ -180,11 +180,10 @@ class MultiTypeQueue : m_maxItems(size) { // Populate the map inside the constructor body - m_queuesMap[MessageType::STATELESS] = - std::make_unique(MessageType::STATELESS, m_maxItems, timeout); - m_queuesMap[MessageType::STATEFUL] = - std::make_unique(MessageType::STATEFUL, m_maxItems, timeout); - m_queuesMap[MessageType::COMMAND] = std::make_unique(MessageType::COMMAND, m_maxItems, timeout); + for (auto type : {STATELESS, STATEFUL, COMMAND}) + { + m_queuesMap[type] = std::make_unique(type, m_maxItems, timeout); + } } // Delete copy constructor diff --git a/src/agent/queue/include/shared.hpp b/src/agent/queue/include/shared.hpp index a89eb2c4c15..31c653e9da7 100644 --- a/src/agent/queue/include/shared.hpp +++ b/src/agent/queue/include/shared.hpp @@ -7,7 +7,7 @@ // TODO: should be moved to Config constexpr char DEFAULT_FILE_PATH[] = "/home/vagrant/FILE_"; -constexpr char DEFAULT_DB_PATH[] = "queue.db"; +constexpr char DEFAULT_DB_PATH[] = "queue-"; /** * @brief Types of messages enum diff --git a/src/agent/queue/include/sqlitestorage.h b/src/agent/queue/include/sqlitestorage.h index 839a680a3f4..fa4d4f7c0e6 100644 --- a/src/agent/queue/include/sqlitestorage.h +++ b/src/agent/queue/include/sqlitestorage.h @@ -121,7 +121,7 @@ class SQLiteStorage : public Persistence const std::string m_tableName; /// Pointer to the SQLite database connection. - std::shared_ptr m_db; + std::unique_ptr m_db; /// Mutex to ensure thread-safe operations. std::mutex m_mutex; diff --git a/src/agent/queue/src/sqlitestorage.cpp b/src/agent/queue/src/sqlitestorage.cpp index 5f32bbaed92..816ff53e256 100644 --- a/src/agent/queue/src/sqlitestorage.cpp +++ b/src/agent/queue/src/sqlitestorage.cpp @@ -15,7 +15,7 @@ SQLiteStorage::SQLiteStorage(const std::string& dbName, const std::string& tableName) : m_dbName(dbName) , m_tableName(tableName) - , m_db(make_shared(dbName, SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE)) + , m_db(make_unique(dbName + tableName + ".db", SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE)) { try { diff --git a/src/agent/queue/tests/queue_test.cpp b/src/agent/queue/tests/queue_test.cpp index f81c848cb95..225121164ee 100644 --- a/src/agent/queue/tests/queue_test.cpp +++ b/src/agent/queue/tests/queue_test.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -58,9 +59,17 @@ std::string unescape_string(const std::string& str) // TODO: Makes sense to add a full_clean method inside the persistence implementation? void cleanPersistence() { - auto filePath = DEFAULT_DB_PATH; - std::error_code ec; - std::filesystem::remove(filePath, ec); + std::string filePath = DEFAULT_DB_PATH; + for (const auto& entry : std::filesystem::directory_iterator(".")) + { + std::string fileFullPath = entry.path(); + size_t found = fileFullPath.find(filePath); + if (found != std::string::npos) + { + std::error_code ec; + std::filesystem::remove(fileFullPath, ec); + } + } } /// Test Methods @@ -219,7 +228,6 @@ TEST_F(QueueTest, SinglePushPopFullWithTimeout) // Accesing different types of queues TEST_F(QueueTest, MultithreadDifferentType) { - GTEST_SKIP(); MultiTypeQueue queue(BIG_QUEUE_QTTY); auto consumerStateLess = [&](int& count) diff --git a/src/agent/queue/tests/sqlitestorage_test.cpp b/src/agent/queue/tests/sqlitestorage_test.cpp index d79b5d7bdf0..bc69a9af7fb 100644 --- a/src/agent/queue/tests/sqlitestorage_test.cpp +++ b/src/agent/queue/tests/sqlitestorage_test.cpp @@ -14,14 +14,24 @@ using json = nlohmann::json; class SQLiteStorageTest : public ::testing::Test { protected: - std::string dbName = "testdb"; + std::string dbName = "testdb-"; std::string tableName = "test_table"; SQLiteStorage* storage; void SetUp() override { // Ensure the database file does not already exist - std::remove(dbName.c_str()); + std::string filePath = dbName; + for (const auto& entry : std::filesystem::directory_iterator(".")) + { + std::string fileFullPath = entry.path(); + size_t found = fileFullPath.find(filePath); + if (found != std::string::npos) + { + std::error_code ec; + std::filesystem::remove(fileFullPath, ec); + } + } // Create a new SQLiteStorage instance storage = new SQLiteStorage(dbName, tableName); From ca453f4586eb0264dd91a61d4b632aa7717d58a5 Mon Sep 17 00:00:00 2001 From: LucioDonda Date: Wed, 31 Jul 2024 22:45:43 -0300 Subject: [PATCH 19/31] feat: single persistence approach --- src/agent/queue/CMakeLists.txt | 3 +- src/agent/queue/include/persistence.h | 62 +++--- src/agent/queue/include/queue.hpp | 177 ++++------------- src/agent/queue/include/shared.hpp | 2 +- src/agent/queue/include/sqlitestorage.h | 25 ++- src/agent/queue/src/queue.cpp | 198 +++++++------------ src/agent/queue/src/sqlitestorage.cpp | 61 +++--- src/agent/queue/tests/CMakeLists.txt | 6 +- src/agent/queue/tests/sqlitestorage_test.cpp | 78 ++++---- 9 files changed, 233 insertions(+), 379 deletions(-) diff --git a/src/agent/queue/CMakeLists.txt b/src/agent/queue/CMakeLists.txt index d4d469e4dbf..02547f30ea8 100644 --- a/src/agent/queue/CMakeLists.txt +++ b/src/agent/queue/CMakeLists.txt @@ -15,9 +15,10 @@ add_library(queue_test find_package(SQLiteCpp REQUIRED) find_package(nlohmann_json REQUIRED) +find_package(fmt REQUIRED) target_include_directories(queue_test PRIVATE include ${SQLiteCpp_INCLUDE_DIRS}) -target_link_libraries(queue_test PRIVATE SQLiteCpp nlohmann_json::nlohmann_json) +target_link_libraries(queue_test PRIVATE SQLiteCpp nlohmann_json::nlohmann_json fmt::fmt) if(BUILD_TESTS) add_subdirectory(tests) diff --git a/src/agent/queue/include/persistence.h b/src/agent/queue/include/persistence.h index bb626239c4e..7b49f08bfbe 100644 --- a/src/agent/queue/include/persistence.h +++ b/src/agent/queue/include/persistence.h @@ -21,50 +21,56 @@ class Persistence virtual ~Persistence() = default; /** - * @brief Store a JSON message in the storage. - * - * @param message The JSON message to store. + * @brief + * + * @param message + * @param queueName */ - virtual void Store(const json& message) = 0; + virtual void Store(const json& message, const std::string& queueName) = 0; /** - * @brief Retrieve a JSON message by its ID. - * - * @param id The ID of the message to retrieve. - * @return The retrieved JSON message. + * @brief + * + * @param id + * @param queueName + * @return json */ - virtual json Retrieve(int id) = 0; + virtual json Retrieve(int id, const std::string& queueName) = 0; /** - * @brief Retrieve multiple JSON messages. - * - * @param n The number of messages to retrieve. - * @return A vector of retrieved JSON messages. + * @brief + * + * @param n + * @param queueName + * @return json */ - virtual json RetrieveMultiple(int n) = 0; + virtual json RetrieveMultiple(int n, const std::string& queueName) = 0; /** - * @brief Remove a JSON message by its ID. - * - * @param id The ID of the message to remove. - * @return The number of removed elements. + * @brief + * + * @param id + * @param queueName + * @return int */ - virtual int Remove(int id) = 0; + virtual int Remove(int id, const std::string& queueName) = 0; /** - * @brief Remove multiple JSON messages. - * - * @param n The number of messages to remove. - * @return The number of removed elements. + * @brief + * + * @param n + * @param queueName + * @return int */ - virtual int RemoveMultiple(int n) = 0; + virtual int RemoveMultiple(int n, const std::string& queueName) = 0; /** - * @brief Get the number of elements in the table. - * - * @return The number of elements in the table. + * @brief Get the Element Count object + * + * @param queueName + * @return int */ - virtual int GetElementCount() = 0; + virtual int GetElementCount(const std::string& queueName) = 0; }; #endif // PERSISTENCE_H diff --git a/src/agent/queue/include/queue.hpp b/src/agent/queue/include/queue.hpp index 07bd8533778..c12626de916 100644 --- a/src/agent/queue/include/queue.hpp +++ b/src/agent/queue/include/queue.hpp @@ -16,16 +16,6 @@ constexpr int DEFAULT_MAX = 10; constexpr int DEFAULT_TIMEOUT_S = 3; -/** - * @brief Map for transforing Message type name to string - * - */ -static const std::map MessageTypeName { - {MessageType::STATELESS, "STATELESS"}, - {MessageType::STATEFUL, "STATEFUL"}, - {MessageType::COMMAND, "COMMAND"}, -}; - // Factory class class PersistenceFactory { @@ -34,156 +24,45 @@ class PersistenceFactory { if (type == "SQLite3") { - if (args.size() != 2 || !std::any_cast(&args[0]) || !std::any_cast(&args[1])) + if (args.size() != 2 || !std::any_cast(&args[0]) || + !std::any_cast>(&args[1])) { - throw std::invalid_argument("SQLite3 requires 2 string arguments"); + throw std::invalid_argument("SQLite3 requires db name and table names as arguments"); } return std::make_unique(std::any_cast(args[0]), - std::any_cast(args[1])); + std::any_cast>(args[1])); } throw std::runtime_error("Unknown persistence type"); } }; /** - * @brief Class representing each single type queue + * @brief * */ -class PersistedQueue +class MultiTypeQueue { private: - const MessageType m_queueType; - const int m_max_size; + const std::vector m_vMessageTypeStrings {"STATELESS", "STATEFUL", "COMMAND"}; + const std::map m_mapMessageTypeName { + {MessageType::STATELESS, "STATELESS"}, + {MessageType::STATEFUL, "STATEFUL"}, + {MessageType::COMMAND, "COMMAND"}, + }; + const int m_maxItems; const std::chrono::seconds m_timeout; std::unique_ptr m_persistenceDest; std::mutex m_mtx; std::condition_variable m_cv; -public: - PersistedQueue(MessageType queueType, int max_size, int timeout) - : m_queueType(queueType) - , m_max_size(max_size) - , m_timeout(timeout) - { - m_persistenceDest = PersistenceFactory::createPersistence( - "SQLite3", {static_cast(DEFAULT_DB_PATH), MessageTypeName.at(m_queueType)}); - } - - // Delete copy constructor - PersistedQueue(const PersistedQueue&) = delete; - - // Delete copy assignment operator - PersistedQueue& operator=(const PersistedQueue&) = delete; - - // Delete move constructor - PersistedQueue(PersistedQueue&&) = delete; - - // Delete move assignment operator - PersistedQueue& operator=(PersistedQueue&&) = delete; - - ~PersistedQueue() { - // m_persistenceDest.close(); - }; - - /** - * @brief Get the Type object - * - * @return const MessageType& - */ - const MessageType& getType() const - { - return m_queueType; - } - - /** - * @brief Get the Items Available object - * - * @return int - */ - int getItemsAvailable(); - - /** - * @brief - * - * @param event - * @return true - * @return false - */ - bool insertMessage(Message event); - - /** - * @brief - * - * @param qttyMessages - * @return true - * @return false - */ - bool removeNMessages(int qttyMessages); - - /** - * @brief Get the Message object - * - * @return Message - */ - Message getMessage(); - - /** - * @brief - * - * @param n - * @return Message - */ - Message getNMessages(int n); - - /** - * @brief - * - * @return true - * @return false - */ - bool empty(); - - /** - * @brief - * - * @return true - * @return false - */ - bool isFull(); - - void waitUntilNotFull(); - - /** - * A function that waits until the queue is not full or until a stop signal is received. - * - * @param stopToken std::stop_token to check for a stop signal - * - * @throws None - */ - void waitUntilNotFullOrStoped(std::stop_token stopToken); -}; - -/** - * @brief - * - */ -class MultiTypeQueue -{ -private: - std::unordered_map> m_queuesMap; - const int m_maxItems; - std::mutex m_mapMutex; - public: // Create a vector with 3 PersistedQueue elements MultiTypeQueue(int size = DEFAULT_MAX, int timeout = DEFAULT_TIMEOUT_S) : m_maxItems(size) + , m_timeout(timeout) { - // Populate the map inside the constructor body - for (auto type : {STATELESS, STATEFUL, COMMAND}) - { - m_queuesMap[type] = std::make_unique(type, m_maxItems, timeout); - } + m_persistenceDest = PersistenceFactory::createPersistence( + "SQLite3", {static_cast(DEFAULT_DB_PATH), m_vMessageTypeStrings}); } // Delete copy constructor @@ -222,6 +101,15 @@ class MultiTypeQueue */ Message getLastMessage(MessageType type); + /** + * @brief Returns N messages from a queue + * + * @param type Of the queue to be used as source + * @param messageQuantity quantity of messages to return + * @return Message Json data othe messages fetched + */ + Message getNMessages(MessageType type, int messageQuantity); + /** * @brief deletes a message from a queue * @@ -242,14 +130,23 @@ class MultiTypeQueue bool popNMessages(MessageType type, int messageQuantity); /** - * @brief + * @brief Checks emptyness of a queue * * @param type - * @return true - * @return false + * @return true when queue empty + * @return false otherwise */ bool isEmptyByType(MessageType type); + /** + * @brief Checks fullness of a queue + * + * @param type + * @return true when queue is full + * @return false otherwise + */ + bool isFullByType(MessageType type); + /** * @brief Get the Items By Type object * diff --git a/src/agent/queue/include/shared.hpp b/src/agent/queue/include/shared.hpp index 31c653e9da7..a89eb2c4c15 100644 --- a/src/agent/queue/include/shared.hpp +++ b/src/agent/queue/include/shared.hpp @@ -7,7 +7,7 @@ // TODO: should be moved to Config constexpr char DEFAULT_FILE_PATH[] = "/home/vagrant/FILE_"; -constexpr char DEFAULT_DB_PATH[] = "queue-"; +constexpr char DEFAULT_DB_PATH[] = "queue.db"; /** * @brief Types of messages enum diff --git a/src/agent/queue/include/sqlitestorage.h b/src/agent/queue/include/sqlitestorage.h index fa4d4f7c0e6..f86a7ac23f1 100644 --- a/src/agent/queue/include/sqlitestorage.h +++ b/src/agent/queue/include/sqlitestorage.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -23,13 +24,8 @@ class SQLiteStorage : public Persistence { public: - /** - * @brief Constructor. - * - * @param dbName The name of the SQLite database file. - * @param tableName The name of the table to use for storing messages. - */ - SQLiteStorage(const std::string& dbName, const std::string& tableName); + + SQLiteStorage(const std::string& dbName, const std::vector tableName); // Delete copy constructor SQLiteStorage(const SQLiteStorage&) = delete; @@ -53,7 +49,7 @@ class SQLiteStorage : public Persistence * * @param message The JSON message to store. */ - void Store(const json& message) override; + void Store(const json& message, const std::string& tableName) override; /** * @brief Retrieve a JSON message by its ID. @@ -61,7 +57,7 @@ class SQLiteStorage : public Persistence * @param id The ID of the message to retrieve. * @return The retrieved JSON message. */ - json Retrieve(int id) override; + json Retrieve(int id, const std::string& tableName) override; /** * @brief Retrieve multiple JSON messages. @@ -69,7 +65,7 @@ class SQLiteStorage : public Persistence * @param n The number of messages to retrieve. * @return A vector of retrieved JSON messages. */ - json RetrieveMultiple(int n) override; + json RetrieveMultiple(int n, const std::string& tableName) override; /** * @brief Remove a JSON message by its ID. @@ -77,7 +73,7 @@ class SQLiteStorage : public Persistence * @param id The ID of the message to remove. * @return The number of removed elements. */ - int Remove(int id) override; + int Remove(int id, const std::string& tableName) override; /** * @brief Remove multiple JSON messages. @@ -85,14 +81,14 @@ class SQLiteStorage : public Persistence * @param n The number of messages to remove. * @return The number of removed elements. */ - int RemoveMultiple(int n) override; + int RemoveMultiple(int n, const std::string& tableName) override; /** * @brief Get the number of elements in the table. * * @return The number of elements in the table. */ - int GetElementCount() override; + int GetElementCount(const std::string& tableName) override; private: /** @@ -100,7 +96,7 @@ class SQLiteStorage : public Persistence * * This method creates the table if it does not already exist. */ - void InitializeTable(); + void InitializeTable(const std::string& tableName); /** * @brief Private method for waiting for database access. @@ -131,6 +127,7 @@ class SQLiteStorage : public Persistence // TODO: should it be atomic? bool m_db_in_use = false; + }; #endif // SQLITE_STORAGE_H diff --git a/src/agent/queue/src/queue.cpp b/src/agent/queue/src/queue.cpp index ee8b82c90d4..574ac028726 100644 --- a/src/agent/queue/src/queue.cpp +++ b/src/agent/queue/src/queue.cpp @@ -12,113 +12,20 @@ #include "queue.hpp" -int PersistedQueue::getItemsAvailable() -{ - std::unique_lock lock(m_mtx); - return m_persistenceDest->GetElementCount(); -} - -bool PersistedQueue::insertMessage(Message event) -{ - std::unique_lock lock(m_mtx); - bool success = false; - size_t spaceAvailable = - (m_max_size > m_persistenceDest->GetElementCount()) ? m_max_size - m_persistenceDest->GetElementCount() : 0; - if (spaceAvailable) - { - // TODO: handle response - success = true; - auto messageData = event.data; - if (messageData.is_array()) - { - if (messageData.size() <= spaceAvailable) - { - for (const auto& singleMessageData : messageData) - { - m_persistenceDest->Store(singleMessageData); - m_cv.notify_all(); - } - } - else - { - success = false; - } - } - else - { - m_persistenceDest->Store(event.data); - m_cv.notify_all(); - } - } - return success; -}; - -bool PersistedQueue::removeNMessages(int qttyMessages) -{ - std::unique_lock lock(m_mtx); - auto linesRemoved = m_persistenceDest->RemoveMultiple(qttyMessages); - m_cv.notify_all(); - return linesRemoved != 0; -}; - -Message PersistedQueue::getMessage() -{ - std::unique_lock lock(m_mtx); - auto messageData = m_persistenceDest->RetrieveMultiple(1); - return Message(m_queueType, messageData); -}; - -Message PersistedQueue::getNMessages(int n) -{ - std::unique_lock lock(m_mtx); - auto messageData = m_persistenceDest->RetrieveMultiple(n); - return Message(m_queueType, messageData); -}; - -bool PersistedQueue::empty() -{ - const auto items = getItemsAvailable(); - return items == 0; -} - -bool PersistedQueue::isFull() -{ - std::unique_lock lock(m_mtx); - return m_persistenceDest->GetElementCount() == m_max_size; -} - -void PersistedQueue::waitUntilNotFull() -{ - std::unique_lock lock(m_mtx); - m_cv.wait_for(lock, m_timeout, [this] { return m_persistenceDest->GetElementCount() < m_max_size; }); -} - -void PersistedQueue::waitUntilNotFullOrStoped(std::stop_token stopToken) -{ - std::unique_lock lock(m_mtx); - m_cv.wait(lock, - [this, stopToken] - { - bool menor = (m_persistenceDest->GetElementCount() < m_max_size); - bool stopped = (stopToken.stop_possible() && stopToken.stop_requested()); - return menor || stopped; - }); -} - bool MultiTypeQueue::timeoutPush(Message message, bool shouldWait) { bool result = false; - std::unique_lock mapLock(m_mapMutex); - if (m_queuesMap.contains(message.type)) + if (m_mapMessageTypeName.contains(message.type)) { - auto& queue = m_queuesMap[message.type]; - mapLock.unlock(); + auto sMessageType = m_mapMessageTypeName.at(message.type); // Wait until the queue is not full if (shouldWait) { - queue->waitUntilNotFull(); + std::unique_lock lock(m_mtx); + m_cv.wait_for( + lock, m_timeout, [&, this] { return m_persistenceDest->GetElementCount(sMessageType) < m_maxItems; }); } // FIXME // else @@ -126,11 +33,33 @@ bool MultiTypeQueue::timeoutPush(Message message, bool shouldWait) // std::cout << "Can failed because os full queue" << std::endl; // } - // Insert the message - result = queue->insertMessage(message); - if (!result) + auto storedMessages = m_persistenceDest->GetElementCount(sMessageType); + size_t spaceAvailable = (m_maxItems > storedMessages) ? m_maxItems - storedMessages : 0; + if (spaceAvailable) { - std::cout << "Failed to insert message: " << message.data << std::endl; + // TODO: handle response + result = true; + auto messageData = message.data; + if (messageData.is_array()) + { + if (messageData.size() <= spaceAvailable) + { + for (const auto& singleMessageData : messageData) + { + m_persistenceDest->Store(singleMessageData, sMessageType); + m_cv.notify_all(); + } + } + else + { + result = false; + } + } + else + { + m_persistenceDest->Store(message.data, m_mapMessageTypeName.at(message.type)); + m_cv.notify_all(); + } } } else @@ -151,12 +80,25 @@ void MultiTypeQueue::timeoutPush(std::vector messages) Message MultiTypeQueue::getLastMessage(MessageType type) { Message result(type, {}); - std::unique_lock mapLock(m_mapMutex); - if (m_queuesMap.contains(type)) + // std::unique_lock mapLock(m_mapMutex); + if (m_mapMessageTypeName.contains(type)) { - auto& queue = m_queuesMap[type]; - mapLock.unlock(); - result = queue->getMessage(); + result.data = m_persistenceDest->RetrieveMultiple(1, m_mapMessageTypeName.at(type)); + } + else + { + // TODO: error / logging handling !!! + std::cout << "error didn't find the queue" << std::endl; + } + return result; +} + +Message MultiTypeQueue::getNMessages(MessageType type, int messageQuantity) +{ + Message result(type, {}); + if (m_mapMessageTypeName.contains(type)) + { + result.data = m_persistenceDest->RetrieveMultiple(messageQuantity, m_mapMessageTypeName.at(type)); } else { @@ -169,13 +111,10 @@ Message MultiTypeQueue::getLastMessage(MessageType type) bool MultiTypeQueue::popLastMessage(MessageType type) { bool result = false; - std::unique_lock mapLock(m_mapMutex); - if (m_queuesMap.contains(type)) + if (m_mapMessageTypeName.contains(type)) { - auto& queue = m_queuesMap[type]; - mapLock.unlock(); // Handle return value - result = queue->removeNMessages(1); + result = m_persistenceDest->RemoveMultiple(1, m_mapMessageTypeName.at(type)); } else { @@ -188,12 +127,9 @@ bool MultiTypeQueue::popLastMessage(MessageType type) bool MultiTypeQueue::popNMessages(MessageType type, int messageQuantity) { bool result = false; - std::unique_lock mapLock(m_mapMutex); - if (m_queuesMap.contains(type)) + if (m_mapMessageTypeName.contains(type)) { - auto& queue = m_queuesMap[type]; - mapLock.unlock(); - result = queue->removeNMessages(messageQuantity); + result = m_persistenceDest->RemoveMultiple(messageQuantity, m_mapMessageTypeName.at(type)); } else { @@ -205,12 +141,23 @@ bool MultiTypeQueue::popNMessages(MessageType type, int messageQuantity) bool MultiTypeQueue::isEmptyByType(MessageType type) { - std::unique_lock mapLock(m_mapMutex); - if (m_queuesMap.contains(type)) + if (m_mapMessageTypeName.contains(type)) + { + return m_persistenceDest->GetElementCount(m_mapMessageTypeName.at(type)) == 0; + } + else + { + // TODO: error / logging handling !!! + std::cout << "error didn't find the queue" << std::endl; + } + return false; +} + +bool MultiTypeQueue::isFullByType(MessageType type) +{ + if (m_mapMessageTypeName.contains(type)) { - auto& queue = m_queuesMap[type]; - mapLock.unlock(); - return queue->empty(); + return m_persistenceDest->GetElementCount(m_mapMessageTypeName.at(type)) == m_maxItems; } else { @@ -222,12 +169,9 @@ bool MultiTypeQueue::isEmptyByType(MessageType type) int MultiTypeQueue::getItemsByType(MessageType type) { - std::unique_lock mapLock(m_mapMutex); - if (m_queuesMap.contains(type)) + if (m_mapMessageTypeName.contains(type)) { - auto& queue = m_queuesMap[type]; - mapLock.unlock(); - return queue->getItemsAvailable(); + return m_persistenceDest->GetElementCount(m_mapMessageTypeName.at(type)); } else { diff --git a/src/agent/queue/src/sqlitestorage.cpp b/src/agent/queue/src/sqlitestorage.cpp index 816ff53e256..c9b7de94138 100644 --- a/src/agent/queue/src/sqlitestorage.cpp +++ b/src/agent/queue/src/sqlitestorage.cpp @@ -1,27 +1,21 @@ #include "sqlitestorage.h" #include +// TODO: only in gcc13, does it worth it? #include +#include #include -// TODO: replace define for constexpr and do only use memebers on runtime -#define CREATE_TABLE_QUERY "CREATE TABLE IF NOT EXISTS " + m_tableName + " (message TEXT NOT NULL);" -#define INSERT_QUERY "INSERT INTO " + m_tableName + " (message) VALUES (?);" -#define SELECT_QUERY "SELECT message FROM " + m_tableName + " WHERE rowid = ?;" -#define SELECT_MULTIPLE_QUERY "SELECT message FROM " + m_tableName + " ORDER BY rowid ASC LIMIT ?;" -#define DELETE_QUERY "DELETE FROM " + m_tableName + " WHERE rowid = ?;" -#define DELETE_MULTIPLE_QUERY \ - "DELETE FROM " + m_tableName + " WHERE rowid IN (SELECT rowid FROM " + m_tableName + " ORDER BY rowid ASC LIMIT ?);" -#define COUNT_QUERY "SELECT COUNT(*) FROM " + m_tableName + ";" - -SQLiteStorage::SQLiteStorage(const std::string& dbName, const std::string& tableName) +SQLiteStorage::SQLiteStorage(const std::string& dbName, const std::vector tableNames) : m_dbName(dbName) - , m_tableName(tableName) - , m_db(make_unique(dbName + tableName + ".db", SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE)) + , m_db(make_unique(dbName, SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE)) { try { // Open the database in WAL mode m_db->exec("PRAGMA journal_mode=WAL;"); - InitializeTable(); + for (auto table : tableNames) + { + InitializeTable(table); + } } catch (const std::exception& e) { @@ -32,12 +26,14 @@ SQLiteStorage::SQLiteStorage(const std::string& dbName, const std::string& table SQLiteStorage::~SQLiteStorage() {} -void SQLiteStorage::InitializeTable() +void SQLiteStorage::InitializeTable(const std::string& tableName) { + // TODO: all queries should be in the same place. + constexpr std::string_view CREATE_TABLE_QUERY {"CREATE TABLE IF NOT EXISTS {} (message TEXT NOT NULL);"}; + auto createTableQuery = fmt::format(CREATE_TABLE_QUERY, tableName); std::lock_guard lock(m_mutex); try { - std::string createTableQuery = CREATE_TABLE_QUERY; m_db->exec(createTableQuery); } catch (const std::exception& e) @@ -60,11 +56,11 @@ void SQLiteStorage::releaseDatabaseAccess() m_cv.notify_one(); } -void SQLiteStorage::Store(const json& message) +void SQLiteStorage::Store(const json& message, const std::string& tableName) { waitForDatabaseAccess(); - - std::string insertQuery = INSERT_QUERY; + constexpr std::string_view INSERT_QUERY {"INSERT INTO {} (message) VALUES (?);"}; + std::string insertQuery = fmt::format(INSERT_QUERY, tableName); SQLite::Statement query = SQLite::Statement(*m_db, insertQuery); if (message.is_array()) @@ -96,12 +92,14 @@ void SQLiteStorage::Store(const json& message) releaseDatabaseAccess(); } -json SQLiteStorage::Retrieve(int id) +json SQLiteStorage::Retrieve(int id, const std::string& tableName) { try { - std::string selectQuery = SELECT_QUERY; + constexpr std::string_view SELECT_QUERY {"SELECT message FROM {} WHERE rowid = ?;"}; + std::string selectQuery = fmt::format(SELECT_QUERY, tableName); SQLite::Statement query(*m_db, selectQuery); + // TODO: delte id? query.bind(1, id); json message; if (query.executeStep()) @@ -121,11 +119,12 @@ json SQLiteStorage::Retrieve(int id) } } -json SQLiteStorage::RetrieveMultiple(int n) +json SQLiteStorage::RetrieveMultiple(int n, const std::string& tableName) { try { - std::string selectQuery = SELECT_MULTIPLE_QUERY; + constexpr std::string_view SELECT_MULTIPLE_QUERY {"SELECT message FROM {} ORDER BY rowid ASC LIMIT ?;"}; + std::string selectQuery = fmt::format(SELECT_MULTIPLE_QUERY, tableName); SQLite::Statement query(*m_db, selectQuery); query.bind(1, n); json messages = json::array(); @@ -143,11 +142,12 @@ json SQLiteStorage::RetrieveMultiple(int n) } } -int SQLiteStorage::Remove(int id) +int SQLiteStorage::Remove(int id, const std::string& tableName) { + constexpr std::string_view DELETE_QUERY {"DELETE FROM {} WHERE rowid = ?;"}; try { - std::string deleteQuery = DELETE_QUERY; + std::string deleteQuery = fmt::format(DELETE_QUERY, tableName); SQLite::Statement query(*m_db, deleteQuery); query.bind(1, id); SQLite::Transaction transaction(*m_db); @@ -162,13 +162,15 @@ int SQLiteStorage::Remove(int id) } } -int SQLiteStorage::RemoveMultiple(int n) +int SQLiteStorage::RemoveMultiple(int n, const std::string& tableName) { + constexpr std::string_view DELETE_MULTIPLE_QUERY { + "DELETE FROM {} WHERE rowid IN (SELECT rowid FROM {} ORDER BY rowid ASC LIMIT ?);"}; try { waitForDatabaseAccess(); SQLite::Transaction transaction(*m_db); - std::string deleteQuery = DELETE_MULTIPLE_QUERY; + std::string deleteQuery = fmt::format(DELETE_MULTIPLE_QUERY, tableName, tableName); SQLite::Statement query(*m_db, deleteQuery); query.bind(1, n); query.exec(); @@ -183,11 +185,12 @@ int SQLiteStorage::RemoveMultiple(int n) } } -int SQLiteStorage::GetElementCount() +int SQLiteStorage::GetElementCount(const std::string& tableName) { + constexpr std::string_view COUNT_QUERY {"SELECT COUNT(*) FROM {}"}; try { - std::string countQuery = COUNT_QUERY; + std::string countQuery = fmt::format(COUNT_QUERY, tableName); SQLite::Statement query(*m_db, countQuery); int count = 0; if (query.executeStep()) diff --git a/src/agent/queue/tests/CMakeLists.txt b/src/agent/queue/tests/CMakeLists.txt index 54d318d36a2..1a45cf81790 100644 --- a/src/agent/queue/tests/CMakeLists.txt +++ b/src/agent/queue/tests/CMakeLists.txt @@ -19,7 +19,8 @@ target_link_libraries(test_queue ${GTEST_MAIN_LIBRARIES} queue_test SQLiteCpp - nlohmann_json::nlohmann_json) + nlohmann_json::nlohmann_json + fmt::fmt) # Create a test executable for SQLiteStorage tests add_executable(test_sqlitestorage @@ -30,4 +31,5 @@ target_link_libraries(test_sqlitestorage ${GTEST_LIBRARIES} ${GTEST_MAIN_LIBRARIES} SQLiteCpp - nlohmann_json::nlohmann_json) \ No newline at end of file + nlohmann_json::nlohmann_json + fmt::fmt) \ No newline at end of file diff --git a/src/agent/queue/tests/sqlitestorage_test.cpp b/src/agent/queue/tests/sqlitestorage_test.cpp index bc69a9af7fb..12258018b47 100644 --- a/src/agent/queue/tests/sqlitestorage_test.cpp +++ b/src/agent/queue/tests/sqlitestorage_test.cpp @@ -14,8 +14,9 @@ using json = nlohmann::json; class SQLiteStorageTest : public ::testing::Test { protected: - std::string dbName = "testdb-"; - std::string tableName = "test_table"; + const std::string dbName = "testdb.db"; + const std::string tableName = "test_table"; + const std::vector m_vMessageTypeStrings {"test_table", "test_table2"}; SQLiteStorage* storage; void SetUp() override @@ -33,8 +34,7 @@ class SQLiteStorageTest : public ::testing::Test } } - // Create a new SQLiteStorage instance - storage = new SQLiteStorage(dbName, tableName); + storage = new SQLiteStorage(dbName, m_vMessageTypeStrings); } void TearDown() override @@ -56,10 +56,10 @@ class SQLiteStorageTest : public ::testing::Test TEST_F(SQLiteStorageTest, StoreSingleMessage) { json message = {{"key", "value"}}; - EXPECT_NO_THROW(storage->Store(message)); - EXPECT_EQ(storage->GetElementCount(), 1); - EXPECT_NO_THROW(storage->Store(message)); - EXPECT_EQ(storage->GetElementCount(), 2); + EXPECT_NO_THROW(storage->Store(message, tableName)); + EXPECT_EQ(storage->GetElementCount(tableName), 1); + EXPECT_NO_THROW(storage->Store(message, tableName)); + EXPECT_EQ(storage->GetElementCount(tableName), 2); } TEST_F(SQLiteStorageTest, StoreMultipleMessages) @@ -67,15 +67,15 @@ TEST_F(SQLiteStorageTest, StoreMultipleMessages) json messages = json::array(); messages.push_back({{"key", "value1"}}); messages.push_back({{"key", "value2"}}); - EXPECT_NO_THROW(storage->Store(messages)); - EXPECT_EQ(storage->GetElementCount(), 2); + EXPECT_NO_THROW(storage->Store(messages, tableName)); + EXPECT_EQ(storage->GetElementCount(tableName), 2); } TEST_F(SQLiteStorageTest, RetrieveMessage) { json message = {{"key", "value"}}; - storage->Store(message); - json retrievedMessage = storage->Retrieve(1); + storage->Store(message, tableName); + json retrievedMessage = storage->Retrieve(1, tableName); EXPECT_EQ(retrievedMessage["key"], "value"); } @@ -84,17 +84,17 @@ TEST_F(SQLiteStorageTest, RetrieveMultipleMessages) json messages = json::array(); messages.push_back({{"key", "value1"}}); messages.push_back({{"key", "value2"}}); - storage->Store(messages); - json retrievedMessages = storage->RetrieveMultiple(2); + storage->Store(messages, tableName); + json retrievedMessages = storage->RetrieveMultiple(2, tableName); EXPECT_EQ(retrievedMessages.size(), 2); } TEST_F(SQLiteStorageTest, RemoveMessage) { json message = {{"key", "value"}}; - storage->Store(message); - EXPECT_EQ(storage->Remove(1), 1); - EXPECT_EQ(storage->GetElementCount(), 0); + storage->Store(message, tableName); + EXPECT_EQ(storage->Remove(1, tableName), 1); + EXPECT_EQ(storage->GetElementCount(tableName), 0); } TEST_F(SQLiteStorageTest, RemoveMultipleMessages) @@ -102,23 +102,23 @@ TEST_F(SQLiteStorageTest, RemoveMultipleMessages) json messages = json::array(); messages.push_back({{"key", "value1"}}); messages.push_back({{"key", "value2"}}); - storage->Store(messages); - EXPECT_EQ(storage->RemoveMultiple(2), 2); - EXPECT_EQ(storage->GetElementCount(), 0); + storage->Store(messages, tableName); + EXPECT_EQ(storage->RemoveMultiple(2, tableName), 2); + EXPECT_EQ(storage->GetElementCount(tableName), 0); } TEST_F(SQLiteStorageTest, GetElementCount) { json message = {{"key", "value"}}; - storage->Store(message); - EXPECT_EQ(storage->GetElementCount(), 1); + storage->Store(message, tableName); + EXPECT_EQ(storage->GetElementCount(tableName), 1); } class SQLiteStorageMultithreadedTest : public ::testing::Test { protected: - std::string dbName = "testdb"; - std::string tableName1 = "test_table1"; + const std::string dbName = "testdb"; + const std::vector m_vMessageTypeStrings {"test_table1", "test_table2", "test_table3"}; void SetUp() override { @@ -137,28 +137,32 @@ class SQLiteStorageMultithreadedTest : public ::testing::Test void TearDown() override {} }; -void storeMessages(SQLiteStorage& storage, const json& messages) +void storeMessages(SQLiteStorage& storage, const json& messages, const std::string& tableName) { for (const auto& message : messages) { - storage.Store(message); + storage.Store(message, tableName); } } -void retrieveMessages(SQLiteStorage& storage, int count, std::vector& retrievedMessages) +void retrieveMessages(SQLiteStorage& storage, + int count, + std::vector& retrievedMessages, + const std::string& tableName) { - retrievedMessages = storage.RetrieveMultiple(count); + retrievedMessages = storage.RetrieveMultiple(count, tableName); } -void removeMessages(SQLiteStorage& storage, int count) +void removeMessages(SQLiteStorage& storage, int count, const std::string& tableName) { - storage.RemoveMultiple(count); + storage.RemoveMultiple(count, tableName); } TEST_F(SQLiteStorageMultithreadedTest, MultithreadedStoreAndRetrieve) { size_t messagesToStore = 100; - SQLiteStorage storage1(dbName, tableName1); + + SQLiteStorage storage1(dbName, m_vMessageTypeStrings); json messages1 = json::array(); json messages2 = json::array(); @@ -169,21 +173,21 @@ TEST_F(SQLiteStorageMultithreadedTest, MultithreadedStoreAndRetrieve) } // Create threads for storing messages - std::thread thread1(storeMessages, std::ref(storage1), messages1); - std::thread thread2(storeMessages, std::ref(storage1), messages2); + std::thread thread1(storeMessages, std::ref(storage1), messages1, m_vMessageTypeStrings[0]); + std::thread thread2(storeMessages, std::ref(storage1), messages2, m_vMessageTypeStrings[0]); // Join the threads to ensure they complete thread1.join(); thread2.join(); - EXPECT_EQ(storage1.GetElementCount(), 2 * messagesToStore); + EXPECT_EQ(storage1.GetElementCount(m_vMessageTypeStrings[0]), 2 * messagesToStore); - std::thread thread3(removeMessages, std::ref(storage1), messagesToStore); - std::thread thread4(removeMessages, std::ref(storage1), messagesToStore); + std::thread thread3(removeMessages, std::ref(storage1), messagesToStore, m_vMessageTypeStrings[0]); + std::thread thread4(removeMessages, std::ref(storage1), messagesToStore, m_vMessageTypeStrings[0]); // Join the threads to ensure they complete thread3.join(); thread4.join(); - EXPECT_EQ(storage1.GetElementCount(), 0); + EXPECT_EQ(storage1.GetElementCount(m_vMessageTypeStrings[0]), 0); } From a4a5b2b1a220b18e096334b744b24928bfafd08a Mon Sep 17 00:00:00 2001 From: LucioDonda Date: Tue, 6 Aug 2024 19:42:02 -0300 Subject: [PATCH 20/31] feat: addition of module field, pending tests --- src/agent/queue/CMakeLists.txt | 16 ++--- src/agent/queue/include/persistence.h | 58 +++++++++--------- src/agent/queue/include/queue.hpp | 15 +++-- src/agent/queue/include/shared.hpp | 4 +- src/agent/queue/include/sqlitestorage.h | 7 +-- src/agent/queue/src/main.cpp | 2 +- src/agent/queue/src/queue.cpp | 23 ++++--- src/agent/queue/src/sqlitestorage.cpp | 80 ++++++++++++++++++------- src/agent/queue/tests/CMakeLists.txt | 2 +- src/agent/queue/tests/queue_test.cpp | 39 +++++++++--- src/vcpkg.json | 3 +- 11 files changed, 160 insertions(+), 89 deletions(-) diff --git a/src/agent/queue/CMakeLists.txt b/src/agent/queue/CMakeLists.txt index 02547f30ea8..783b9cc5fd2 100644 --- a/src/agent/queue/CMakeLists.txt +++ b/src/agent/queue/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.22) -project(queue_test LANGUAGES CXX) +project(queue LANGUAGES CXX) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) @@ -8,17 +8,17 @@ set(CMAKE_CXX_FLAGS "-Wall -Wextra -Wunused -pthread") include_directories(${CMAKE_SOURCE_DIR}/include) -add_library(queue_test - src/sqlitestorage.cpp - src/queue.cpp -) - find_package(SQLiteCpp REQUIRED) find_package(nlohmann_json REQUIRED) find_package(fmt REQUIRED) -target_include_directories(queue_test PRIVATE include ${SQLiteCpp_INCLUDE_DIRS}) -target_link_libraries(queue_test PRIVATE SQLiteCpp nlohmann_json::nlohmann_json fmt::fmt) +add_library(queue + src/sqlitestorage.cpp + src/queue.cpp +) + +target_include_directories(queue PRIVATE include ${SQLiteCpp_INCLUDE_DIRS}) +target_link_libraries(queue PRIVATE SQLiteCpp nlohmann_json::nlohmann_json fmt::fmt) if(BUILD_TESTS) add_subdirectory(tests) diff --git a/src/agent/queue/include/persistence.h b/src/agent/queue/include/persistence.h index 7b49f08bfbe..468086d8689 100644 --- a/src/agent/queue/include/persistence.h +++ b/src/agent/queue/include/persistence.h @@ -21,54 +21,54 @@ class Persistence virtual ~Persistence() = default; /** - * @brief - * - * @param message - * @param queueName + * @brief + * + * @param message + * @param queueName */ - virtual void Store(const json& message, const std::string& queueName) = 0; + virtual void Store(const json& message, const std::string& tableName, const std::string& moduleName = "") = 0; /** - * @brief - * - * @param id - * @param queueName - * @return json + * @brief + * + * @param id + * @param queueName + * @return json */ virtual json Retrieve(int id, const std::string& queueName) = 0; /** - * @brief - * - * @param n - * @param queueName - * @return json + * @brief + * + * @param n + * @param queueName + * @return json */ - virtual json RetrieveMultiple(int n, const std::string& queueName) = 0; + virtual json RetrieveMultiple(int n, const std::string& queueName, const std::string& moduleName = "") = 0; /** - * @brief - * - * @param id - * @param queueName - * @return int + * @brief + * + * @param id + * @param queueName + * @return int */ virtual int Remove(int id, const std::string& queueName) = 0; /** - * @brief - * - * @param n - * @param queueName - * @return int + * @brief + * + * @param n + * @param queueName + * @return int */ virtual int RemoveMultiple(int n, const std::string& queueName) = 0; /** * @brief Get the Element Count object - * - * @param queueName - * @return int + * + * @param queueName + * @return int */ virtual int GetElementCount(const std::string& queueName) = 0; }; diff --git a/src/agent/queue/include/queue.hpp b/src/agent/queue/include/queue.hpp index c12626de916..d6b4a009b30 100644 --- a/src/agent/queue/include/queue.hpp +++ b/src/agent/queue/include/queue.hpp @@ -61,8 +61,15 @@ class MultiTypeQueue : m_maxItems(size) , m_timeout(timeout) { - m_persistenceDest = PersistenceFactory::createPersistence( - "SQLite3", {static_cast(DEFAULT_DB_PATH), m_vMessageTypeStrings}); + try + { + m_persistenceDest = PersistenceFactory::createPersistence( + "SQLite3", {static_cast(DEFAULT_DB_PATH), m_vMessageTypeStrings}); + } + catch (const std::exception& e) + { + std::cerr << "Error creating persistence: " << e.what() << '\n'; + } } // Delete copy constructor @@ -99,7 +106,7 @@ class MultiTypeQueue * @param type * @return Message */ - Message getLastMessage(MessageType type); + Message getLastMessage(MessageType type, const std::string module = ""); /** * @brief Returns N messages from a queue @@ -108,7 +115,7 @@ class MultiTypeQueue * @param messageQuantity quantity of messages to return * @return Message Json data othe messages fetched */ - Message getNMessages(MessageType type, int messageQuantity); + Message getNMessages(MessageType type, int messageQuantity, const std::string moduleName = ""); /** * @brief deletes a message from a queue diff --git a/src/agent/queue/include/shared.hpp b/src/agent/queue/include/shared.hpp index a89eb2c4c15..0561f0db8ea 100644 --- a/src/agent/queue/include/shared.hpp +++ b/src/agent/queue/include/shared.hpp @@ -29,9 +29,11 @@ class Message public: MessageType type; nlohmann::json data; - Message(MessageType t, nlohmann::json d) + std::string moduleName; + Message(MessageType t, nlohmann::json d, std::string mN = "") : type(t) , data(d) + , moduleName(mN) { } }; diff --git a/src/agent/queue/include/sqlitestorage.h b/src/agent/queue/include/sqlitestorage.h index f86a7ac23f1..572eaff84dd 100644 --- a/src/agent/queue/include/sqlitestorage.h +++ b/src/agent/queue/include/sqlitestorage.h @@ -24,7 +24,6 @@ class SQLiteStorage : public Persistence { public: - SQLiteStorage(const std::string& dbName, const std::vector tableName); // Delete copy constructor @@ -49,8 +48,9 @@ class SQLiteStorage : public Persistence * * @param message The JSON message to store. */ - void Store(const json& message, const std::string& tableName) override; + void Store(const json& message, const std::string& tableName, const std::string& moduleName = "") override; + // TODO: pending tests! /** * @brief Retrieve a JSON message by its ID. * @@ -65,7 +65,7 @@ class SQLiteStorage : public Persistence * @param n The number of messages to retrieve. * @return A vector of retrieved JSON messages. */ - json RetrieveMultiple(int n, const std::string& tableName) override; + json RetrieveMultiple(int n, const std::string& tableName, const std::string& moduleName = "") override; /** * @brief Remove a JSON message by its ID. @@ -127,7 +127,6 @@ class SQLiteStorage : public Persistence // TODO: should it be atomic? bool m_db_in_use = false; - }; #endif // SQLITE_STORAGE_H diff --git a/src/agent/queue/src/main.cpp b/src/agent/queue/src/main.cpp index ae8983c9baa..398d4e93ecb 100644 --- a/src/agent/queue/src/main.cpp +++ b/src/agent/queue/src/main.cpp @@ -6,7 +6,7 @@ int main() { - MultiTypeQueue queue(10); + MultiTypeQueue queue(100); return 0; } diff --git a/src/agent/queue/src/queue.cpp b/src/agent/queue/src/queue.cpp index 574ac028726..2d11337c31e 100644 --- a/src/agent/queue/src/queue.cpp +++ b/src/agent/queue/src/queue.cpp @@ -27,11 +27,6 @@ bool MultiTypeQueue::timeoutPush(Message message, bool shouldWait) m_cv.wait_for( lock, m_timeout, [&, this] { return m_persistenceDest->GetElementCount(sMessageType) < m_maxItems; }); } - // FIXME - // else - // { - // std::cout << "Can failed because os full queue" << std::endl; - // } auto storedMessages = m_persistenceDest->GetElementCount(sMessageType); size_t spaceAvailable = (m_maxItems > storedMessages) ? m_maxItems - storedMessages : 0; @@ -46,7 +41,7 @@ bool MultiTypeQueue::timeoutPush(Message message, bool shouldWait) { for (const auto& singleMessageData : messageData) { - m_persistenceDest->Store(singleMessageData, sMessageType); + m_persistenceDest->Store(singleMessageData, sMessageType, message.moduleName); m_cv.notify_all(); } } @@ -57,7 +52,7 @@ bool MultiTypeQueue::timeoutPush(Message message, bool shouldWait) } else { - m_persistenceDest->Store(message.data, m_mapMessageTypeName.at(message.type)); + m_persistenceDest->Store(message.data, m_mapMessageTypeName.at(message.type), message.moduleName); m_cv.notify_all(); } } @@ -77,13 +72,17 @@ void MultiTypeQueue::timeoutPush(std::vector messages) } } -Message MultiTypeQueue::getLastMessage(MessageType type) +Message MultiTypeQueue::getLastMessage(MessageType type, const std::string moduleName) { - Message result(type, {}); + Message result(type, "{}"_json, moduleName); // std::unique_lock mapLock(m_mapMutex); if (m_mapMessageTypeName.contains(type)) { - result.data = m_persistenceDest->RetrieveMultiple(1, m_mapMessageTypeName.at(type)); + auto resultData = m_persistenceDest->RetrieveMultiple(1, m_mapMessageTypeName.at(type), moduleName); + if (!resultData.empty()) + { + result.data = resultData; + } } else { @@ -93,12 +92,12 @@ Message MultiTypeQueue::getLastMessage(MessageType type) return result; } -Message MultiTypeQueue::getNMessages(MessageType type, int messageQuantity) +Message MultiTypeQueue::getNMessages(MessageType type, int messageQuantity, const std::string moduleName) { Message result(type, {}); if (m_mapMessageTypeName.contains(type)) { - result.data = m_persistenceDest->RetrieveMultiple(messageQuantity, m_mapMessageTypeName.at(type)); + result.data = m_persistenceDest->RetrieveMultiple(messageQuantity, m_mapMessageTypeName.at(type), moduleName); } else { diff --git a/src/agent/queue/src/sqlitestorage.cpp b/src/agent/queue/src/sqlitestorage.cpp index c9b7de94138..dc17628a2a6 100644 --- a/src/agent/queue/src/sqlitestorage.cpp +++ b/src/agent/queue/src/sqlitestorage.cpp @@ -29,7 +29,8 @@ SQLiteStorage::~SQLiteStorage() {} void SQLiteStorage::InitializeTable(const std::string& tableName) { // TODO: all queries should be in the same place. - constexpr std::string_view CREATE_TABLE_QUERY {"CREATE TABLE IF NOT EXISTS {} (message TEXT NOT NULL);"}; + constexpr std::string_view CREATE_TABLE_QUERY { + "CREATE TABLE IF NOT EXISTS {} (module TEXT, message TEXT NOT NULL);"}; auto createTableQuery = fmt::format(CREATE_TABLE_QUERY, tableName); std::lock_guard lock(m_mutex); try @@ -56,11 +57,11 @@ void SQLiteStorage::releaseDatabaseAccess() m_cv.notify_one(); } -void SQLiteStorage::Store(const json& message, const std::string& tableName) +void SQLiteStorage::Store(const json& message, const std::string& tableName, const std::string& moduleName) { waitForDatabaseAccess(); - constexpr std::string_view INSERT_QUERY {"INSERT INTO {} (message) VALUES (?);"}; - std::string insertQuery = fmt::format(INSERT_QUERY, tableName); + constexpr std::string_view INSERT_QUERY {"INSERT INTO {} (module, message) VALUES (\"{}\", ?);"}; + std::string insertQuery = fmt::format(INSERT_QUERY, tableName, moduleName); SQLite::Statement query = SQLite::Statement(*m_db, insertQuery); if (message.is_array()) @@ -75,7 +76,7 @@ void SQLiteStorage::Store(const json& message, const std::string& tableName) } catch (const SQLite::Exception& e) { - std::cerr << "2 " << e.what() << '\n'; + std::cerr << "Error SqliteStorage Store: " << e.what() << '\n'; } // Reset the query to reuse it for the next message query.reset(); @@ -104,6 +105,7 @@ json SQLiteStorage::Retrieve(int id, const std::string& tableName) json message; if (query.executeStep()) { + // TODO: Pending retrieve module message = json::parse(query.getColumn(0).getString()); } else @@ -114,31 +116,69 @@ json SQLiteStorage::Retrieve(int id, const std::string& tableName) } catch (const SQLite::Exception& e) { - std::cerr << "Error retrieving message: " << e.what() << std::endl; - throw; + std::cerr << "Error SQLiteStorage retrieve: " << e.what() << std::endl; + return {}; } } -json SQLiteStorage::RetrieveMultiple(int n, const std::string& tableName) +json SQLiteStorage::RetrieveMultiple(int n, const std::string& tableName, const std::string& moduleName) { + std::string selectQuery; + if (moduleName.empty()) + { + constexpr std::string_view SELECT_MULTIPLE_QUERY {"SELECT module, message FROM {} ORDER BY rowid ASC LIMIT ?;"}; + selectQuery = fmt::format(SELECT_MULTIPLE_QUERY, tableName); + } + else + { + constexpr std::string_view SELECT_QUERY { + "SELECT module, message FROM {} WHERE module LIKE \"{}\" ORDER BY rowid ASC LIMIT ?;"}; + selectQuery = fmt::format(SELECT_QUERY, tableName, moduleName); + } + try { - constexpr std::string_view SELECT_MULTIPLE_QUERY {"SELECT message FROM {} ORDER BY rowid ASC LIMIT ?;"}; - std::string selectQuery = fmt::format(SELECT_MULTIPLE_QUERY, tableName); SQLite::Statement query(*m_db, selectQuery); query.bind(1, n); json messages = json::array(); while (query.executeStep()) { - messages.push_back(json::parse(query.getColumn(0).getString())); + // getting data json + std::string dataString; + std::string moduleString; + + // TODO: recheck type! + if (query.getColumnCount() == 2 && query.getColumn(1).getType() == 3 && query.getColumn(0).getType() == 3) + { + moduleString = query.getColumn(0).getString(); + dataString = query.getColumn(1).getString(); + + json outputJson = {{"module", ""}, {"data", {}}}; + + if (!dataString.empty()) + { + outputJson["data"] = json::parse(dataString); + } + + if (!moduleString.empty()) + { + outputJson["module"] = moduleString; + } + + messages.push_back(outputJson); + } + } + + if (!messages.empty()) + { + std::reverse(messages.begin(), messages.end()); } - std::reverse(messages.begin(), messages.end()); return messages; } catch (const SQLite::Exception& e) { - std::cerr << "Error retrieving multiple messages: " << e.what() << std::endl; - throw; + std::cerr << "Error SQLiteStorage retrieve multiple: " << e.what() << std::endl; + return {}; } } @@ -157,8 +197,8 @@ int SQLiteStorage::Remove(int id, const std::string& tableName) } catch (const SQLite::Exception& e) { - std::cerr << "Error removing message: " << e.what() << std::endl; - throw; + std::cerr << "Error SQLiteStorage remove: " << e.what() << std::endl; + return {}; } } @@ -180,8 +220,8 @@ int SQLiteStorage::RemoveMultiple(int n, const std::string& tableName) } catch (const SQLite::Exception& e) { - std::cerr << "Error removing multiple messages: " << e.what() << std::endl; - throw; + std::cerr << "Error SQLiteStorage remove multiple: " << e.what() << std::endl; + return {}; } } @@ -201,7 +241,7 @@ int SQLiteStorage::GetElementCount(const std::string& tableName) } catch (const SQLite::Exception& e) { - std::cerr << "Error getting element count: " << e.what() << std::endl; - throw; + std::cerr << "Error SQLiteStorage get element count: " << e.what() << std::endl; + return {}; } } diff --git a/src/agent/queue/tests/CMakeLists.txt b/src/agent/queue/tests/CMakeLists.txt index 1a45cf81790..40b9490bdfe 100644 --- a/src/agent/queue/tests/CMakeLists.txt +++ b/src/agent/queue/tests/CMakeLists.txt @@ -17,7 +17,7 @@ add_executable(test_queue target_link_libraries(test_queue ${GTEST_LIBRARIES} ${GTEST_MAIN_LIBRARIES} - queue_test + queue SQLiteCpp nlohmann_json::nlohmann_json fmt::fmt) diff --git a/src/agent/queue/tests/queue_test.cpp b/src/agent/queue/tests/queue_test.cpp index 225121164ee..75c6e6aea99 100644 --- a/src/agent/queue/tests/queue_test.cpp +++ b/src/agent/queue/tests/queue_test.cpp @@ -137,9 +137,8 @@ TEST_F(QueueTest, SinglePushGetNotEmpty) auto typeReceived = messageResponse.type; EXPECT_TRUE(typeSend == typeReceived); - auto dataResponse = messageResponse.data[0].template get(); - auto dataToSend = messageToSend.data.template get(); - EXPECT_EQ(dataResponse, dataToSend); + auto dataResponse = messageResponse.data.at(0).at("data"); + EXPECT_EQ(dataResponse, baseDataContent); EXPECT_FALSE(queue.isEmptyByType(MessageType::STATELESS)); } @@ -158,9 +157,8 @@ TEST_F(QueueTest, SinglePushPopEmpty) auto typeReceived = messageResponse.type; EXPECT_TRUE(typeSend == typeReceived); - auto dataResponse = messageResponse.data[0].template get(); - auto dataToSend = messageToSend.data.template get(); - EXPECT_EQ(dataResponse, dataToSend); + auto dataResponse = messageResponse.data.at(0).at("data"); + EXPECT_EQ(dataResponse, baseDataContent); queue.popLastMessage(MessageType::STATELESS); EXPECT_TRUE(queue.isEmptyByType(MessageType::STATELESS)); @@ -193,12 +191,37 @@ TEST_F(QueueTest, SingleGetPopOnEmpty) auto messageResponse = queue.getLastMessage(MessageType::STATEFUL); EXPECT_EQ(messageResponse.type, MessageType::STATEFUL); - EXPECT_EQ(messageResponse.data[0], nullptr); + EXPECT_EQ(messageResponse.data, "{}"_json); queue.popLastMessage(MessageType::STATEFUL); EXPECT_TRUE(queue.isEmptyByType(MessageType::STATEFUL)); } +TEST_F(QueueTest, SinglePushGetWithModule) +{ + MultiTypeQueue queue(BIG_QUEUE_QTTY); + const MessageType messageType {MessageType::STATELESS}; + const std::string moduleFakeName = "fake-module"; + const std::string moduleName = "module"; + const Message messageToSend {messageType, baseDataContent, moduleName}; + + queue.timeoutPush(messageToSend); + auto messageResponseWrongModule = queue.getLastMessage(MessageType::STATELESS, moduleFakeName); + + auto typeSend = messageToSend.type; + auto typeReceived = messageResponseWrongModule.type; + EXPECT_TRUE(typeSend == typeReceived); + + EXPECT_EQ(messageResponseWrongModule.data, "{}"_json); + + auto messageResponseCorrectModule = queue.getLastMessage(MessageType::STATELESS, moduleName); + + auto dataResponse = messageResponseCorrectModule.data.at(0).at("data"); + EXPECT_EQ(dataResponse, baseDataContent); + + EXPECT_EQ(moduleName, messageResponseCorrectModule.moduleName); +} + // Push, get and check while the queue is full TEST_F(QueueTest, SinglePushPopFullWithTimeout) { @@ -368,7 +391,7 @@ TEST_F(QueueTest, MultiplePushSeveralSingleGets) for (int i : {0, 1, 2}) { auto messageResponse = queue.getLastMessage(MessageType::STATELESS); - auto responseData = messageResponse.data[0].template get(); + auto responseData = messageResponse.data.at(0).at("data"); auto sentData = messageToSend.data[i].template get(); EXPECT_EQ(responseData, sentData); queue.popLastMessage(MessageType::STATELESS); diff --git a/src/vcpkg.json b/src/vcpkg.json index 65ad1a5c380..474d01a633b 100644 --- a/src/vcpkg.json +++ b/src/vcpkg.json @@ -10,7 +10,8 @@ "nlohmann-json", "openssl", "sqlitecpp", - "toml11" + "toml11", + "fmt" ], "overrides": [ { From 3b3b58b7ac02c375c749e5d05afb5692fca0439b Mon Sep 17 00:00:00 2001 From: LucioDonda Date: Wed, 7 Aug 2024 15:19:17 -0300 Subject: [PATCH 21/31] feat: add module field on the other methods --- src/agent/queue/include/persistence.h | 14 ++- src/agent/queue/include/sqlitestorage.h | 11 +- src/agent/queue/src/sqlitestorage.cpp | 111 ++++++++++++++----- src/agent/queue/tests/sqlitestorage_test.cpp | 48 +++++++- 4 files changed, 146 insertions(+), 38 deletions(-) diff --git a/src/agent/queue/include/persistence.h b/src/agent/queue/include/persistence.h index 468086d8689..43ae7ec0ce8 100644 --- a/src/agent/queue/include/persistence.h +++ b/src/agent/queue/include/persistence.h @@ -24,7 +24,8 @@ class Persistence * @brief * * @param message - * @param queueName + * @param tableName + * @param moduleName */ virtual void Store(const json& message, const std::string& tableName, const std::string& moduleName = "") = 0; @@ -33,15 +34,17 @@ class Persistence * * @param id * @param queueName + * @param moduleName * @return json */ - virtual json Retrieve(int id, const std::string& queueName) = 0; + virtual json Retrieve(int id, const std::string& tableName, const std::string& moduleName = "") = 0; /** * @brief * * @param n * @param queueName + * @param moduleName * @return json */ virtual json RetrieveMultiple(int n, const std::string& queueName, const std::string& moduleName = "") = 0; @@ -53,7 +56,7 @@ class Persistence * @param queueName * @return int */ - virtual int Remove(int id, const std::string& queueName) = 0; + virtual int Remove(int id, const std::string& queueName, const std::string& moduleName = "") = 0; /** * @brief @@ -62,15 +65,16 @@ class Persistence * @param queueName * @return int */ - virtual int RemoveMultiple(int n, const std::string& queueName) = 0; + virtual int RemoveMultiple(int n, const std::string& queueName, const std::string& moduleName = "") = 0; /** * @brief Get the Element Count object * * @param queueName + * @param moduleName * @return int */ - virtual int GetElementCount(const std::string& queueName) = 0; + virtual int GetElementCount(const std::string& tableName, const std::string& moduleName = "") = 0; }; #endif // PERSISTENCE_H diff --git a/src/agent/queue/include/sqlitestorage.h b/src/agent/queue/include/sqlitestorage.h index 572eaff84dd..ab6bf736960 100644 --- a/src/agent/queue/include/sqlitestorage.h +++ b/src/agent/queue/include/sqlitestorage.h @@ -50,14 +50,13 @@ class SQLiteStorage : public Persistence */ void Store(const json& message, const std::string& tableName, const std::string& moduleName = "") override; - // TODO: pending tests! /** * @brief Retrieve a JSON message by its ID. * * @param id The ID of the message to retrieve. * @return The retrieved JSON message. */ - json Retrieve(int id, const std::string& tableName) override; + json Retrieve(int id, const std::string& tableName, const std::string& moduleName = "") override; /** * @brief Retrieve multiple JSON messages. @@ -71,24 +70,26 @@ class SQLiteStorage : public Persistence * @brief Remove a JSON message by its ID. * * @param id The ID of the message to remove. + * @param moduleName * @return The number of removed elements. */ - int Remove(int id, const std::string& tableName) override; + int Remove(int id, const std::string& tableName, const std::string& moduleName = "") override; /** * @brief Remove multiple JSON messages. * * @param n The number of messages to remove. + * @param moduleName * @return The number of removed elements. */ - int RemoveMultiple(int n, const std::string& tableName) override; + int RemoveMultiple(int n, const std::string& tableName, const std::string& moduleName = "") override; /** * @brief Get the number of elements in the table. * * @return The number of elements in the table. */ - int GetElementCount(const std::string& tableName) override; + int GetElementCount(const std::string& tableName, const std::string& moduleName = "") override; private: /** diff --git a/src/agent/queue/src/sqlitestorage.cpp b/src/agent/queue/src/sqlitestorage.cpp index dc17628a2a6..c9503440d64 100644 --- a/src/agent/queue/src/sqlitestorage.cpp +++ b/src/agent/queue/src/sqlitestorage.cpp @@ -93,26 +93,52 @@ void SQLiteStorage::Store(const json& message, const std::string& tableName, con releaseDatabaseAccess(); } -json SQLiteStorage::Retrieve(int id, const std::string& tableName) +// TODO: we shouldn't use rowid outside the table itself +json SQLiteStorage::Retrieve(int id, const std::string& tableName, const std::string& moduleName) { + + std::string selectQuery; + if (moduleName.empty()) + { + constexpr std::string_view SELECT_QUERY {"SELECT module, message FROM {} WHERE rowid = ?;"}; + selectQuery = fmt::format(SELECT_QUERY, tableName); + } + else + { + constexpr std::string_view SELECT_QUERY { + "SELECT module, message FROM {} WHERE module LIKE \"{}\" AND rowid = ?;"}; + selectQuery = fmt::format(SELECT_QUERY, tableName, moduleName); + } + try { - constexpr std::string_view SELECT_QUERY {"SELECT message FROM {} WHERE rowid = ?;"}; - std::string selectQuery = fmt::format(SELECT_QUERY, tableName); SQLite::Statement query(*m_db, selectQuery); - // TODO: delte id? query.bind(1, id); - json message; + json outputJson = {{"module", ""}, {"data", {}}}; if (query.executeStep()) { - // TODO: Pending retrieve module - message = json::parse(query.getColumn(0).getString()); - } - else - { - message = nullptr; + std::string dataString; + std::string moduleString; + + if (query.getColumnCount() == 2 && query.getColumn(1).getType() == SQLite::TEXT && + query.getColumn(0).getType() == SQLite::TEXT) + { + moduleString = query.getColumn(0).getString(); + dataString = query.getColumn(1).getString(); + + if (!dataString.empty()) + { + outputJson["data"] = json::parse(dataString); + } + + if (!moduleString.empty()) + { + outputJson["module"] = moduleString; + } + } } - return message; + + return outputJson; } catch (const SQLite::Exception& e) { @@ -131,9 +157,9 @@ json SQLiteStorage::RetrieveMultiple(int n, const std::string& tableName, const } else { - constexpr std::string_view SELECT_QUERY { + constexpr std::string_view SELECT_MULTIPLE_QUERY { "SELECT module, message FROM {} WHERE module LIKE \"{}\" ORDER BY rowid ASC LIMIT ?;"}; - selectQuery = fmt::format(SELECT_QUERY, tableName, moduleName); + selectQuery = fmt::format(SELECT_MULTIPLE_QUERY, tableName, moduleName); } try @@ -147,8 +173,8 @@ json SQLiteStorage::RetrieveMultiple(int n, const std::string& tableName, const std::string dataString; std::string moduleString; - // TODO: recheck type! - if (query.getColumnCount() == 2 && query.getColumn(1).getType() == 3 && query.getColumn(0).getType() == 3) + if (query.getColumnCount() == 2 && query.getColumn(1).getType() == SQLite::TEXT && + query.getColumn(0).getType() == SQLite::TEXT) { moduleString = query.getColumn(0).getString(); dataString = query.getColumn(1).getString(); @@ -182,12 +208,22 @@ json SQLiteStorage::RetrieveMultiple(int n, const std::string& tableName, const } } -int SQLiteStorage::Remove(int id, const std::string& tableName) +int SQLiteStorage::Remove(int id, const std::string& tableName, const std::string& moduleName) { - constexpr std::string_view DELETE_QUERY {"DELETE FROM {} WHERE rowid = ?;"}; + std::string deleteQuery; + if (moduleName.empty()) + { + constexpr std::string_view DELETE_QUERY {"DELETE FROM {} WHERE rowid = ?;"}; + deleteQuery = fmt::format(DELETE_QUERY, tableName); + } + else + { + constexpr std::string_view DELETE_QUERY {"DELETE FROM {} WHERE module LIKE \"{}\" AND rowid = ?;"}; + deleteQuery = fmt::format(DELETE_QUERY, tableName, moduleName); + } + try { - std::string deleteQuery = fmt::format(DELETE_QUERY, tableName); SQLite::Statement query(*m_db, deleteQuery); query.bind(1, id); SQLite::Transaction transaction(*m_db); @@ -202,16 +238,27 @@ int SQLiteStorage::Remove(int id, const std::string& tableName) } } -int SQLiteStorage::RemoveMultiple(int n, const std::string& tableName) +int SQLiteStorage::RemoveMultiple(int n, const std::string& tableName, const std::string& moduleName) { - constexpr std::string_view DELETE_MULTIPLE_QUERY { - "DELETE FROM {} WHERE rowid IN (SELECT rowid FROM {} ORDER BY rowid ASC LIMIT ?);"}; + std::string deleteQuery; + if (moduleName.empty()) + { + constexpr std::string_view DELETE_MULTIPLE_QUERY { + "DELETE FROM {} WHERE rowid IN (SELECT rowid FROM {} ORDER BY rowid ASC LIMIT ?);"}; + deleteQuery = fmt::format(DELETE_MULTIPLE_QUERY, tableName, tableName); + } + else + { + constexpr std::string_view DELETE_MULTIPLE_QUERY { + "DELETE FROM {} WHERE module LIKE \"{}\" AND rowid IN (SELECT rowid FROM {} WHERE module LIKE \"{}\" ORDER BY rowid ASC LIMIT ?);"}; + deleteQuery = fmt::format(DELETE_MULTIPLE_QUERY, tableName, moduleName, tableName, moduleName); + } + try { waitForDatabaseAccess(); - SQLite::Transaction transaction(*m_db); - std::string deleteQuery = fmt::format(DELETE_MULTIPLE_QUERY, tableName, tableName); SQLite::Statement query(*m_db, deleteQuery); + SQLite::Transaction transaction(*m_db); query.bind(1, n); query.exec(); transaction.commit(); @@ -225,12 +272,22 @@ int SQLiteStorage::RemoveMultiple(int n, const std::string& tableName) } } -int SQLiteStorage::GetElementCount(const std::string& tableName) +int SQLiteStorage::GetElementCount(const std::string& tableName, const std::string& moduleName) { - constexpr std::string_view COUNT_QUERY {"SELECT COUNT(*) FROM {}"}; + std::string countQuery; + if (moduleName.empty()) + { + constexpr std::string_view COUNT_QUERY {"SELECT COUNT(*) FROM {}"}; + countQuery = fmt::format(COUNT_QUERY, tableName); + } + else + { + constexpr std::string_view COUNT_QUERY {"SELECT COUNT(*) FROM {} WHERE module LIKE \"{}\""}; + countQuery = fmt::format(COUNT_QUERY, tableName, moduleName); + } + try { - std::string countQuery = fmt::format(COUNT_QUERY, tableName); SQLite::Statement query(*m_db, countQuery); int count = 0; if (query.executeStep()) diff --git a/src/agent/queue/tests/sqlitestorage_test.cpp b/src/agent/queue/tests/sqlitestorage_test.cpp index 12258018b47..d34907f9027 100644 --- a/src/agent/queue/tests/sqlitestorage_test.cpp +++ b/src/agent/queue/tests/sqlitestorage_test.cpp @@ -16,6 +16,7 @@ class SQLiteStorageTest : public ::testing::Test protected: const std::string dbName = "testdb.db"; const std::string tableName = "test_table"; + const std::string moduleName = "moduleX"; const std::vector m_vMessageTypeStrings {"test_table", "test_table2"}; SQLiteStorage* storage; @@ -62,6 +63,17 @@ TEST_F(SQLiteStorageTest, StoreSingleMessage) EXPECT_EQ(storage->GetElementCount(tableName), 2); } +TEST_F(SQLiteStorageTest, StoreSingleMessageWithModule) +{ + json message = {{"key", "value"}}; + EXPECT_NO_THROW(storage->Store(message, tableName, moduleName)); + EXPECT_EQ(storage->GetElementCount(tableName), 1); + EXPECT_NO_THROW(storage->Store(message, tableName)); + EXPECT_EQ(storage->GetElementCount(tableName), 2); + EXPECT_EQ(storage->GetElementCount(tableName, "fakeModuleName"), 0); + EXPECT_EQ(storage->GetElementCount(tableName, moduleName), 1); +} + TEST_F(SQLiteStorageTest, StoreMultipleMessages) { json messages = json::array(); @@ -76,7 +88,17 @@ TEST_F(SQLiteStorageTest, RetrieveMessage) json message = {{"key", "value"}}; storage->Store(message, tableName); json retrievedMessage = storage->Retrieve(1, tableName); - EXPECT_EQ(retrievedMessage["key"], "value"); + EXPECT_EQ(retrievedMessage.at("data").at("key"), "value"); +} + +TEST_F(SQLiteStorageTest, RetrieveMessageWithModule) +{ + json message = {{"key", "value"}}; + storage->Store(message, tableName, moduleName); + json retrievedMessage = storage->Retrieve(1, tableName, "fakeModuleName"); + EXPECT_EQ(retrievedMessage.at("data"), nullptr); + retrievedMessage = storage->Retrieve(1, tableName, moduleName); + EXPECT_EQ(retrievedMessage.at("data").at("key"), "value"); } TEST_F(SQLiteStorageTest, RetrieveMultipleMessages) @@ -97,6 +119,18 @@ TEST_F(SQLiteStorageTest, RemoveMessage) EXPECT_EQ(storage->GetElementCount(tableName), 0); } +TEST_F(SQLiteStorageTest, RemoveMessageWithModule) +{ + json message = {{"key", "value"}}; + storage->Store(message, tableName, moduleName); + EXPECT_EQ(storage->Remove(1, tableName), 1); + storage->Store(message, tableName, moduleName); + EXPECT_EQ(storage->Remove(1, tableName, "fakeModuleName"), 1); + EXPECT_EQ(storage->GetElementCount(tableName), 1); + EXPECT_EQ(storage->Remove(1, tableName, moduleName), 1); + EXPECT_EQ(storage->GetElementCount(tableName), 0); +} + TEST_F(SQLiteStorageTest, RemoveMultipleMessages) { json messages = json::array(); @@ -107,6 +141,18 @@ TEST_F(SQLiteStorageTest, RemoveMultipleMessages) EXPECT_EQ(storage->GetElementCount(tableName), 0); } +TEST_F(SQLiteStorageTest, RemoveMultipleMessagesWithModule) +{ + json messages = json::array(); + messages.push_back({{"key", "value1"}}); + messages.push_back({{"key", "value2"}}); + storage->Store(messages, tableName, moduleName); + EXPECT_EQ(storage->RemoveMultiple(2, tableName, "fakeModuleName"), 2); + EXPECT_EQ(storage->GetElementCount(tableName), 2); + EXPECT_EQ(storage->RemoveMultiple(2, tableName, moduleName), 2); + EXPECT_EQ(storage->GetElementCount(tableName), 0); +} + TEST_F(SQLiteStorageTest, GetElementCount) { json message = {{"key", "value"}}; From 30f3e6b2b6a05017ac759e3c53e0232499ee42fa Mon Sep 17 00:00:00 2001 From: LucioDonda Date: Wed, 7 Aug 2024 23:26:35 -0300 Subject: [PATCH 22/31] feat: getting N message and tests for it --- src/agent/queue/include/queue.hpp | 18 +++-- src/agent/queue/src/queue.cpp | 54 ++++++++------ src/agent/queue/tests/queue_test.cpp | 78 +++++++++++++++++++- src/agent/queue/tests/sqlitestorage_test.cpp | 46 +++++++++++- 4 files changed, 162 insertions(+), 34 deletions(-) diff --git a/src/agent/queue/include/queue.hpp b/src/agent/queue/include/queue.hpp index d6b4a009b30..ac581279881 100644 --- a/src/agent/queue/include/queue.hpp +++ b/src/agent/queue/include/queue.hpp @@ -112,56 +112,62 @@ class MultiTypeQueue * @brief Returns N messages from a queue * * @param type Of the queue to be used as source + * @param moduleName * @param messageQuantity quantity of messages to return * @return Message Json data othe messages fetched */ - Message getNMessages(MessageType type, int messageQuantity, const std::string moduleName = ""); + std::vector getNMessages(MessageType type, int messageQuantity, const std::string moduleName = ""); /** * @brief deletes a message from a queue * * @param type MessageType queue to pop + * @param moduleName * @return true popped succesfully * @return false wasn't able to pop message */ - bool popLastMessage(MessageType type); + bool popLastMessage(MessageType type, const std::string moduleName = ""); /** * @brief * * @param type + * @param moduleName * @param messageQuantity * @return true * @return false */ - bool popNMessages(MessageType type, int messageQuantity); + bool popNMessages(MessageType type, int messageQuantity, const std::string moduleName = ""); /** * @brief Checks emptyness of a queue * * @param type + * @param moduleName * @return true when queue empty * @return false otherwise */ - bool isEmptyByType(MessageType type); + bool isEmptyByType(MessageType type, const std::string moduleName = ""); /** * @brief Checks fullness of a queue * * @param type + * @param moduleName * @return true when queue is full * @return false otherwise */ - bool isFullByType(MessageType type); + bool isFullByType(MessageType type, const std::string moduleName = ""); /** * @brief Get the Items By Type object * * @param type + * @param moduleName * @return true * @return false */ - int getItemsByType(MessageType type); + int getItemsByType(MessageType type, const std::string moduleName = ""); }; #endif // QUEUE_H diff --git a/src/agent/queue/src/queue.cpp b/src/agent/queue/src/queue.cpp index 2d11337c31e..c7144a4d376 100644 --- a/src/agent/queue/src/queue.cpp +++ b/src/agent/queue/src/queue.cpp @@ -74,8 +74,8 @@ void MultiTypeQueue::timeoutPush(std::vector messages) Message MultiTypeQueue::getLastMessage(MessageType type, const std::string moduleName) { + // TODO: If moduleName is not emty should be part of result on the pther cases should be parsed from answer Message result(type, "{}"_json, moduleName); - // std::unique_lock mapLock(m_mapMutex); if (m_mapMessageTypeName.contains(type)) { auto resultData = m_persistenceDest->RetrieveMultiple(1, m_mapMessageTypeName.at(type), moduleName); @@ -86,95 +86,105 @@ Message MultiTypeQueue::getLastMessage(MessageType type, const std::string modul } else { - // TODO: error / logging handling !!! + // TODO: error handling and logging std::cout << "error didn't find the queue" << std::endl; } return result; } -Message MultiTypeQueue::getNMessages(MessageType type, int messageQuantity, const std::string moduleName) +std::vector MultiTypeQueue::getNMessages(MessageType type, int messageQuantity, const std::string moduleName) { - Message result(type, {}); + std::vector result; if (m_mapMessageTypeName.contains(type)) { - result.data = m_persistenceDest->RetrieveMultiple(messageQuantity, m_mapMessageTypeName.at(type), moduleName); + auto arrayData = + m_persistenceDest->RetrieveMultiple(messageQuantity, m_mapMessageTypeName.at(type), moduleName); + for (auto singleJson : arrayData) + { + auto finalModuleName = moduleName; + if (moduleName.empty()) + { + finalModuleName = singleJson["module"]; + } + result.push_back(Message(type, singleJson, finalModuleName)); + } } else { - // TODO: error / logging handling !!! + // TODO: error handling and logging std::cout << "error didn't find the queue" << std::endl; } return result; } -bool MultiTypeQueue::popLastMessage(MessageType type) +bool MultiTypeQueue::popLastMessage(MessageType type, const std::string moduleName) { bool result = false; if (m_mapMessageTypeName.contains(type)) { - // Handle return value - result = m_persistenceDest->RemoveMultiple(1, m_mapMessageTypeName.at(type)); + // TODO: Handle return value -> should show how many rows where deleted + result = m_persistenceDest->RemoveMultiple(1, m_mapMessageTypeName.at(type), moduleName); } else { - // TODO: error / logging handling !!! + // TODO: error handling and logging std::cout << "error didn't find the queue" << std::endl; } return result; } -bool MultiTypeQueue::popNMessages(MessageType type, int messageQuantity) +bool MultiTypeQueue::popNMessages(MessageType type, int messageQuantity, const std::string moduleName) { bool result = false; if (m_mapMessageTypeName.contains(type)) { - result = m_persistenceDest->RemoveMultiple(messageQuantity, m_mapMessageTypeName.at(type)); + result = m_persistenceDest->RemoveMultiple(messageQuantity, m_mapMessageTypeName.at(type), moduleName); } else { - // TODO: error / logging handling !!! + // TODO: error handling and logging std::cout << "error didn't find the queue" << std::endl; } return result; } -bool MultiTypeQueue::isEmptyByType(MessageType type) +bool MultiTypeQueue::isEmptyByType(MessageType type, const std::string moduleName) { if (m_mapMessageTypeName.contains(type)) { - return m_persistenceDest->GetElementCount(m_mapMessageTypeName.at(type)) == 0; + return m_persistenceDest->GetElementCount(m_mapMessageTypeName.at(type), moduleName) == 0; } else { - // TODO: error / logging handling !!! + // TODO: error handling and logging std::cout << "error didn't find the queue" << std::endl; } return false; } -bool MultiTypeQueue::isFullByType(MessageType type) +bool MultiTypeQueue::isFullByType(MessageType type, const std::string moduleName) { if (m_mapMessageTypeName.contains(type)) { - return m_persistenceDest->GetElementCount(m_mapMessageTypeName.at(type)) == m_maxItems; + return m_persistenceDest->GetElementCount(m_mapMessageTypeName.at(type), moduleName) == m_maxItems; } else { - // TODO: error / logging handling !!! + // TODO: error handling and logging std::cout << "error didn't find the queue" << std::endl; } return false; } -int MultiTypeQueue::getItemsByType(MessageType type) +int MultiTypeQueue::getItemsByType(MessageType type, const std::string moduleName) { if (m_mapMessageTypeName.contains(type)) { - return m_persistenceDest->GetElementCount(m_mapMessageTypeName.at(type)); + return m_persistenceDest->GetElementCount(m_mapMessageTypeName.at(type), moduleName); } else { - // TODO: error / logging handling !!! + // TODO: error handling and logging std::cout << "error didn't find the queue" << std::endl; } return false; diff --git a/src/agent/queue/tests/queue_test.cpp b/src/agent/queue/tests/queue_test.cpp index 75c6e6aea99..6fbc58e72bf 100644 --- a/src/agent/queue/tests/queue_test.cpp +++ b/src/agent/queue/tests/queue_test.cpp @@ -98,9 +98,17 @@ TEST_F(JsonTest, JSONConversionComparisson) auto versionUj2 = uj2["version"].template get(); EXPECT_EQ(versionUj1, versionUj2); + auto versionUj12 = uj1.at("version"); + auto versionUj22 = uj2.at("version"); + EXPECT_EQ(versionUj12, versionUj22); + auto typeUj1 = uj1["type"].template get(); auto typeUj2 = uj2["type"].template get(); EXPECT_EQ(typeUj1, typeUj2); + + auto typeUj12 = uj1.at("type"); + auto typeUj22 = uj2.at("type"); + EXPECT_EQ(typeUj12, typeUj22); } TEST_F(JsonTest, JSONArrays) @@ -241,6 +249,7 @@ TEST_F(QueueTest, SinglePushPopFullWithTimeout) auto items = queue.getItemsByType(MessageType::COMMAND); EXPECT_EQ(items, SMALL_QUEUE_QTTY); + EXPECT_TRUE(queue.isFullByType(MessageType::COMMAND)); EXPECT_TRUE(queue.isEmptyByType(MessageType::STATELESS)); queue.popLastMessage(MessageType::COMMAND); @@ -378,7 +387,7 @@ TEST_F(QueueTest, MultithreadSameType) // Push Multiple with single message and data array, // several gets, checks and pops -TEST_F(QueueTest, MultiplePushSeveralSingleGets) +TEST_F(QueueTest, PushMultipleSeveralSingleGets) { MultiTypeQueue queue(BIG_QUEUE_QTTY); const MessageType messageType {MessageType::STATELESS}; @@ -400,8 +409,24 @@ TEST_F(QueueTest, MultiplePushSeveralSingleGets) EXPECT_EQ(queue.getItemsByType(MessageType::STATELESS), 0); } +TEST_F(QueueTest, PushMultipleWithMessageVector) +{ + MultiTypeQueue queue(BIG_QUEUE_QTTY); + + std::vector messages; + const MessageType messageType {MessageType::STATELESS}; + for (std::string i : {"0", "1", "2"}) + { + const json multipleDataContent = {"content " + i}; + messages.push_back({messageType, multipleDataContent}); + } + EXPECT_EQ(messages.size(), 3); + queue.timeoutPush(messages); + EXPECT_EQ(queue.getItemsByType(MessageType::STATELESS), 3); +} + // Push Multiple, pop multiples -TEST_F(QueueTest, MultiplePushSeveralMultiplePops) +TEST_F(QueueTest, PushMultipleGetMultiple) { MultiTypeQueue queue(BIG_QUEUE_QTTY); const MessageType messageType {MessageType::STATELESS}; @@ -414,3 +439,52 @@ TEST_F(QueueTest, MultiplePushSeveralMultiplePops) EXPECT_TRUE(queue.isEmptyByType(MessageType::STATELESS)); EXPECT_EQ(0, queue.getItemsByType(MessageType::STATELESS)); } + +// Push Multiple, pop multiples +TEST_F(QueueTest, PushMultipleGetMultipleWithModule) +{ + MultiTypeQueue queue(BIG_QUEUE_QTTY); + const MessageType messageType {MessageType::STATELESS}; + const json multipleDataContent = {"content 1", "content 2", "content 3"}; + const std::string moduleName = "testModule"; + const Message messageToSend {messageType, multipleDataContent, moduleName}; + + queue.timeoutPush(messageToSend); + + // Altough we're asking for 10 messages only the availables are returned. + auto messagesReceived = queue.getNMessages(MessageType::STATELESS, 10); + int i = 3; + for (auto singleMessage : messagesReceived) + { + EXPECT_EQ("content " + std::to_string(i--), singleMessage.data.at("data")); + } + + EXPECT_EQ(0, queue.getItemsByType(MessageType::STATELESS, "fakemodule")); + EXPECT_EQ(3, queue.getItemsByType(MessageType::STATELESS)); + EXPECT_EQ(3, queue.getItemsByType(MessageType::STATELESS, moduleName)); +} + +TEST_F(QueueTest, PushSinglesleGetMultipleWithModule) +{ + MultiTypeQueue queue(BIG_QUEUE_QTTY); + + for (std::string i : {"1", "2", "3", "4", "5"}) + { + const MessageType messageType {MessageType::STATELESS}; + const json multipleDataContent = {"content-" + i}; + const std::string moduleName = "module-" + i; + const Message messageToSend {messageType, multipleDataContent, moduleName}; + queue.timeoutPush(messageToSend); + } + + auto messagesReceived = queue.getNMessages(MessageType::STATELESS, 10); + EXPECT_EQ(5, messagesReceived.size()); + int i = 5; + for (auto singleMessage : messagesReceived) + { + EXPECT_EQ("content-" + std::to_string(i--), singleMessage.data.at("data")); + } + + auto messageReceivedContent1 = queue.getNMessages(MessageType::STATELESS, 10, "module-1"); + EXPECT_EQ(1, messageReceivedContent1.size()); +} diff --git a/src/agent/queue/tests/sqlitestorage_test.cpp b/src/agent/queue/tests/sqlitestorage_test.cpp index d34907f9027..4f0fe153ca1 100644 --- a/src/agent/queue/tests/sqlitestorage_test.cpp +++ b/src/agent/queue/tests/sqlitestorage_test.cpp @@ -70,7 +70,7 @@ TEST_F(SQLiteStorageTest, StoreSingleMessageWithModule) EXPECT_EQ(storage->GetElementCount(tableName), 1); EXPECT_NO_THROW(storage->Store(message, tableName)); EXPECT_EQ(storage->GetElementCount(tableName), 2); - EXPECT_EQ(storage->GetElementCount(tableName, "fakeModuleName"), 0); + EXPECT_EQ(storage->GetElementCount(tableName, "unavailableModuleName"), 0); EXPECT_EQ(storage->GetElementCount(tableName, moduleName), 1); } @@ -83,6 +83,17 @@ TEST_F(SQLiteStorageTest, StoreMultipleMessages) EXPECT_EQ(storage->GetElementCount(tableName), 2); } +TEST_F(SQLiteStorageTest, StoreMultipleMessagesWithModule) +{ + json messages = json::array(); + messages.push_back({{"key", "value1"}}); + messages.push_back({{"key", "value2"}}); + EXPECT_NO_THROW(storage->Store(messages, tableName, moduleName)); + EXPECT_EQ(storage->GetElementCount(tableName), 2); + EXPECT_EQ(storage->GetElementCount(tableName, moduleName), 2); + EXPECT_EQ(storage->GetElementCount(tableName, "unavailableModuleName"), 0); +} + TEST_F(SQLiteStorageTest, RetrieveMessage) { json message = {{"key", "value"}}; @@ -95,7 +106,7 @@ TEST_F(SQLiteStorageTest, RetrieveMessageWithModule) { json message = {{"key", "value"}}; storage->Store(message, tableName, moduleName); - json retrievedMessage = storage->Retrieve(1, tableName, "fakeModuleName"); + json retrievedMessage = storage->Retrieve(1, tableName, "unavailableModuleName"); EXPECT_EQ(retrievedMessage.at("data"), nullptr); retrievedMessage = storage->Retrieve(1, tableName, moduleName); EXPECT_EQ(retrievedMessage.at("data").at("key"), "value"); @@ -111,6 +122,24 @@ TEST_F(SQLiteStorageTest, RetrieveMultipleMessages) EXPECT_EQ(retrievedMessages.size(), 2); } +TEST_F(SQLiteStorageTest, RetrieveMultipleMessagesWithModule) +{ + json messages = json::array(); + messages.push_back({{"key", "value1"}}); + messages.push_back({{"key", "value2"}}); + messages.push_back({{"key", "value3"}}); + messages.push_back({{"key", "value4"}}); + storage->Store(messages, tableName, moduleName); + json retrievedMessages = storage->RetrieveMultiple(4, tableName, moduleName); + EXPECT_EQ(retrievedMessages.size(), 4); + + int i = 4; + for (auto singleMessage : retrievedMessages) + { + EXPECT_EQ("value" + std::to_string(i--), singleMessage.at("data").at("key")); + } +} + TEST_F(SQLiteStorageTest, RemoveMessage) { json message = {{"key", "value"}}; @@ -125,7 +154,7 @@ TEST_F(SQLiteStorageTest, RemoveMessageWithModule) storage->Store(message, tableName, moduleName); EXPECT_EQ(storage->Remove(1, tableName), 1); storage->Store(message, tableName, moduleName); - EXPECT_EQ(storage->Remove(1, tableName, "fakeModuleName"), 1); + EXPECT_EQ(storage->Remove(1, tableName, "unavailableModuleName"), 1); EXPECT_EQ(storage->GetElementCount(tableName), 1); EXPECT_EQ(storage->Remove(1, tableName, moduleName), 1); EXPECT_EQ(storage->GetElementCount(tableName), 0); @@ -147,7 +176,7 @@ TEST_F(SQLiteStorageTest, RemoveMultipleMessagesWithModule) messages.push_back({{"key", "value1"}}); messages.push_back({{"key", "value2"}}); storage->Store(messages, tableName, moduleName); - EXPECT_EQ(storage->RemoveMultiple(2, tableName, "fakeModuleName"), 2); + EXPECT_EQ(storage->RemoveMultiple(2, tableName, "unavailableModuleName"), 2); EXPECT_EQ(storage->GetElementCount(tableName), 2); EXPECT_EQ(storage->RemoveMultiple(2, tableName, moduleName), 2); EXPECT_EQ(storage->GetElementCount(tableName), 0); @@ -160,6 +189,15 @@ TEST_F(SQLiteStorageTest, GetElementCount) EXPECT_EQ(storage->GetElementCount(tableName), 1); } +TEST_F(SQLiteStorageTest, GetElementCountWithModule) +{ + json message = {{"key", "value"}}; + storage->Store(message, tableName, moduleName); + EXPECT_EQ(storage->GetElementCount(tableName), 1); + EXPECT_EQ(storage->GetElementCount(tableName, moduleName), 1); + EXPECT_EQ(storage->GetElementCount(tableName, "unavailableModuleName"), 0); +} + class SQLiteStorageMultithreadedTest : public ::testing::Test { protected: From 9a4cc9a1934e5ca425b4c7dfe68edf6707cca4fe Mon Sep 17 00:00:00 2001 From: LucioDonda Date: Thu, 8 Aug 2024 19:05:04 -0300 Subject: [PATCH 23/31] feat: returning rows modified, message vector, test and cleaning --- src/agent/queue/include/persistence.h | 11 +- src/agent/queue/include/queue.hpp | 21 ++-- src/agent/queue/include/sqlitestorage.h | 5 +- src/agent/queue/src/queue.cpp | 26 +++-- src/agent/queue/src/sqlitestorage.cpp | 23 ++-- src/agent/queue/tests/queue_test.cpp | 115 +++++++------------ src/agent/queue/tests/sqlitestorage_test.cpp | 30 ++--- 7 files changed, 111 insertions(+), 120 deletions(-) diff --git a/src/agent/queue/include/persistence.h b/src/agent/queue/include/persistence.h index 43ae7ec0ce8..8ef69b6dd11 100644 --- a/src/agent/queue/include/persistence.h +++ b/src/agent/queue/include/persistence.h @@ -24,10 +24,11 @@ class Persistence * @brief * * @param message - * @param tableName + * @param queueName * @param moduleName + * @return int */ - virtual void Store(const json& message, const std::string& tableName, const std::string& moduleName = "") = 0; + virtual int Store(const json& message, const std::string& queueName, const std::string& moduleName = "") = 0; /** * @brief @@ -37,7 +38,7 @@ class Persistence * @param moduleName * @return json */ - virtual json Retrieve(int id, const std::string& tableName, const std::string& moduleName = "") = 0; + virtual json Retrieve(int id, const std::string& queueName, const std::string& moduleName = "") = 0; /** * @brief @@ -54,6 +55,7 @@ class Persistence * * @param id * @param queueName + * @param moduleName * @return int */ virtual int Remove(int id, const std::string& queueName, const std::string& moduleName = "") = 0; @@ -63,6 +65,7 @@ class Persistence * * @param n * @param queueName + * @param moduleName * @return int */ virtual int RemoveMultiple(int n, const std::string& queueName, const std::string& moduleName = "") = 0; @@ -74,7 +77,7 @@ class Persistence * @param moduleName * @return int */ - virtual int GetElementCount(const std::string& tableName, const std::string& moduleName = "") = 0; + virtual int GetElementCount(const std::string& queueName, const std::string& moduleName = "") = 0; }; #endif // PERSISTENCE_H diff --git a/src/agent/queue/include/queue.hpp b/src/agent/queue/include/queue.hpp index ac581279881..d99d6af9377 100644 --- a/src/agent/queue/include/queue.hpp +++ b/src/agent/queue/include/queue.hpp @@ -87,18 +87,22 @@ class MultiTypeQueue ~MultiTypeQueue() {}; /** - * @brief: timeoutPush message to a queue of t + * @brief pushes a message * - * @param message + * @param message to be pushed + * @param shouldWait when true, the function will wait until the message is pushed + * @return int number of messages pushed */ - bool timeoutPush(Message message, bool shouldWait = false); + int timeoutPush(Message message, bool shouldWait = false); /** - * @brief + * @brief pushes a vector of messages * - * @param messages + * @param messages to be pushed + * @param shouldWait when true, the function will wait until the message is pushed + * @return int number of messages pushed */ - void timeoutPush(std::vector messages); + int timeoutPush(std::vector messages); /** * @brief Get the Last Message object @@ -134,10 +138,9 @@ class MultiTypeQueue * @param type * @param moduleName * @param messageQuantity - * @return true - * @return false + * @return Number of messages deleted */ - bool popNMessages(MessageType type, int messageQuantity, const std::string moduleName = ""); + int popNMessages(MessageType type, int messageQuantity, const std::string moduleName = ""); /** * @brief Checks emptyness of a queue diff --git a/src/agent/queue/include/sqlitestorage.h b/src/agent/queue/include/sqlitestorage.h index ab6bf736960..bbc4b6c53dc 100644 --- a/src/agent/queue/include/sqlitestorage.h +++ b/src/agent/queue/include/sqlitestorage.h @@ -47,8 +47,11 @@ class SQLiteStorage : public Persistence * @brief Store a JSON message in the storage. * * @param message The JSON message to store. + * @param tableName The name of the table to store the message in. + * @param moduleName + * @return The number of stored elements. */ - void Store(const json& message, const std::string& tableName, const std::string& moduleName = "") override; + int Store(const json& message, const std::string& tableName, const std::string& moduleName = "") override; /** * @brief Retrieve a JSON message by its ID. diff --git a/src/agent/queue/src/queue.cpp b/src/agent/queue/src/queue.cpp index c7144a4d376..fe670cf0d21 100644 --- a/src/agent/queue/src/queue.cpp +++ b/src/agent/queue/src/queue.cpp @@ -12,9 +12,9 @@ #include "queue.hpp" -bool MultiTypeQueue::timeoutPush(Message message, bool shouldWait) +int MultiTypeQueue::timeoutPush(Message message, bool shouldWait) { - bool result = false; + int result = 0; if (m_mapMessageTypeName.contains(message.type)) { @@ -32,8 +32,6 @@ bool MultiTypeQueue::timeoutPush(Message message, bool shouldWait) size_t spaceAvailable = (m_maxItems > storedMessages) ? m_maxItems - storedMessages : 0; if (spaceAvailable) { - // TODO: handle response - result = true; auto messageData = message.data; if (messageData.is_array()) { @@ -41,7 +39,7 @@ bool MultiTypeQueue::timeoutPush(Message message, bool shouldWait) { for (const auto& singleMessageData : messageData) { - m_persistenceDest->Store(singleMessageData, sMessageType, message.moduleName); + result += m_persistenceDest->Store(singleMessageData, sMessageType, message.moduleName); m_cv.notify_all(); } } @@ -52,7 +50,8 @@ bool MultiTypeQueue::timeoutPush(Message message, bool shouldWait) } else { - m_persistenceDest->Store(message.data, m_mapMessageTypeName.at(message.type), message.moduleName); + result = + m_persistenceDest->Store(message.data, m_mapMessageTypeName.at(message.type), message.moduleName); m_cv.notify_all(); } } @@ -64,17 +63,18 @@ bool MultiTypeQueue::timeoutPush(Message message, bool shouldWait) return result; } -void MultiTypeQueue::timeoutPush(std::vector messages) +int MultiTypeQueue::timeoutPush(std::vector messages) { + int result = 0; for (const auto& singleMessage : messages) { - timeoutPush(singleMessage); + result += timeoutPush(singleMessage); } + return result; } Message MultiTypeQueue::getLastMessage(MessageType type, const std::string moduleName) { - // TODO: If moduleName is not emty should be part of result on the pther cases should be parsed from answer Message result(type, "{}"_json, moduleName); if (m_mapMessageTypeName.contains(type)) { @@ -82,6 +82,10 @@ Message MultiTypeQueue::getLastMessage(MessageType type, const std::string modul if (!resultData.empty()) { result.data = resultData; + if (moduleName.empty()) + { + result.moduleName = result.data.at(0).at("module"); + } } } else @@ -133,9 +137,9 @@ bool MultiTypeQueue::popLastMessage(MessageType type, const std::string moduleNa return result; } -bool MultiTypeQueue::popNMessages(MessageType type, int messageQuantity, const std::string moduleName) +int MultiTypeQueue::popNMessages(MessageType type, int messageQuantity, const std::string moduleName) { - bool result = false; + int result = 0; if (m_mapMessageTypeName.contains(type)) { result = m_persistenceDest->RemoveMultiple(messageQuantity, m_mapMessageTypeName.at(type), moduleName); diff --git a/src/agent/queue/src/sqlitestorage.cpp b/src/agent/queue/src/sqlitestorage.cpp index c9503440d64..e5f776c67dc 100644 --- a/src/agent/queue/src/sqlitestorage.cpp +++ b/src/agent/queue/src/sqlitestorage.cpp @@ -57,11 +57,13 @@ void SQLiteStorage::releaseDatabaseAccess() m_cv.notify_one(); } -void SQLiteStorage::Store(const json& message, const std::string& tableName, const std::string& moduleName) +int SQLiteStorage::Store(const json& message, const std::string& tableName, const std::string& moduleName) { - waitForDatabaseAccess(); constexpr std::string_view INSERT_QUERY {"INSERT INTO {} (module, message) VALUES (\"{}\", ?);"}; std::string insertQuery = fmt::format(INSERT_QUERY, tableName, moduleName); + int result = 0; + + waitForDatabaseAccess(); SQLite::Statement query = SQLite::Statement(*m_db, insertQuery); if (message.is_array()) @@ -72,11 +74,12 @@ void SQLiteStorage::Store(const json& message, const std::string& tableName, con try { query.bind(1, singleMessageData.dump()); - query.exec(); + result += query.exec(); } catch (const SQLite::Exception& e) { std::cerr << "Error SqliteStorage Store: " << e.what() << '\n'; + break; } // Reset the query to reuse it for the next message query.reset(); @@ -87,10 +90,12 @@ void SQLiteStorage::Store(const json& message, const std::string& tableName, con { SQLite::Transaction transaction(*m_db); query.bind(1, message.dump()); - query.exec(); + result = query.exec(); transaction.commit(); } releaseDatabaseAccess(); + + return result; } // TODO: we shouldn't use rowid outside the table itself @@ -241,6 +246,7 @@ int SQLiteStorage::Remove(int id, const std::string& tableName, const std::strin int SQLiteStorage::RemoveMultiple(int n, const std::string& tableName, const std::string& moduleName) { std::string deleteQuery; + int rowsModified = 0; if (moduleName.empty()) { constexpr std::string_view DELETE_MULTIPLE_QUERY { @@ -250,7 +256,8 @@ int SQLiteStorage::RemoveMultiple(int n, const std::string& tableName, const std else { constexpr std::string_view DELETE_MULTIPLE_QUERY { - "DELETE FROM {} WHERE module LIKE \"{}\" AND rowid IN (SELECT rowid FROM {} WHERE module LIKE \"{}\" ORDER BY rowid ASC LIMIT ?);"}; + "DELETE FROM {} WHERE module LIKE \"{}\" AND rowid IN (SELECT rowid FROM {} WHERE module LIKE \"{}\" ORDER " + "BY rowid ASC LIMIT ?);"}; deleteQuery = fmt::format(DELETE_MULTIPLE_QUERY, tableName, moduleName, tableName, moduleName); } @@ -260,15 +267,15 @@ int SQLiteStorage::RemoveMultiple(int n, const std::string& tableName, const std SQLite::Statement query(*m_db, deleteQuery); SQLite::Transaction transaction(*m_db); query.bind(1, n); - query.exec(); + rowsModified = query.exec(); transaction.commit(); releaseDatabaseAccess(); - return n; + return rowsModified; } catch (const SQLite::Exception& e) { std::cerr << "Error SQLiteStorage remove multiple: " << e.what() << std::endl; - return {}; + return rowsModified; } } diff --git a/src/agent/queue/tests/queue_test.cpp b/src/agent/queue/tests/queue_test.cpp index 6fbc58e72bf..bcc1dd4e81b 100644 --- a/src/agent/queue/tests/queue_test.cpp +++ b/src/agent/queue/tests/queue_test.cpp @@ -14,12 +14,11 @@ using json = nlohmann::json; -#define BIG_QUEUE_QTTY 10 -#define SMALL_QUEUE_QTTY 2 +#define BIG_QUEUE_CAPACITY 10 +#define SMALL_QUEUE_CAPACITY 2 const json baseDataContent = R"({{"data": "for STATELESS_0"}})"; -const json multipleDataContent = R"({{"content 1", "content 2", "content 3"}})"; -// TODO: test string: const std::string multipleDataContent = R"({"data": {"content 1", "content 2", "content 3"})"; +const json multipleDataContent = {"content 1", "content 2", "content 3"}; /// Helper functions @@ -116,7 +115,7 @@ TEST_F(JsonTest, JSONArrays) // create JSON values json j_object = {{"one", 1}, {"two", 2}, {"three", 3}}; json j_array = {1, 2, 4, 8, 16}; - json multipleDataContent = {"content 1", "content 2", "content 3"}; + // TODO: test string: const std::string multipleDataContent = R"({"data": {"content 1", "content 2", "content 3"})"; // call is_array() EXPECT_FALSE(j_object.is_array()); @@ -134,11 +133,11 @@ TEST_F(JsonTest, JSONArrays) // Push, get and check the queue is not empty TEST_F(QueueTest, SinglePushGetNotEmpty) { - MultiTypeQueue queue(BIG_QUEUE_QTTY); + MultiTypeQueue queue(BIG_QUEUE_CAPACITY); const MessageType messageType {MessageType::STATELESS}; const Message messageToSend {messageType, baseDataContent}; - queue.timeoutPush(messageToSend); + EXPECT_EQ(queue.timeoutPush(messageToSend), 1); auto messageResponse = queue.getLastMessage(MessageType::STATELESS); auto typeSend = messageToSend.type; @@ -154,72 +153,44 @@ TEST_F(QueueTest, SinglePushGetNotEmpty) // timeoutPush and pop on a non-full queue TEST_F(QueueTest, SinglePushPopEmpty) { - MultiTypeQueue queue(BIG_QUEUE_QTTY); + MultiTypeQueue queue(BIG_QUEUE_CAPACITY); const MessageType messageType {MessageType::STATELESS}; const Message messageToSend {messageType, baseDataContent}; - queue.timeoutPush(messageToSend); + EXPECT_EQ(queue.timeoutPush(messageToSend), 1); auto messageResponse = queue.getLastMessage(MessageType::STATELESS); - - auto typeSend = messageToSend.type; - auto typeReceived = messageResponse.type; - EXPECT_TRUE(typeSend == typeReceived); - auto dataResponse = messageResponse.data.at(0).at("data"); EXPECT_EQ(dataResponse, baseDataContent); + EXPECT_EQ(messageType, messageResponse.type); + + auto messageResponseStateFul = queue.getLastMessage(MessageType::STATEFUL); + // TODO: this behavior can be change to return an empty message (type and module empty) + EXPECT_EQ(messageResponseStateFul.type, MessageType::STATEFUL); + EXPECT_EQ(messageResponseStateFul.data, "{}"_json); queue.popLastMessage(MessageType::STATELESS); EXPECT_TRUE(queue.isEmptyByType(MessageType::STATELESS)); -} - -// Push, pop and check the queue is empty -TEST_F(QueueTest, SinglePushPop) -{ - MultiTypeQueue queue(BIG_QUEUE_QTTY); - const MessageType messageType {MessageType::STATELESS}; - const Message messageToSend {messageType, baseDataContent}; - - queue.timeoutPush(messageToSend); - EXPECT_FALSE(queue.isEmptyByType(MessageType::STATELESS)); queue.popLastMessage(MessageType::STATELESS); EXPECT_TRUE(queue.isEmptyByType(MessageType::STATELESS)); } -// get and pop on a empty queue -TEST_F(QueueTest, SingleGetPopOnEmpty) -{ - MultiTypeQueue queue(BIG_QUEUE_QTTY); - const MessageType messageType {MessageType::STATELESS}; - const Message messageToSend {messageType, baseDataContent}; - - queue.timeoutPush(messageToSend); - EXPECT_FALSE(queue.isEmptyByType(MessageType::STATELESS)); - EXPECT_TRUE(queue.isEmptyByType(MessageType::STATEFUL)); - - auto messageResponse = queue.getLastMessage(MessageType::STATEFUL); - EXPECT_EQ(messageResponse.type, MessageType::STATEFUL); - EXPECT_EQ(messageResponse.data, "{}"_json); - - queue.popLastMessage(MessageType::STATEFUL); - EXPECT_TRUE(queue.isEmptyByType(MessageType::STATEFUL)); -} - TEST_F(QueueTest, SinglePushGetWithModule) { - MultiTypeQueue queue(BIG_QUEUE_QTTY); + MultiTypeQueue queue(BIG_QUEUE_CAPACITY); const MessageType messageType {MessageType::STATELESS}; const std::string moduleFakeName = "fake-module"; const std::string moduleName = "module"; const Message messageToSend {messageType, baseDataContent, moduleName}; - queue.timeoutPush(messageToSend); + EXPECT_EQ(queue.timeoutPush(messageToSend), 1); auto messageResponseWrongModule = queue.getLastMessage(MessageType::STATELESS, moduleFakeName); auto typeSend = messageToSend.type; auto typeReceived = messageResponseWrongModule.type; EXPECT_TRUE(typeSend == typeReceived); + EXPECT_EQ(messageResponseWrongModule.moduleName, moduleFakeName); EXPECT_EQ(messageResponseWrongModule.data, "{}"_json); auto messageResponseCorrectModule = queue.getLastMessage(MessageType::STATELESS, moduleName); @@ -233,34 +204,34 @@ TEST_F(QueueTest, SinglePushGetWithModule) // Push, get and check while the queue is full TEST_F(QueueTest, SinglePushPopFullWithTimeout) { - MultiTypeQueue queue(SMALL_QUEUE_QTTY); + MultiTypeQueue queue(SMALL_QUEUE_CAPACITY); // complete the queue with messages const MessageType messageType {MessageType::COMMAND}; for (int i : {1, 2}) { const json dataContent = R"({"Data" : "for COMMAND)" + std::to_string(i) + R"("})"; - queue.timeoutPush({messageType, dataContent}); + EXPECT_EQ(queue.timeoutPush({messageType, dataContent}), 1); } const json dataContent = R"({"Data" : "for COMMAND3"})"; Message exampleMessage {messageType, dataContent}; - queue.timeoutPush({messageType, dataContent}, true); + EXPECT_EQ(queue.timeoutPush({messageType, dataContent}, true), 0); auto items = queue.getItemsByType(MessageType::COMMAND); - EXPECT_EQ(items, SMALL_QUEUE_QTTY); + EXPECT_EQ(items, SMALL_QUEUE_CAPACITY); EXPECT_TRUE(queue.isFullByType(MessageType::COMMAND)); EXPECT_TRUE(queue.isEmptyByType(MessageType::STATELESS)); queue.popLastMessage(MessageType::COMMAND); items = queue.getItemsByType(MessageType::COMMAND); - EXPECT_NE(items, SMALL_QUEUE_QTTY); + EXPECT_NE(items, SMALL_QUEUE_CAPACITY); } // Accesing different types of queues TEST_F(QueueTest, MultithreadDifferentType) { - MultiTypeQueue queue(BIG_QUEUE_QTTY); + MultiTypeQueue queue(BIG_QUEUE_CAPACITY); auto consumerStateLess = [&](int& count) { @@ -283,8 +254,8 @@ TEST_F(QueueTest, MultithreadDifferentType) for (int i = 0; i < count; ++i) { const json dataContent = R"({{"Data", "for STATELESS)" + std::to_string(i) + R"("}})"; - queue.timeoutPush(Message(MessageType::STATELESS, dataContent)); - queue.timeoutPush(Message(MessageType::STATEFUL, dataContent)); + EXPECT_EQ(queue.timeoutPush(Message(MessageType::STATELESS, dataContent)), 1); + EXPECT_EQ(queue.timeoutPush(Message(MessageType::STATEFUL, dataContent)), 1); } }; @@ -335,7 +306,7 @@ TEST_F(QueueTest, MultithreadDifferentType) // Accesing same queue TEST_F(QueueTest, MultithreadSameType) { - MultiTypeQueue queue(BIG_QUEUE_QTTY); + MultiTypeQueue queue(BIG_QUEUE_CAPACITY); auto consumerCommand1 = [&](int& count) { @@ -358,7 +329,7 @@ TEST_F(QueueTest, MultithreadSameType) for (int i = 0; i < count; ++i) { const json dataContent = R"({{"Data": "for COMMAND)" + std::to_string(i) + R"("}})"; - queue.timeoutPush(Message(MessageType::COMMAND, dataContent)); + EXPECT_EQ(queue.timeoutPush(Message(MessageType::COMMAND, dataContent)), 1); } }; @@ -389,13 +360,12 @@ TEST_F(QueueTest, MultithreadSameType) // several gets, checks and pops TEST_F(QueueTest, PushMultipleSeveralSingleGets) { - MultiTypeQueue queue(BIG_QUEUE_QTTY); + MultiTypeQueue queue(BIG_QUEUE_CAPACITY); const MessageType messageType {MessageType::STATELESS}; // TODO: double check array of objects - const json multipleDataContent = {"content 1", "content 2", "content 3"}; const Message messageToSend {messageType, multipleDataContent}; - queue.timeoutPush(messageToSend); + EXPECT_EQ(3, queue.timeoutPush(messageToSend)); for (int i : {0, 1, 2}) { @@ -411,7 +381,7 @@ TEST_F(QueueTest, PushMultipleSeveralSingleGets) TEST_F(QueueTest, PushMultipleWithMessageVector) { - MultiTypeQueue queue(BIG_QUEUE_QTTY); + MultiTypeQueue queue(BIG_QUEUE_CAPACITY); std::vector messages; const MessageType messageType {MessageType::STATELESS}; @@ -421,21 +391,21 @@ TEST_F(QueueTest, PushMultipleWithMessageVector) messages.push_back({messageType, multipleDataContent}); } EXPECT_EQ(messages.size(), 3); - queue.timeoutPush(messages); + EXPECT_EQ(3, queue.timeoutPush(messages)); EXPECT_EQ(queue.getItemsByType(MessageType::STATELESS), 3); } // Push Multiple, pop multiples TEST_F(QueueTest, PushMultipleGetMultiple) { - MultiTypeQueue queue(BIG_QUEUE_QTTY); + MultiTypeQueue queue(BIG_QUEUE_CAPACITY); const MessageType messageType {MessageType::STATELESS}; - const json multipleDataContent = {"content 1", "content 2", "content 3"}; const Message messageToSend {messageType, multipleDataContent}; - queue.timeoutPush(messageToSend); - - EXPECT_TRUE(queue.popNMessages(MessageType::STATELESS, 3)); + EXPECT_EQ(3, queue.timeoutPush(messageToSend)); + EXPECT_EQ(queue.getItemsByType(MessageType::STATELESS), 3); + EXPECT_EQ(queue.popNMessages(MessageType::STATELESS, 1), 1); + EXPECT_EQ(queue.popNMessages(MessageType::STATELESS, 3), 2); EXPECT_TRUE(queue.isEmptyByType(MessageType::STATELESS)); EXPECT_EQ(0, queue.getItemsByType(MessageType::STATELESS)); } @@ -443,13 +413,12 @@ TEST_F(QueueTest, PushMultipleGetMultiple) // Push Multiple, pop multiples TEST_F(QueueTest, PushMultipleGetMultipleWithModule) { - MultiTypeQueue queue(BIG_QUEUE_QTTY); + MultiTypeQueue queue(BIG_QUEUE_CAPACITY); const MessageType messageType {MessageType::STATELESS}; - const json multipleDataContent = {"content 1", "content 2", "content 3"}; const std::string moduleName = "testModule"; const Message messageToSend {messageType, multipleDataContent, moduleName}; - queue.timeoutPush(messageToSend); + EXPECT_EQ(3, queue.timeoutPush(messageToSend)); // Altough we're asking for 10 messages only the availables are returned. auto messagesReceived = queue.getNMessages(MessageType::STATELESS, 10); @@ -466,7 +435,7 @@ TEST_F(QueueTest, PushMultipleGetMultipleWithModule) TEST_F(QueueTest, PushSinglesleGetMultipleWithModule) { - MultiTypeQueue queue(BIG_QUEUE_QTTY); + MultiTypeQueue queue(BIG_QUEUE_CAPACITY); for (std::string i : {"1", "2", "3", "4", "5"}) { @@ -474,7 +443,7 @@ TEST_F(QueueTest, PushSinglesleGetMultipleWithModule) const json multipleDataContent = {"content-" + i}; const std::string moduleName = "module-" + i; const Message messageToSend {messageType, multipleDataContent, moduleName}; - queue.timeoutPush(messageToSend); + EXPECT_EQ(1, queue.timeoutPush(messageToSend)); } auto messagesReceived = queue.getNMessages(MessageType::STATELESS, 10); @@ -482,7 +451,9 @@ TEST_F(QueueTest, PushSinglesleGetMultipleWithModule) int i = 5; for (auto singleMessage : messagesReceived) { - EXPECT_EQ("content-" + std::to_string(i--), singleMessage.data.at("data")); + auto val = i--; + EXPECT_EQ("content-" + std::to_string(val), singleMessage.data.at("data")); + EXPECT_EQ("module-" + std::to_string(val), singleMessage.data.at("module")); } auto messageReceivedContent1 = queue.getNMessages(MessageType::STATELESS, 10, "module-1"); diff --git a/src/agent/queue/tests/sqlitestorage_test.cpp b/src/agent/queue/tests/sqlitestorage_test.cpp index 4f0fe153ca1..5f04df57d40 100644 --- a/src/agent/queue/tests/sqlitestorage_test.cpp +++ b/src/agent/queue/tests/sqlitestorage_test.cpp @@ -57,18 +57,18 @@ class SQLiteStorageTest : public ::testing::Test TEST_F(SQLiteStorageTest, StoreSingleMessage) { json message = {{"key", "value"}}; - EXPECT_NO_THROW(storage->Store(message, tableName)); + EXPECT_EQ(storage->Store(message, tableName), 1); EXPECT_EQ(storage->GetElementCount(tableName), 1); - EXPECT_NO_THROW(storage->Store(message, tableName)); + EXPECT_EQ(storage->Store(message, tableName), 1); EXPECT_EQ(storage->GetElementCount(tableName), 2); } TEST_F(SQLiteStorageTest, StoreSingleMessageWithModule) { json message = {{"key", "value"}}; - EXPECT_NO_THROW(storage->Store(message, tableName, moduleName)); + EXPECT_EQ(storage->Store(message, tableName, moduleName), 1); EXPECT_EQ(storage->GetElementCount(tableName), 1); - EXPECT_NO_THROW(storage->Store(message, tableName)); + EXPECT_EQ(storage->Store(message, tableName), 1); EXPECT_EQ(storage->GetElementCount(tableName), 2); EXPECT_EQ(storage->GetElementCount(tableName, "unavailableModuleName"), 0); EXPECT_EQ(storage->GetElementCount(tableName, moduleName), 1); @@ -79,7 +79,7 @@ TEST_F(SQLiteStorageTest, StoreMultipleMessages) json messages = json::array(); messages.push_back({{"key", "value1"}}); messages.push_back({{"key", "value2"}}); - EXPECT_NO_THROW(storage->Store(messages, tableName)); + EXPECT_EQ(storage->Store(messages, tableName), 2); EXPECT_EQ(storage->GetElementCount(tableName), 2); } @@ -88,7 +88,7 @@ TEST_F(SQLiteStorageTest, StoreMultipleMessagesWithModule) json messages = json::array(); messages.push_back({{"key", "value1"}}); messages.push_back({{"key", "value2"}}); - EXPECT_NO_THROW(storage->Store(messages, tableName, moduleName)); + EXPECT_EQ(storage->Store(messages, tableName, moduleName), 2); EXPECT_EQ(storage->GetElementCount(tableName), 2); EXPECT_EQ(storage->GetElementCount(tableName, moduleName), 2); EXPECT_EQ(storage->GetElementCount(tableName, "unavailableModuleName"), 0); @@ -97,7 +97,7 @@ TEST_F(SQLiteStorageTest, StoreMultipleMessagesWithModule) TEST_F(SQLiteStorageTest, RetrieveMessage) { json message = {{"key", "value"}}; - storage->Store(message, tableName); + EXPECT_EQ(storage->Store(message, tableName), 1); json retrievedMessage = storage->Retrieve(1, tableName); EXPECT_EQ(retrievedMessage.at("data").at("key"), "value"); } @@ -143,7 +143,7 @@ TEST_F(SQLiteStorageTest, RetrieveMultipleMessagesWithModule) TEST_F(SQLiteStorageTest, RemoveMessage) { json message = {{"key", "value"}}; - storage->Store(message, tableName); + EXPECT_EQ(storage->Store(message, tableName), 1); EXPECT_EQ(storage->Remove(1, tableName), 1); EXPECT_EQ(storage->GetElementCount(tableName), 0); } @@ -151,9 +151,9 @@ TEST_F(SQLiteStorageTest, RemoveMessage) TEST_F(SQLiteStorageTest, RemoveMessageWithModule) { json message = {{"key", "value"}}; - storage->Store(message, tableName, moduleName); + EXPECT_EQ(storage->Store(message, tableName, moduleName), 1); EXPECT_EQ(storage->Remove(1, tableName), 1); - storage->Store(message, tableName, moduleName); + EXPECT_EQ(storage->Store(message, tableName, moduleName), 1); EXPECT_EQ(storage->Remove(1, tableName, "unavailableModuleName"), 1); EXPECT_EQ(storage->GetElementCount(tableName), 1); EXPECT_EQ(storage->Remove(1, tableName, moduleName), 1); @@ -165,7 +165,7 @@ TEST_F(SQLiteStorageTest, RemoveMultipleMessages) json messages = json::array(); messages.push_back({{"key", "value1"}}); messages.push_back({{"key", "value2"}}); - storage->Store(messages, tableName); + EXPECT_EQ(storage->Store(messages, tableName), 2); EXPECT_EQ(storage->RemoveMultiple(2, tableName), 2); EXPECT_EQ(storage->GetElementCount(tableName), 0); } @@ -175,8 +175,8 @@ TEST_F(SQLiteStorageTest, RemoveMultipleMessagesWithModule) json messages = json::array(); messages.push_back({{"key", "value1"}}); messages.push_back({{"key", "value2"}}); - storage->Store(messages, tableName, moduleName); - EXPECT_EQ(storage->RemoveMultiple(2, tableName, "unavailableModuleName"), 2); + EXPECT_EQ(storage->Store(messages, tableName, moduleName), 2); + EXPECT_EQ(storage->RemoveMultiple(2, tableName, "unavailableModuleName"), 0); EXPECT_EQ(storage->GetElementCount(tableName), 2); EXPECT_EQ(storage->RemoveMultiple(2, tableName, moduleName), 2); EXPECT_EQ(storage->GetElementCount(tableName), 0); @@ -185,14 +185,14 @@ TEST_F(SQLiteStorageTest, RemoveMultipleMessagesWithModule) TEST_F(SQLiteStorageTest, GetElementCount) { json message = {{"key", "value"}}; - storage->Store(message, tableName); + EXPECT_EQ(storage->Store(message, tableName), 1); EXPECT_EQ(storage->GetElementCount(tableName), 1); } TEST_F(SQLiteStorageTest, GetElementCountWithModule) { json message = {{"key", "value"}}; - storage->Store(message, tableName, moduleName); + EXPECT_EQ(storage->Store(message, tableName, moduleName), 1); EXPECT_EQ(storage->GetElementCount(tableName), 1); EXPECT_EQ(storage->GetElementCount(tableName, moduleName), 1); EXPECT_EQ(storage->GetElementCount(tableName, "unavailableModuleName"), 0); From 117ce4f0e8d06621d725236bfb26cec835d99fd6 Mon Sep 17 00:00:00 2001 From: LucioDonda Date: Fri, 9 Aug 2024 12:12:25 -0300 Subject: [PATCH 24/31] style: queue functions renaming --- src/agent/queue/include/queue.hpp | 18 ++--- src/agent/queue/src/queue.cpp | 20 ++--- src/agent/queue/tests/queue_test.cpp | 116 +++++++++++++-------------- 3 files changed, 77 insertions(+), 77 deletions(-) diff --git a/src/agent/queue/include/queue.hpp b/src/agent/queue/include/queue.hpp index d99d6af9377..482a37691d6 100644 --- a/src/agent/queue/include/queue.hpp +++ b/src/agent/queue/include/queue.hpp @@ -93,7 +93,7 @@ class MultiTypeQueue * @param shouldWait when true, the function will wait until the message is pushed * @return int number of messages pushed */ - int timeoutPush(Message message, bool shouldWait = false); + int push(Message message, bool shouldWait = false); /** * @brief pushes a vector of messages @@ -102,7 +102,7 @@ class MultiTypeQueue * @param shouldWait when true, the function will wait until the message is pushed * @return int number of messages pushed */ - int timeoutPush(std::vector messages); + int push(std::vector messages); /** * @brief Get the Last Message object @@ -110,7 +110,7 @@ class MultiTypeQueue * @param type * @return Message */ - Message getLastMessage(MessageType type, const std::string module = ""); + Message getNext(MessageType type, const std::string module = ""); /** * @brief Returns N messages from a queue @@ -120,7 +120,7 @@ class MultiTypeQueue * @param messageQuantity quantity of messages to return * @return Message Json data othe messages fetched */ - std::vector getNMessages(MessageType type, int messageQuantity, const std::string moduleName = ""); + std::vector getNextN(MessageType type, int messageQuantity, const std::string moduleName = ""); /** * @brief deletes a message from a queue @@ -130,7 +130,7 @@ class MultiTypeQueue * @return true popped succesfully * @return false wasn't able to pop message */ - bool popLastMessage(MessageType type, const std::string moduleName = ""); + bool pop(MessageType type, const std::string moduleName = ""); /** * @brief @@ -140,7 +140,7 @@ class MultiTypeQueue * @param messageQuantity * @return Number of messages deleted */ - int popNMessages(MessageType type, int messageQuantity, const std::string moduleName = ""); + int popN(MessageType type, int messageQuantity, const std::string moduleName = ""); /** * @brief Checks emptyness of a queue @@ -150,7 +150,7 @@ class MultiTypeQueue * @return true when queue empty * @return false otherwise */ - bool isEmptyByType(MessageType type, const std::string moduleName = ""); + bool isEmpty(MessageType type, const std::string moduleName = ""); /** * @brief Checks fullness of a queue @@ -160,7 +160,7 @@ class MultiTypeQueue * @return true when queue is full * @return false otherwise */ - bool isFullByType(MessageType type, const std::string moduleName = ""); + bool isFull(MessageType type, const std::string moduleName = ""); /** * @brief Get the Items By Type object @@ -170,7 +170,7 @@ class MultiTypeQueue * @return true * @return false */ - int getItemsByType(MessageType type, const std::string moduleName = ""); + int storedItems(MessageType type, const std::string moduleName = ""); }; #endif // QUEUE_H diff --git a/src/agent/queue/src/queue.cpp b/src/agent/queue/src/queue.cpp index fe670cf0d21..ca001131f07 100644 --- a/src/agent/queue/src/queue.cpp +++ b/src/agent/queue/src/queue.cpp @@ -12,7 +12,7 @@ #include "queue.hpp" -int MultiTypeQueue::timeoutPush(Message message, bool shouldWait) +int MultiTypeQueue::push(Message message, bool shouldWait) { int result = 0; @@ -63,17 +63,17 @@ int MultiTypeQueue::timeoutPush(Message message, bool shouldWait) return result; } -int MultiTypeQueue::timeoutPush(std::vector messages) +int MultiTypeQueue::push(std::vector messages) { int result = 0; for (const auto& singleMessage : messages) { - result += timeoutPush(singleMessage); + result += push(singleMessage); } return result; } -Message MultiTypeQueue::getLastMessage(MessageType type, const std::string moduleName) +Message MultiTypeQueue::getNext(MessageType type, const std::string moduleName) { Message result(type, "{}"_json, moduleName); if (m_mapMessageTypeName.contains(type)) @@ -96,7 +96,7 @@ Message MultiTypeQueue::getLastMessage(MessageType type, const std::string modul return result; } -std::vector MultiTypeQueue::getNMessages(MessageType type, int messageQuantity, const std::string moduleName) +std::vector MultiTypeQueue::getNextN(MessageType type, int messageQuantity, const std::string moduleName) { std::vector result; if (m_mapMessageTypeName.contains(type)) @@ -121,7 +121,7 @@ std::vector MultiTypeQueue::getNMessages(MessageType type, int messageQ return result; } -bool MultiTypeQueue::popLastMessage(MessageType type, const std::string moduleName) +bool MultiTypeQueue::pop(MessageType type, const std::string moduleName) { bool result = false; if (m_mapMessageTypeName.contains(type)) @@ -137,7 +137,7 @@ bool MultiTypeQueue::popLastMessage(MessageType type, const std::string moduleNa return result; } -int MultiTypeQueue::popNMessages(MessageType type, int messageQuantity, const std::string moduleName) +int MultiTypeQueue::popN(MessageType type, int messageQuantity, const std::string moduleName) { int result = 0; if (m_mapMessageTypeName.contains(type)) @@ -152,7 +152,7 @@ int MultiTypeQueue::popNMessages(MessageType type, int messageQuantity, const st return result; } -bool MultiTypeQueue::isEmptyByType(MessageType type, const std::string moduleName) +bool MultiTypeQueue::isEmpty(MessageType type, const std::string moduleName) { if (m_mapMessageTypeName.contains(type)) { @@ -166,7 +166,7 @@ bool MultiTypeQueue::isEmptyByType(MessageType type, const std::string moduleNam return false; } -bool MultiTypeQueue::isFullByType(MessageType type, const std::string moduleName) +bool MultiTypeQueue::isFull(MessageType type, const std::string moduleName) { if (m_mapMessageTypeName.contains(type)) { @@ -180,7 +180,7 @@ bool MultiTypeQueue::isFullByType(MessageType type, const std::string moduleName return false; } -int MultiTypeQueue::getItemsByType(MessageType type, const std::string moduleName) +int MultiTypeQueue::storedItems(MessageType type, const std::string moduleName) { if (m_mapMessageTypeName.contains(type)) { diff --git a/src/agent/queue/tests/queue_test.cpp b/src/agent/queue/tests/queue_test.cpp index bcc1dd4e81b..1023c524fa8 100644 --- a/src/agent/queue/tests/queue_test.cpp +++ b/src/agent/queue/tests/queue_test.cpp @@ -137,8 +137,8 @@ TEST_F(QueueTest, SinglePushGetNotEmpty) const MessageType messageType {MessageType::STATELESS}; const Message messageToSend {messageType, baseDataContent}; - EXPECT_EQ(queue.timeoutPush(messageToSend), 1); - auto messageResponse = queue.getLastMessage(MessageType::STATELESS); + EXPECT_EQ(queue.push(messageToSend), 1); + auto messageResponse = queue.getNext(MessageType::STATELESS); auto typeSend = messageToSend.type; auto typeReceived = messageResponse.type; @@ -147,32 +147,32 @@ TEST_F(QueueTest, SinglePushGetNotEmpty) auto dataResponse = messageResponse.data.at(0).at("data"); EXPECT_EQ(dataResponse, baseDataContent); - EXPECT_FALSE(queue.isEmptyByType(MessageType::STATELESS)); + EXPECT_FALSE(queue.isEmpty(MessageType::STATELESS)); } -// timeoutPush and pop on a non-full queue +// push and pop on a non-full queue TEST_F(QueueTest, SinglePushPopEmpty) { MultiTypeQueue queue(BIG_QUEUE_CAPACITY); const MessageType messageType {MessageType::STATELESS}; const Message messageToSend {messageType, baseDataContent}; - EXPECT_EQ(queue.timeoutPush(messageToSend), 1); - auto messageResponse = queue.getLastMessage(MessageType::STATELESS); + EXPECT_EQ(queue.push(messageToSend), 1); + auto messageResponse = queue.getNext(MessageType::STATELESS); auto dataResponse = messageResponse.data.at(0).at("data"); EXPECT_EQ(dataResponse, baseDataContent); EXPECT_EQ(messageType, messageResponse.type); - auto messageResponseStateFul = queue.getLastMessage(MessageType::STATEFUL); + auto messageResponseStateFul = queue.getNext(MessageType::STATEFUL); // TODO: this behavior can be change to return an empty message (type and module empty) EXPECT_EQ(messageResponseStateFul.type, MessageType::STATEFUL); EXPECT_EQ(messageResponseStateFul.data, "{}"_json); - queue.popLastMessage(MessageType::STATELESS); - EXPECT_TRUE(queue.isEmptyByType(MessageType::STATELESS)); + queue.pop(MessageType::STATELESS); + EXPECT_TRUE(queue.isEmpty(MessageType::STATELESS)); - queue.popLastMessage(MessageType::STATELESS); - EXPECT_TRUE(queue.isEmptyByType(MessageType::STATELESS)); + queue.pop(MessageType::STATELESS); + EXPECT_TRUE(queue.isEmpty(MessageType::STATELESS)); } TEST_F(QueueTest, SinglePushGetWithModule) @@ -183,8 +183,8 @@ TEST_F(QueueTest, SinglePushGetWithModule) const std::string moduleName = "module"; const Message messageToSend {messageType, baseDataContent, moduleName}; - EXPECT_EQ(queue.timeoutPush(messageToSend), 1); - auto messageResponseWrongModule = queue.getLastMessage(MessageType::STATELESS, moduleFakeName); + EXPECT_EQ(queue.push(messageToSend), 1); + auto messageResponseWrongModule = queue.getNext(MessageType::STATELESS, moduleFakeName); auto typeSend = messageToSend.type; auto typeReceived = messageResponseWrongModule.type; @@ -193,7 +193,7 @@ TEST_F(QueueTest, SinglePushGetWithModule) EXPECT_EQ(messageResponseWrongModule.moduleName, moduleFakeName); EXPECT_EQ(messageResponseWrongModule.data, "{}"_json); - auto messageResponseCorrectModule = queue.getLastMessage(MessageType::STATELESS, moduleName); + auto messageResponseCorrectModule = queue.getNext(MessageType::STATELESS, moduleName); auto dataResponse = messageResponseCorrectModule.data.at(0).at("data"); EXPECT_EQ(dataResponse, baseDataContent); @@ -211,20 +211,20 @@ TEST_F(QueueTest, SinglePushPopFullWithTimeout) for (int i : {1, 2}) { const json dataContent = R"({"Data" : "for COMMAND)" + std::to_string(i) + R"("})"; - EXPECT_EQ(queue.timeoutPush({messageType, dataContent}), 1); + EXPECT_EQ(queue.push({messageType, dataContent}), 1); } const json dataContent = R"({"Data" : "for COMMAND3"})"; Message exampleMessage {messageType, dataContent}; - EXPECT_EQ(queue.timeoutPush({messageType, dataContent}, true), 0); + EXPECT_EQ(queue.push({messageType, dataContent}, true), 0); - auto items = queue.getItemsByType(MessageType::COMMAND); + auto items = queue.storedItems(MessageType::COMMAND); EXPECT_EQ(items, SMALL_QUEUE_CAPACITY); - EXPECT_TRUE(queue.isFullByType(MessageType::COMMAND)); - EXPECT_TRUE(queue.isEmptyByType(MessageType::STATELESS)); + EXPECT_TRUE(queue.isFull(MessageType::COMMAND)); + EXPECT_TRUE(queue.isEmpty(MessageType::STATELESS)); - queue.popLastMessage(MessageType::COMMAND); - items = queue.getItemsByType(MessageType::COMMAND); + queue.pop(MessageType::COMMAND); + items = queue.storedItems(MessageType::COMMAND); EXPECT_NE(items, SMALL_QUEUE_CAPACITY); } @@ -237,7 +237,7 @@ TEST_F(QueueTest, MultithreadDifferentType) { for (int i = 0; i < count; ++i) { - queue.popLastMessage(MessageType::STATELESS); + queue.pop(MessageType::STATELESS); } }; @@ -245,7 +245,7 @@ TEST_F(QueueTest, MultithreadDifferentType) { for (int i = 0; i < count; ++i) { - queue.popLastMessage(MessageType::STATEFUL); + queue.pop(MessageType::STATEFUL); } }; @@ -254,8 +254,8 @@ TEST_F(QueueTest, MultithreadDifferentType) for (int i = 0; i < count; ++i) { const json dataContent = R"({{"Data", "for STATELESS)" + std::to_string(i) + R"("}})"; - EXPECT_EQ(queue.timeoutPush(Message(MessageType::STATELESS, dataContent)), 1); - EXPECT_EQ(queue.timeoutPush(Message(MessageType::STATEFUL, dataContent)), 1); + EXPECT_EQ(queue.push(Message(MessageType::STATELESS, dataContent)), 1); + EXPECT_EQ(queue.push(Message(MessageType::STATEFUL, dataContent)), 1); } }; @@ -277,10 +277,10 @@ TEST_F(QueueTest, MultithreadDifferentType) consumerThread2.join(); } - EXPECT_NE(0, queue.getItemsByType(MessageType::STATELESS)); - EXPECT_NE(0, queue.getItemsByType(MessageType::STATEFUL)); - EXPECT_FALSE(queue.isEmptyByType(MessageType::STATELESS)); - EXPECT_FALSE(queue.isEmptyByType(MessageType::STATEFUL)); + EXPECT_NE(0, queue.storedItems(MessageType::STATELESS)); + EXPECT_NE(0, queue.storedItems(MessageType::STATEFUL)); + EXPECT_FALSE(queue.isEmpty(MessageType::STATELESS)); + EXPECT_FALSE(queue.isEmpty(MessageType::STATEFUL)); // Consume the rest of the messages std::thread consumerThread12(consumerStateLess, std::ref(itemsToConsume)); @@ -297,10 +297,10 @@ TEST_F(QueueTest, MultithreadDifferentType) } // FIXME: this doesn't match - EXPECT_EQ(0, queue.getItemsByType(MessageType::STATELESS)); - EXPECT_EQ(0, queue.getItemsByType(MessageType::STATEFUL)); - EXPECT_TRUE(queue.isEmptyByType(MessageType::STATELESS)); - EXPECT_TRUE(queue.isEmptyByType(MessageType::STATEFUL)); + EXPECT_EQ(0, queue.storedItems(MessageType::STATELESS)); + EXPECT_EQ(0, queue.storedItems(MessageType::STATEFUL)); + EXPECT_TRUE(queue.isEmpty(MessageType::STATELESS)); + EXPECT_TRUE(queue.isEmpty(MessageType::STATEFUL)); } // Accesing same queue @@ -312,7 +312,7 @@ TEST_F(QueueTest, MultithreadSameType) { for (int i = 0; i < count; ++i) { - queue.popLastMessage(MessageType::COMMAND); + queue.pop(MessageType::COMMAND); } }; @@ -320,7 +320,7 @@ TEST_F(QueueTest, MultithreadSameType) { for (int i = 0; i < count; ++i) { - queue.popLastMessage(MessageType::COMMAND); + queue.pop(MessageType::COMMAND); } }; @@ -329,7 +329,7 @@ TEST_F(QueueTest, MultithreadSameType) for (int i = 0; i < count; ++i) { const json dataContent = R"({{"Data": "for COMMAND)" + std::to_string(i) + R"("}})"; - EXPECT_EQ(queue.timeoutPush(Message(MessageType::COMMAND, dataContent)), 1); + EXPECT_EQ(queue.push(Message(MessageType::COMMAND, dataContent)), 1); } }; @@ -338,7 +338,7 @@ TEST_F(QueueTest, MultithreadSameType) messageProducer(itemsToInsert); - EXPECT_EQ(itemsToInsert, queue.getItemsByType(MessageType::COMMAND)); + EXPECT_EQ(itemsToInsert, queue.storedItems(MessageType::COMMAND)); std::thread consumerThread1(consumerCommand1, std::ref(itemsToConsume)); std::thread messageProducerThread1(consumerCommand2, std::ref(itemsToConsume)); @@ -353,7 +353,7 @@ TEST_F(QueueTest, MultithreadSameType) consumerThread1.join(); } - EXPECT_TRUE(queue.isEmptyByType(MessageType::COMMAND)); + EXPECT_TRUE(queue.isEmpty(MessageType::COMMAND)); } // Push Multiple with single message and data array, @@ -365,18 +365,18 @@ TEST_F(QueueTest, PushMultipleSeveralSingleGets) // TODO: double check array of objects const Message messageToSend {messageType, multipleDataContent}; - EXPECT_EQ(3, queue.timeoutPush(messageToSend)); + EXPECT_EQ(3, queue.push(messageToSend)); for (int i : {0, 1, 2}) { - auto messageResponse = queue.getLastMessage(MessageType::STATELESS); + auto messageResponse = queue.getNext(MessageType::STATELESS); auto responseData = messageResponse.data.at(0).at("data"); auto sentData = messageToSend.data[i].template get(); EXPECT_EQ(responseData, sentData); - queue.popLastMessage(MessageType::STATELESS); + queue.pop(MessageType::STATELESS); } - EXPECT_EQ(queue.getItemsByType(MessageType::STATELESS), 0); + EXPECT_EQ(queue.storedItems(MessageType::STATELESS), 0); } TEST_F(QueueTest, PushMultipleWithMessageVector) @@ -391,8 +391,8 @@ TEST_F(QueueTest, PushMultipleWithMessageVector) messages.push_back({messageType, multipleDataContent}); } EXPECT_EQ(messages.size(), 3); - EXPECT_EQ(3, queue.timeoutPush(messages)); - EXPECT_EQ(queue.getItemsByType(MessageType::STATELESS), 3); + EXPECT_EQ(3, queue.push(messages)); + EXPECT_EQ(queue.storedItems(MessageType::STATELESS), 3); } // Push Multiple, pop multiples @@ -402,12 +402,12 @@ TEST_F(QueueTest, PushMultipleGetMultiple) const MessageType messageType {MessageType::STATELESS}; const Message messageToSend {messageType, multipleDataContent}; - EXPECT_EQ(3, queue.timeoutPush(messageToSend)); - EXPECT_EQ(queue.getItemsByType(MessageType::STATELESS), 3); - EXPECT_EQ(queue.popNMessages(MessageType::STATELESS, 1), 1); - EXPECT_EQ(queue.popNMessages(MessageType::STATELESS, 3), 2); - EXPECT_TRUE(queue.isEmptyByType(MessageType::STATELESS)); - EXPECT_EQ(0, queue.getItemsByType(MessageType::STATELESS)); + EXPECT_EQ(3, queue.push(messageToSend)); + EXPECT_EQ(queue.storedItems(MessageType::STATELESS), 3); + EXPECT_EQ(queue.popN(MessageType::STATELESS, 1), 1); + EXPECT_EQ(queue.popN(MessageType::STATELESS, 3), 2); + EXPECT_TRUE(queue.isEmpty(MessageType::STATELESS)); + EXPECT_EQ(0, queue.storedItems(MessageType::STATELESS)); } // Push Multiple, pop multiples @@ -418,19 +418,19 @@ TEST_F(QueueTest, PushMultipleGetMultipleWithModule) const std::string moduleName = "testModule"; const Message messageToSend {messageType, multipleDataContent, moduleName}; - EXPECT_EQ(3, queue.timeoutPush(messageToSend)); + EXPECT_EQ(3, queue.push(messageToSend)); // Altough we're asking for 10 messages only the availables are returned. - auto messagesReceived = queue.getNMessages(MessageType::STATELESS, 10); + auto messagesReceived = queue.getNextN(MessageType::STATELESS, 10); int i = 3; for (auto singleMessage : messagesReceived) { EXPECT_EQ("content " + std::to_string(i--), singleMessage.data.at("data")); } - EXPECT_EQ(0, queue.getItemsByType(MessageType::STATELESS, "fakemodule")); - EXPECT_EQ(3, queue.getItemsByType(MessageType::STATELESS)); - EXPECT_EQ(3, queue.getItemsByType(MessageType::STATELESS, moduleName)); + EXPECT_EQ(0, queue.storedItems(MessageType::STATELESS, "fakemodule")); + EXPECT_EQ(3, queue.storedItems(MessageType::STATELESS)); + EXPECT_EQ(3, queue.storedItems(MessageType::STATELESS, moduleName)); } TEST_F(QueueTest, PushSinglesleGetMultipleWithModule) @@ -443,10 +443,10 @@ TEST_F(QueueTest, PushSinglesleGetMultipleWithModule) const json multipleDataContent = {"content-" + i}; const std::string moduleName = "module-" + i; const Message messageToSend {messageType, multipleDataContent, moduleName}; - EXPECT_EQ(1, queue.timeoutPush(messageToSend)); + EXPECT_EQ(1, queue.push(messageToSend)); } - auto messagesReceived = queue.getNMessages(MessageType::STATELESS, 10); + auto messagesReceived = queue.getNextN(MessageType::STATELESS, 10); EXPECT_EQ(5, messagesReceived.size()); int i = 5; for (auto singleMessage : messagesReceived) @@ -456,6 +456,6 @@ TEST_F(QueueTest, PushSinglesleGetMultipleWithModule) EXPECT_EQ("module-" + std::to_string(val), singleMessage.data.at("module")); } - auto messageReceivedContent1 = queue.getNMessages(MessageType::STATELESS, 10, "module-1"); + auto messageReceivedContent1 = queue.getNextN(MessageType::STATELESS, 10, "module-1"); EXPECT_EQ(1, messageReceivedContent1.size()); } From 1ce35f5b3d8876bb56b4919e6af56f93324382d0 Mon Sep 17 00:00:00 2001 From: LucioDonda Date: Sun, 11 Aug 2024 19:31:10 -0300 Subject: [PATCH 25/31] feat: push and get awaitable approach with base tests --- src/agent/queue/CMakeLists.txt | 3 +- src/agent/queue/include/queue.hpp | 24 ++++++++- src/agent/queue/src/queue.cpp | 79 +++++++++++++++++++++++++-- src/agent/queue/tests/CMakeLists.txt | 3 +- src/agent/queue/tests/queue_test.cpp | 81 ++++++++++++++++++++++++++++ 5 files changed, 183 insertions(+), 7 deletions(-) diff --git a/src/agent/queue/CMakeLists.txt b/src/agent/queue/CMakeLists.txt index 783b9cc5fd2..f8e4283a011 100644 --- a/src/agent/queue/CMakeLists.txt +++ b/src/agent/queue/CMakeLists.txt @@ -11,6 +11,7 @@ include_directories(${CMAKE_SOURCE_DIR}/include) find_package(SQLiteCpp REQUIRED) find_package(nlohmann_json REQUIRED) find_package(fmt REQUIRED) +find_package(Boost REQUIRED COMPONENTS asio beast) add_library(queue src/sqlitestorage.cpp @@ -18,7 +19,7 @@ add_library(queue ) target_include_directories(queue PRIVATE include ${SQLiteCpp_INCLUDE_DIRS}) -target_link_libraries(queue PRIVATE SQLiteCpp nlohmann_json::nlohmann_json fmt::fmt) +target_link_libraries(queue PRIVATE SQLiteCpp nlohmann_json::nlohmann_json fmt::fmt Boost::asio) if(BUILD_TESTS) add_subdirectory(tests) diff --git a/src/agent/queue/include/queue.hpp b/src/agent/queue/include/queue.hpp index 482a37691d6..8158ad53fd2 100644 --- a/src/agent/queue/include/queue.hpp +++ b/src/agent/queue/include/queue.hpp @@ -9,6 +9,10 @@ #include #include +#include +// #include +#include + #include "shared.hpp" #include "sqlitestorage.h" @@ -95,6 +99,14 @@ class MultiTypeQueue */ int push(Message message, bool shouldWait = false); + /** + * @brief pushes a message + * + * @param message to be pushed + * @return boost::asio::awaitable number of messages pushed + */ + boost::asio::awaitable pushAwaitable(Message message); + /** * @brief pushes a vector of messages * @@ -107,11 +119,19 @@ class MultiTypeQueue /** * @brief Get the Last Message object * - * @param type - * @return Message + * @param type of the queue to be used as source + * @return Message type object taken from the queue */ Message getNext(MessageType type, const std::string module = ""); + /** + * @brief Get the Next Awaitable object + * + * @param type of the queue to be used as source + * @param module module name + * @return boost::asio::awaitable + */ + boost::asio::awaitable getNextAwaitable(MessageType type, const std::string module = ""); /** * @brief Returns N messages from a queue * diff --git a/src/agent/queue/src/queue.cpp b/src/agent/queue/src/queue.cpp index ca001131f07..7c6671dc985 100644 --- a/src/agent/queue/src/queue.cpp +++ b/src/agent/queue/src/queue.cpp @@ -43,9 +43,51 @@ int MultiTypeQueue::push(Message message, bool shouldWait) m_cv.notify_all(); } } - else + } + else + { + result = + m_persistenceDest->Store(message.data, m_mapMessageTypeName.at(message.type), message.moduleName); + m_cv.notify_all(); + } + } + } + else + { + std::cout << "error didn't find the queue" << std::endl; + } + return result; +} + +boost::asio::awaitable MultiTypeQueue::pushAwaitable(Message message) +{ + int result = 0; + boost::asio::steady_timer timer(co_await boost::asio::this_coro::executor); + + if (m_mapMessageTypeName.contains(message.type)) + { + auto sMessageType = m_mapMessageTypeName.at(message.type); + + while (m_persistenceDest->GetElementCount(sMessageType) >= m_maxItems) + { + timer.expires_after(std::chrono::milliseconds(100)); + co_await timer.async_wait(boost::asio::use_awaitable); + } + + auto storedMessages = m_persistenceDest->GetElementCount(sMessageType); + size_t spaceAvailable = (m_maxItems > storedMessages) ? m_maxItems - storedMessages : 0; + if (spaceAvailable) + { + auto messageData = message.data; + if (messageData.is_array()) + { + if (messageData.size() <= spaceAvailable) { - result = false; + for (const auto& singleMessageData : messageData) + { + result += m_persistenceDest->Store(singleMessageData, sMessageType, message.moduleName); + m_cv.notify_all(); + } } } else @@ -60,7 +102,7 @@ int MultiTypeQueue::push(Message message, bool shouldWait) { std::cout << "error didn't find the queue" << std::endl; } - return result; + co_return result; } int MultiTypeQueue::push(std::vector messages) @@ -96,6 +138,37 @@ Message MultiTypeQueue::getNext(MessageType type, const std::string moduleName) return result; } +boost::asio::awaitable MultiTypeQueue::getNextAwaitable(MessageType type, const std::string moduleName) +{ + boost::asio::steady_timer timer(co_await boost::asio::this_coro::executor); + + Message result(type, "{}"_json, moduleName); + if (m_mapMessageTypeName.contains(type)) + { + while (isEmpty(type)) + { + timer.expires_after(std::chrono::milliseconds(100)); + co_await timer.async_wait(boost::asio::use_awaitable); + } + + auto resultData = m_persistenceDest->RetrieveMultiple(1, m_mapMessageTypeName.at(type), moduleName); + if (!resultData.empty()) + { + result.data = resultData; + if (moduleName.empty()) + { + result.moduleName = result.data.at(0).at("module"); + } + } + } + else + { + // TODO: error handling and logging + std::cout << "error didn't find the queue" << std::endl; + } + co_return result; +} + std::vector MultiTypeQueue::getNextN(MessageType type, int messageQuantity, const std::string moduleName) { std::vector result; diff --git a/src/agent/queue/tests/CMakeLists.txt b/src/agent/queue/tests/CMakeLists.txt index 40b9490bdfe..aea65975ffb 100644 --- a/src/agent/queue/tests/CMakeLists.txt +++ b/src/agent/queue/tests/CMakeLists.txt @@ -20,7 +20,8 @@ target_link_libraries(test_queue queue SQLiteCpp nlohmann_json::nlohmann_json - fmt::fmt) + fmt::fmt + Boost::asio) # Create a test executable for SQLiteStorage tests add_executable(test_sqlitestorage diff --git a/src/agent/queue/tests/queue_test.cpp b/src/agent/queue/tests/queue_test.cpp index 1023c524fa8..b18eafb6410 100644 --- a/src/agent/queue/tests/queue_test.cpp +++ b/src/agent/queue/tests/queue_test.cpp @@ -9,6 +9,9 @@ #include #include +#include +#include + #include "queue.hpp" #include "queue_test.hpp" @@ -459,3 +462,81 @@ TEST_F(QueueTest, PushSinglesleGetMultipleWithModule) auto messageReceivedContent1 = queue.getNextN(MessageType::STATELESS, 10, "module-1"); EXPECT_EQ(1, messageReceivedContent1.size()); } + +TEST_F(QueueTest, getNextAwaitable) +{ + MultiTypeQueue queue(BIG_QUEUE_CAPACITY); + boost::asio::io_context io_context; + + const MessageType messageType {MessageType::STATEFUL}; + const json multipleDataContent = {"content-1"}; + const Message messageToSend {messageType, multipleDataContent}; + + // Coroutine that waits till there's a message of the needed type on the queue + boost::asio::co_spawn( + io_context, + [&queue]() -> boost::asio::awaitable + { + auto messageReceived = co_await queue.getNextAwaitable(MessageType::STATELESS); + EXPECT_EQ(messageReceived.data.at(0).at("data"), "content-0"); + }, + boost::asio::detached); + + // Simulate the addition of needed message to the queue after some time + std::thread producer( + [&queue, &io_context]() + { + std::this_thread::sleep_for(std::chrono::seconds(2)); + const MessageType messageType {MessageType::STATELESS}; + const json multipleDataContent = {"content-0"}; + const Message messageToSend {messageType, multipleDataContent}; + EXPECT_EQ(queue.push(messageToSend), 1); + io_context.stop(); + }); + + io_context.run(); + producer.join(); +} + +TEST_F(QueueTest, pushAwaitable) +{ + MultiTypeQueue queue(SMALL_QUEUE_CAPACITY); + boost::asio::io_context io_context; + + // complete the queue with messages + const MessageType messageType {MessageType::STATEFUL}; + for (int i : {1, 2}) + { + const json dataContent = R"({"Data" : "for STATEFUL)" + std::to_string(i) + R"("})"; + EXPECT_EQ(queue.push({messageType, dataContent}), 1); + } + + EXPECT_TRUE(queue.isFull(MessageType::STATEFUL)); + + // Coroutine that waits till there's space on the to push a new messagequeue + boost::asio::co_spawn( + io_context, + [&queue]() -> boost::asio::awaitable + { + const MessageType messageType {MessageType::STATEFUL}; + const json multipleDataContent = {"content-1"}; + const Message messageToSend {messageType, multipleDataContent}; + auto messagesPushed = co_await queue.pushAwaitable(messageToSend); + EXPECT_EQ(messagesPushed, 1); + }, + boost::asio::detached); + + // Simulate poping one message tll there's space to push a new one + std::thread consumer( + [&queue, &io_context]() + { + std::this_thread::sleep_for(std::chrono::seconds(2)); + EXPECT_EQ(queue.pop(MessageType::STATEFUL), 1); + io_context.stop(); + }); + + io_context.run(); + consumer.join(); + + EXPECT_TRUE(queue.isFull(MessageType::STATEFUL)); +} From ad080822b7980550c4ad6444198c2a110f40eecf Mon Sep 17 00:00:00 2001 From: LucioDonda Date: Mon, 12 Aug 2024 13:36:02 -0300 Subject: [PATCH 26/31] feat: awaitable improvement for N messages --- src/agent/queue/CMakeLists.txt | 6 ++--- src/agent/queue/include/queue.hpp | 3 ++- src/agent/queue/src/queue.cpp | 5 ++-- src/agent/queue/src/sqlitestorage.cpp | 4 ++++ src/agent/queue/tests/queue_test.cpp | 34 +++++++++++++-------------- 5 files changed, 28 insertions(+), 24 deletions(-) diff --git a/src/agent/queue/CMakeLists.txt b/src/agent/queue/CMakeLists.txt index f8e4283a011..ebc1e389f57 100644 --- a/src/agent/queue/CMakeLists.txt +++ b/src/agent/queue/CMakeLists.txt @@ -21,6 +21,6 @@ add_library(queue target_include_directories(queue PRIVATE include ${SQLiteCpp_INCLUDE_DIRS}) target_link_libraries(queue PRIVATE SQLiteCpp nlohmann_json::nlohmann_json fmt::fmt Boost::asio) -if(BUILD_TESTS) - add_subdirectory(tests) -endif() +# if(BUILD_TESTS) +# add_subdirectory(tests) +# endif() diff --git a/src/agent/queue/include/queue.hpp b/src/agent/queue/include/queue.hpp index 8158ad53fd2..73d1c03ca77 100644 --- a/src/agent/queue/include/queue.hpp +++ b/src/agent/queue/include/queue.hpp @@ -129,9 +129,10 @@ class MultiTypeQueue * * @param type of the queue to be used as source * @param module module name + * @param messageQuantity quantity of messages to return * @return boost::asio::awaitable */ - boost::asio::awaitable getNextAwaitable(MessageType type, const std::string module = ""); + boost::asio::awaitable getNextNAwaitable(MessageType type, int messageQuantity, const std::string moduleName = ""); /** * @brief Returns N messages from a queue * diff --git a/src/agent/queue/src/queue.cpp b/src/agent/queue/src/queue.cpp index 7c6671dc985..77aa60d3222 100644 --- a/src/agent/queue/src/queue.cpp +++ b/src/agent/queue/src/queue.cpp @@ -138,7 +138,7 @@ Message MultiTypeQueue::getNext(MessageType type, const std::string moduleName) return result; } -boost::asio::awaitable MultiTypeQueue::getNextAwaitable(MessageType type, const std::string moduleName) +boost::asio::awaitable MultiTypeQueue::getNextNAwaitable(MessageType type, int messageQuantity, const std::string moduleName) { boost::asio::steady_timer timer(co_await boost::asio::this_coro::executor); @@ -151,7 +151,7 @@ boost::asio::awaitable MultiTypeQueue::getNextAwaitable(MessageType typ co_await timer.async_wait(boost::asio::use_awaitable); } - auto resultData = m_persistenceDest->RetrieveMultiple(1, m_mapMessageTypeName.at(type), moduleName); + auto resultData = m_persistenceDest->RetrieveMultiple(messageQuantity, m_mapMessageTypeName.at(type), moduleName); if (!resultData.empty()) { result.data = resultData; @@ -199,7 +199,6 @@ bool MultiTypeQueue::pop(MessageType type, const std::string moduleName) bool result = false; if (m_mapMessageTypeName.contains(type)) { - // TODO: Handle return value -> should show how many rows where deleted result = m_persistenceDest->RemoveMultiple(1, m_mapMessageTypeName.at(type), moduleName); } else diff --git a/src/agent/queue/src/sqlitestorage.cpp b/src/agent/queue/src/sqlitestorage.cpp index e5f776c67dc..36e62e24fc0 100644 --- a/src/agent/queue/src/sqlitestorage.cpp +++ b/src/agent/queue/src/sqlitestorage.cpp @@ -301,6 +301,10 @@ int SQLiteStorage::GetElementCount(const std::string& tableName, const std::stri { count = query.getColumn(0).getInt(); } + else + { + std::cerr << "Error SQLiteStorage get element count." << std::endl; + } return count; } catch (const SQLite::Exception& e) diff --git a/src/agent/queue/tests/queue_test.cpp b/src/agent/queue/tests/queue_test.cpp index b18eafb6410..c1f9599d017 100644 --- a/src/agent/queue/tests/queue_test.cpp +++ b/src/agent/queue/tests/queue_test.cpp @@ -58,7 +58,6 @@ std::string unescape_string(const std::string& str) return result; } -// TODO: Makes sense to add a full_clean method inside the persistence implementation? void cleanPersistence() { std::string filePath = DEFAULT_DB_PATH; @@ -463,22 +462,19 @@ TEST_F(QueueTest, PushSinglesleGetMultipleWithModule) EXPECT_EQ(1, messageReceivedContent1.size()); } -TEST_F(QueueTest, getNextAwaitable) +TEST_F(QueueTest, getNextAwaitableBase) { MultiTypeQueue queue(BIG_QUEUE_CAPACITY); boost::asio::io_context io_context; - const MessageType messageType {MessageType::STATEFUL}; - const json multipleDataContent = {"content-1"}; - const Message messageToSend {messageType, multipleDataContent}; - // Coroutine that waits till there's a message of the needed type on the queue boost::asio::co_spawn( io_context, [&queue]() -> boost::asio::awaitable { - auto messageReceived = co_await queue.getNextAwaitable(MessageType::STATELESS); - EXPECT_EQ(messageReceived.data.at(0).at("data"), "content-0"); + auto messageReceived = co_await queue.getNextNAwaitable(MessageType::STATELESS, 2); + EXPECT_EQ(messageReceived.data.at(0).at("data"), "content-2"); + EXPECT_EQ(messageReceived.data.at(1).at("data"), "content-1"); }, boost::asio::detached); @@ -488,10 +484,10 @@ TEST_F(QueueTest, getNextAwaitable) { std::this_thread::sleep_for(std::chrono::seconds(2)); const MessageType messageType {MessageType::STATELESS}; - const json multipleDataContent = {"content-0"}; + const json multipleDataContent = {"content-1","content-2","content-3"}; const Message messageToSend {messageType, multipleDataContent}; - EXPECT_EQ(queue.push(messageToSend), 1); - io_context.stop(); + EXPECT_EQ(queue.push(messageToSend), 3); + // io_context.stop(); }); io_context.run(); @@ -512,27 +508,31 @@ TEST_F(QueueTest, pushAwaitable) } EXPECT_TRUE(queue.isFull(MessageType::STATEFUL)); + EXPECT_EQ(queue.storedItems(MessageType::STATEFUL),2); - // Coroutine that waits till there's space on the to push a new messagequeue + // Coroutine that waits till there's space to push a new message boost::asio::co_spawn( io_context, [&queue]() -> boost::asio::awaitable { const MessageType messageType {MessageType::STATEFUL}; - const json multipleDataContent = {"content-1"}; - const Message messageToSend {messageType, multipleDataContent}; + const json dataContent = {"content-1"}; + const Message messageToSend {messageType, dataContent}; + EXPECT_EQ(queue.storedItems(MessageType::STATEFUL),2); auto messagesPushed = co_await queue.pushAwaitable(messageToSend); EXPECT_EQ(messagesPushed, 1); + EXPECT_EQ(queue.storedItems(MessageType::STATEFUL),2); }, boost::asio::detached); - // Simulate poping one message tll there's space to push a new one + // Simulate poping one message so there's space to push a new one std::thread consumer( [&queue, &io_context]() { std::this_thread::sleep_for(std::chrono::seconds(2)); - EXPECT_EQ(queue.pop(MessageType::STATEFUL), 1); - io_context.stop(); + EXPECT_EQ(queue.popN(MessageType::STATEFUL,1), 1); + // TODO: double check this behavior, is it mandatory to stop the context here? + // io_context.stop(); }); io_context.run(); From 2e93ebd1d657b5a63f83b6b7e71be3ceb96a8e2e Mon Sep 17 00:00:00 2001 From: LucioDonda Date: Mon, 12 Aug 2024 15:40:36 -0300 Subject: [PATCH 27/31] fix: correct fifo order with additional tests correction --- src/agent/queue/CMakeLists.txt | 6 +- src/agent/queue/include/queue.hpp | 3 +- src/agent/queue/src/queue.cpp | 6 +- src/agent/queue/src/sqlitestorage.cpp | 4 - src/agent/queue/tests/queue_test.cpp | 106 +++++++++++++------ src/agent/queue/tests/sqlitestorage_test.cpp | 4 +- 6 files changed, 87 insertions(+), 42 deletions(-) diff --git a/src/agent/queue/CMakeLists.txt b/src/agent/queue/CMakeLists.txt index ebc1e389f57..f8e4283a011 100644 --- a/src/agent/queue/CMakeLists.txt +++ b/src/agent/queue/CMakeLists.txt @@ -21,6 +21,6 @@ add_library(queue target_include_directories(queue PRIVATE include ${SQLiteCpp_INCLUDE_DIRS}) target_link_libraries(queue PRIVATE SQLiteCpp nlohmann_json::nlohmann_json fmt::fmt Boost::asio) -# if(BUILD_TESTS) -# add_subdirectory(tests) -# endif() +if(BUILD_TESTS) + add_subdirectory(tests) +endif() diff --git a/src/agent/queue/include/queue.hpp b/src/agent/queue/include/queue.hpp index 73d1c03ca77..f400aa8ab6e 100644 --- a/src/agent/queue/include/queue.hpp +++ b/src/agent/queue/include/queue.hpp @@ -132,7 +132,8 @@ class MultiTypeQueue * @param messageQuantity quantity of messages to return * @return boost::asio::awaitable */ - boost::asio::awaitable getNextNAwaitable(MessageType type, int messageQuantity, const std::string moduleName = ""); + boost::asio::awaitable + getNextNAwaitable(MessageType type, int messageQuantity, const std::string moduleName = ""); /** * @brief Returns N messages from a queue * diff --git a/src/agent/queue/src/queue.cpp b/src/agent/queue/src/queue.cpp index 77aa60d3222..9052c3ccb12 100644 --- a/src/agent/queue/src/queue.cpp +++ b/src/agent/queue/src/queue.cpp @@ -138,7 +138,8 @@ Message MultiTypeQueue::getNext(MessageType type, const std::string moduleName) return result; } -boost::asio::awaitable MultiTypeQueue::getNextNAwaitable(MessageType type, int messageQuantity, const std::string moduleName) +boost::asio::awaitable +MultiTypeQueue::getNextNAwaitable(MessageType type, int messageQuantity, const std::string moduleName) { boost::asio::steady_timer timer(co_await boost::asio::this_coro::executor); @@ -151,7 +152,8 @@ boost::asio::awaitable MultiTypeQueue::getNextNAwaitable(MessageType ty co_await timer.async_wait(boost::asio::use_awaitable); } - auto resultData = m_persistenceDest->RetrieveMultiple(messageQuantity, m_mapMessageTypeName.at(type), moduleName); + auto resultData = + m_persistenceDest->RetrieveMultiple(messageQuantity, m_mapMessageTypeName.at(type), moduleName); if (!resultData.empty()) { result.data = resultData; diff --git a/src/agent/queue/src/sqlitestorage.cpp b/src/agent/queue/src/sqlitestorage.cpp index 36e62e24fc0..fde8f4a8e52 100644 --- a/src/agent/queue/src/sqlitestorage.cpp +++ b/src/agent/queue/src/sqlitestorage.cpp @@ -200,10 +200,6 @@ json SQLiteStorage::RetrieveMultiple(int n, const std::string& tableName, const } } - if (!messages.empty()) - { - std::reverse(messages.begin(), messages.end()); - } return messages; } catch (const SQLite::Exception& e) diff --git a/src/agent/queue/tests/queue_test.cpp b/src/agent/queue/tests/queue_test.cpp index c1f9599d017..c706c16b905 100644 --- a/src/agent/queue/tests/queue_test.cpp +++ b/src/agent/queue/tests/queue_test.cpp @@ -230,7 +230,7 @@ TEST_F(QueueTest, SinglePushPopFullWithTimeout) EXPECT_NE(items, SMALL_QUEUE_CAPACITY); } -// Accesing different types of queues +// Accesing different types of queues from several threads TEST_F(QueueTest, MultithreadDifferentType) { MultiTypeQueue queue(BIG_QUEUE_CAPACITY); @@ -255,7 +255,7 @@ TEST_F(QueueTest, MultithreadDifferentType) { for (int i = 0; i < count; ++i) { - const json dataContent = R"({{"Data", "for STATELESS)" + std::to_string(i) + R"("}})"; + const json dataContent = R"({{"Data", "Number )" + std::to_string(i) + R"("}})"; EXPECT_EQ(queue.push(Message(MessageType::STATELESS, dataContent)), 1); EXPECT_EQ(queue.push(Message(MessageType::STATEFUL, dataContent)), 1); } @@ -279,10 +279,8 @@ TEST_F(QueueTest, MultithreadDifferentType) consumerThread2.join(); } - EXPECT_NE(0, queue.storedItems(MessageType::STATELESS)); - EXPECT_NE(0, queue.storedItems(MessageType::STATEFUL)); - EXPECT_FALSE(queue.isEmpty(MessageType::STATELESS)); - EXPECT_FALSE(queue.isEmpty(MessageType::STATEFUL)); + EXPECT_EQ(5, queue.storedItems(MessageType::STATELESS)); + EXPECT_EQ(5, queue.storedItems(MessageType::STATEFUL)); // Consume the rest of the messages std::thread consumerThread12(consumerStateLess, std::ref(itemsToConsume)); @@ -298,23 +296,21 @@ TEST_F(QueueTest, MultithreadDifferentType) consumerThread22.join(); } - // FIXME: this doesn't match - EXPECT_EQ(0, queue.storedItems(MessageType::STATELESS)); - EXPECT_EQ(0, queue.storedItems(MessageType::STATEFUL)); EXPECT_TRUE(queue.isEmpty(MessageType::STATELESS)); EXPECT_TRUE(queue.isEmpty(MessageType::STATEFUL)); } -// Accesing same queue +// Accesing same queue from 2 different threads TEST_F(QueueTest, MultithreadSameType) { MultiTypeQueue queue(BIG_QUEUE_CAPACITY); + auto messageType = MessageType::COMMAND; auto consumerCommand1 = [&](int& count) { for (int i = 0; i < count; ++i) { - queue.pop(MessageType::COMMAND); + queue.pop(messageType); } }; @@ -322,7 +318,7 @@ TEST_F(QueueTest, MultithreadSameType) { for (int i = 0; i < count; ++i) { - queue.pop(MessageType::COMMAND); + queue.pop(messageType); } }; @@ -331,7 +327,7 @@ TEST_F(QueueTest, MultithreadSameType) for (int i = 0; i < count; ++i) { const json dataContent = R"({{"Data": "for COMMAND)" + std::to_string(i) + R"("}})"; - EXPECT_EQ(queue.push(Message(MessageType::COMMAND, dataContent)), 1); + EXPECT_EQ(queue.push(Message(messageType, dataContent)), 1); } }; @@ -340,7 +336,7 @@ TEST_F(QueueTest, MultithreadSameType) messageProducer(itemsToInsert); - EXPECT_EQ(itemsToInsert, queue.storedItems(MessageType::COMMAND)); + EXPECT_EQ(itemsToInsert, queue.storedItems(messageType)); std::thread consumerThread1(consumerCommand1, std::ref(itemsToConsume)); std::thread messageProducerThread1(consumerCommand2, std::ref(itemsToConsume)); @@ -355,7 +351,7 @@ TEST_F(QueueTest, MultithreadSameType) consumerThread1.join(); } - EXPECT_TRUE(queue.isEmpty(MessageType::COMMAND)); + EXPECT_TRUE(queue.isEmpty(messageType)); } // Push Multiple with single message and data array, @@ -364,7 +360,6 @@ TEST_F(QueueTest, PushMultipleSeveralSingleGets) { MultiTypeQueue queue(BIG_QUEUE_CAPACITY); const MessageType messageType {MessageType::STATELESS}; - // TODO: double check array of objects const Message messageToSend {messageType, multipleDataContent}; EXPECT_EQ(3, queue.push(messageToSend)); @@ -393,10 +388,32 @@ TEST_F(QueueTest, PushMultipleWithMessageVector) messages.push_back({messageType, multipleDataContent}); } EXPECT_EQ(messages.size(), 3); - EXPECT_EQ(3, queue.push(messages)); + EXPECT_EQ(queue.push(messages), 3); EXPECT_EQ(queue.storedItems(MessageType::STATELESS), 3); } +// push message vector with a mutiple data element +TEST_F(QueueTest, PushVectorWithAMultipleInside) +{ + MultiTypeQueue queue(BIG_QUEUE_CAPACITY); + + std::vector messages; + + // triple data content message + const MessageType messageType {MessageType::STATELESS}; + const Message messageToSend {messageType, multipleDataContent}; + messages.push_back(messageToSend); + + // triple message vector + for (std::string i : {"0", "1", "2"}) + { + const json dataContent = {"content " + i}; + messages.push_back({messageType, dataContent}); + } + + EXPECT_EQ(6, queue.push(messages)); +} + // Push Multiple, pop multiples TEST_F(QueueTest, PushMultipleGetMultiple) { @@ -424,10 +441,10 @@ TEST_F(QueueTest, PushMultipleGetMultipleWithModule) // Altough we're asking for 10 messages only the availables are returned. auto messagesReceived = queue.getNextN(MessageType::STATELESS, 10); - int i = 3; + int i = 0; for (auto singleMessage : messagesReceived) { - EXPECT_EQ("content " + std::to_string(i--), singleMessage.data.at("data")); + EXPECT_EQ("content " + std::to_string(++i), singleMessage.data.at("data")); } EXPECT_EQ(0, queue.storedItems(MessageType::STATELESS, "fakemodule")); @@ -450,10 +467,10 @@ TEST_F(QueueTest, PushSinglesleGetMultipleWithModule) auto messagesReceived = queue.getNextN(MessageType::STATELESS, 10); EXPECT_EQ(5, messagesReceived.size()); - int i = 5; + int i = 0; for (auto singleMessage : messagesReceived) { - auto val = i--; + auto val = ++i; EXPECT_EQ("content-" + std::to_string(val), singleMessage.data.at("data")); EXPECT_EQ("module-" + std::to_string(val), singleMessage.data.at("module")); } @@ -462,7 +479,7 @@ TEST_F(QueueTest, PushSinglesleGetMultipleWithModule) EXPECT_EQ(1, messageReceivedContent1.size()); } -TEST_F(QueueTest, getNextAwaitableBase) +TEST_F(QueueTest, GetNextAwaitableBase) { MultiTypeQueue queue(BIG_QUEUE_CAPACITY); boost::asio::io_context io_context; @@ -473,8 +490,8 @@ TEST_F(QueueTest, getNextAwaitableBase) [&queue]() -> boost::asio::awaitable { auto messageReceived = co_await queue.getNextNAwaitable(MessageType::STATELESS, 2); - EXPECT_EQ(messageReceived.data.at(0).at("data"), "content-2"); - EXPECT_EQ(messageReceived.data.at(1).at("data"), "content-1"); + EXPECT_EQ(messageReceived.data.at(0).at("data"), "content-1"); + EXPECT_EQ(messageReceived.data.at(1).at("data"), "content-2"); }, boost::asio::detached); @@ -484,7 +501,7 @@ TEST_F(QueueTest, getNextAwaitableBase) { std::this_thread::sleep_for(std::chrono::seconds(2)); const MessageType messageType {MessageType::STATELESS}; - const json multipleDataContent = {"content-1","content-2","content-3"}; + const json multipleDataContent = {"content-1", "content-2", "content-3"}; const Message messageToSend {messageType, multipleDataContent}; EXPECT_EQ(queue.push(messageToSend), 3); // io_context.stop(); @@ -494,7 +511,7 @@ TEST_F(QueueTest, getNextAwaitableBase) producer.join(); } -TEST_F(QueueTest, pushAwaitable) +TEST_F(QueueTest, PushAwaitable) { MultiTypeQueue queue(SMALL_QUEUE_CAPACITY); boost::asio::io_context io_context; @@ -508,7 +525,7 @@ TEST_F(QueueTest, pushAwaitable) } EXPECT_TRUE(queue.isFull(MessageType::STATEFUL)); - EXPECT_EQ(queue.storedItems(MessageType::STATEFUL),2); + EXPECT_EQ(queue.storedItems(MessageType::STATEFUL), 2); // Coroutine that waits till there's space to push a new message boost::asio::co_spawn( @@ -518,10 +535,10 @@ TEST_F(QueueTest, pushAwaitable) const MessageType messageType {MessageType::STATEFUL}; const json dataContent = {"content-1"}; const Message messageToSend {messageType, dataContent}; - EXPECT_EQ(queue.storedItems(MessageType::STATEFUL),2); + EXPECT_EQ(queue.storedItems(MessageType::STATEFUL), 2); auto messagesPushed = co_await queue.pushAwaitable(messageToSend); EXPECT_EQ(messagesPushed, 1); - EXPECT_EQ(queue.storedItems(MessageType::STATEFUL),2); + EXPECT_EQ(queue.storedItems(MessageType::STATEFUL), 2); }, boost::asio::detached); @@ -530,7 +547,7 @@ TEST_F(QueueTest, pushAwaitable) [&queue, &io_context]() { std::this_thread::sleep_for(std::chrono::seconds(2)); - EXPECT_EQ(queue.popN(MessageType::STATEFUL,1), 1); + EXPECT_EQ(queue.popN(MessageType::STATEFUL, 1), 1); // TODO: double check this behavior, is it mandatory to stop the context here? // io_context.stop(); }); @@ -540,3 +557,32 @@ TEST_F(QueueTest, pushAwaitable) EXPECT_TRUE(queue.isFull(MessageType::STATEFUL)); } + +TEST_F(QueueTest, FifoOrderCheck) +{ + MultiTypeQueue queue(BIG_QUEUE_CAPACITY); + + // complete the queue with messages + const MessageType messageType {MessageType::STATEFUL}; + for (int i : {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + { + const json dataContent = R"({"Data" : "for STATEFUL)" + std::to_string(i) + R"("})"; + EXPECT_EQ(queue.push({messageType, dataContent}), 1); + } + + auto messageReceivedVector = queue.getNextN(messageType, 10); + EXPECT_EQ(messageReceivedVector.size(), 10); + int i = 0; + for (auto singleMessage : messageReceivedVector) + { + EXPECT_EQ(singleMessage.data.at("data"), R"({"Data" : "for STATEFUL)" + std::to_string(++i) + R"("})"); + } + + // Kepp the order of the message: FIFO + for (int i : {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + { + auto messageReceived = queue.getNextN(messageType, 1); + EXPECT_EQ(messageReceived.at(0).data.at("data"), R"({"Data" : "for STATEFUL)" + std::to_string(i) + R"("})"); + EXPECT_TRUE(queue.pop(messageType)); + } +} diff --git a/src/agent/queue/tests/sqlitestorage_test.cpp b/src/agent/queue/tests/sqlitestorage_test.cpp index 5f04df57d40..fdb36e4769c 100644 --- a/src/agent/queue/tests/sqlitestorage_test.cpp +++ b/src/agent/queue/tests/sqlitestorage_test.cpp @@ -133,10 +133,10 @@ TEST_F(SQLiteStorageTest, RetrieveMultipleMessagesWithModule) json retrievedMessages = storage->RetrieveMultiple(4, tableName, moduleName); EXPECT_EQ(retrievedMessages.size(), 4); - int i = 4; + int i = 0; for (auto singleMessage : retrievedMessages) { - EXPECT_EQ("value" + std::to_string(i--), singleMessage.at("data").at("key")); + EXPECT_EQ("value" + std::to_string(++i), singleMessage.at("data").at("key")); } } From 05591d70286fd7c51ec59b04c2639779e114f6e1 Mon Sep 17 00:00:00 2001 From: LucioDonda Date: Tue, 13 Aug 2024 11:58:00 -0300 Subject: [PATCH 28/31] fix: cmakes changes, headers and memeber names --- src/agent/CMakeLists.txt | 2 +- src/agent/queue/CMakeLists.txt | 13 ++++---- src/agent/queue/include/queue.hpp | 2 +- src/agent/queue/include/sqlitestorage.h | 6 ++-- src/agent/queue/src/queue.cpp | 7 ---- src/agent/queue/src/sqlitestorage.cpp | 6 ++-- src/agent/queue/tests/CMakeLists.txt | 43 +++++++++---------------- src/agent/queue/tests/queue_test.cpp | 2 +- 8 files changed, 30 insertions(+), 51 deletions(-) diff --git a/src/agent/CMakeLists.txt b/src/agent/CMakeLists.txt index 51cda998b9a..af39fd21165 100644 --- a/src/agent/CMakeLists.txt +++ b/src/agent/CMakeLists.txt @@ -20,6 +20,7 @@ set(SOURCES add_subdirectory(agent_info) add_subdirectory(communicator) add_subdirectory(configuration_parser) +add_subdirectory(queue) find_package(OpenSSL REQUIRED) find_package(Boost REQUIRED COMPONENTS asio beast) @@ -31,5 +32,4 @@ target_link_libraries(Agent PUBLIC ConfigurationParser Communicator AgentInfo PR if(BUILD_TESTS) enable_testing() add_subdirectory(tests) - add_subdirectory(queue) endif() diff --git a/src/agent/queue/CMakeLists.txt b/src/agent/queue/CMakeLists.txt index f8e4283a011..bf7c6aa0906 100644 --- a/src/agent/queue/CMakeLists.txt +++ b/src/agent/queue/CMakeLists.txt @@ -1,26 +1,25 @@ cmake_minimum_required(VERSION 3.22) -project(queue LANGUAGES CXX) +project(AgentQueue LANGUAGES CXX) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) set(CMAKE_CXX_FLAGS "-Wall -Wextra -Wunused -pthread") -include_directories(${CMAKE_SOURCE_DIR}/include) - find_package(SQLiteCpp REQUIRED) find_package(nlohmann_json REQUIRED) find_package(fmt REQUIRED) -find_package(Boost REQUIRED COMPONENTS asio beast) +find_package(Boost REQUIRED COMPONENTS asio) -add_library(queue +add_library(AgentQueue src/sqlitestorage.cpp src/queue.cpp ) -target_include_directories(queue PRIVATE include ${SQLiteCpp_INCLUDE_DIRS}) -target_link_libraries(queue PRIVATE SQLiteCpp nlohmann_json::nlohmann_json fmt::fmt Boost::asio) +target_include_directories(AgentQueue PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include PRIVATE ${SQLiteCpp_INCLUDE_DIRS}) +target_link_libraries(AgentQueue PUBLIC nlohmann_json::nlohmann_json Boost::asio PRIVATE SQLiteCpp nlohmann_json::nlohmann_json fmt::fmt Boost::asio) if(BUILD_TESTS) + enable_testing() add_subdirectory(tests) endif() diff --git a/src/agent/queue/include/queue.hpp b/src/agent/queue/include/queue.hpp index f400aa8ab6e..5390fe1d172 100644 --- a/src/agent/queue/include/queue.hpp +++ b/src/agent/queue/include/queue.hpp @@ -17,7 +17,7 @@ #include "sqlitestorage.h" // TODO: move to a configuration setting -constexpr int DEFAULT_MAX = 10; +constexpr int DEFAULT_MAX = 10000; constexpr int DEFAULT_TIMEOUT_S = 3; // Factory class diff --git a/src/agent/queue/include/sqlitestorage.h b/src/agent/queue/include/sqlitestorage.h index bbc4b6c53dc..e01d085de0c 100644 --- a/src/agent/queue/include/sqlitestorage.h +++ b/src/agent/queue/include/sqlitestorage.h @@ -126,11 +126,11 @@ class SQLiteStorage : public Persistence /// Mutex to ensure thread-safe operations. std::mutex m_mutex; - /// @brief condition variable to wait for database access + /// @brief condition variable to wait for database access. std::condition_variable m_cv; - // TODO: should it be atomic? - bool m_db_in_use = false; + // @brief flag for notifying the use of the db. + bool m_dbInUse = false; }; #endif // SQLITE_STORAGE_H diff --git a/src/agent/queue/src/queue.cpp b/src/agent/queue/src/queue.cpp index 9052c3ccb12..0b4ea0062cb 100644 --- a/src/agent/queue/src/queue.cpp +++ b/src/agent/queue/src/queue.cpp @@ -1,14 +1,7 @@ -#include #include -#include #include -#include -#include -#include #include -#include #include -#include #include "queue.hpp" diff --git a/src/agent/queue/src/sqlitestorage.cpp b/src/agent/queue/src/sqlitestorage.cpp index fde8f4a8e52..855d209e75f 100644 --- a/src/agent/queue/src/sqlitestorage.cpp +++ b/src/agent/queue/src/sqlitestorage.cpp @@ -46,14 +46,14 @@ void SQLiteStorage::InitializeTable(const std::string& tableName) void SQLiteStorage::waitForDatabaseAccess() { std::unique_lock lock(m_mutex); - m_cv.wait(lock, [this] { return !m_db_in_use; }); - m_db_in_use = true; + m_cv.wait(lock, [this] { return !m_dbInUse; }); + m_dbInUse = true; } void SQLiteStorage::releaseDatabaseAccess() { std::lock_guard lock(m_mutex); - m_db_in_use = false; + m_dbInUse = false; m_cv.notify_one(); } diff --git a/src/agent/queue/tests/CMakeLists.txt b/src/agent/queue/tests/CMakeLists.txt index aea65975ffb..8af9802bdc9 100644 --- a/src/agent/queue/tests/CMakeLists.txt +++ b/src/agent/queue/tests/CMakeLists.txt @@ -1,36 +1,23 @@ find_package(GTest REQUIRED) -# find_package(SQLiteCpp REQUIRED) -# find_package(nlohmann_json REQUIRED) -include_directories(../include) -include_directories(${GTEST_INCLUDE_DIRS}) -include_directories(${SQLiteCpp_INCLUDE_DIRS}) +# AgentQueue tests +add_executable(test_AgentQueue queue_test.cpp) -#TODO: is this reusable ? -# file(GLOB QUEUE_UNIT_TEST_SRC "*.cpp") +target_link_libraries(test_AgentQueue PUBLIC + AgentQueue + GTest::gtest + GTest::gtest_main + GTest::gmock + GTest::gmock_main) -add_executable(test_queue - queue_test.cpp - ../src/sqlitestorage.cpp - ) -target_link_libraries(test_queue - ${GTEST_LIBRARIES} - ${GTEST_MAIN_LIBRARIES} - queue - SQLiteCpp - nlohmann_json::nlohmann_json - fmt::fmt - Boost::asio) - -# Create a test executable for SQLiteStorage tests -add_executable(test_sqlitestorage - sqlitestorage_test.cpp - ../src/sqlitestorage.cpp) +# SQLiteStorage tests +add_executable(test_sqlitestorage sqlitestorage_test.cpp) target_link_libraries(test_sqlitestorage - ${GTEST_LIBRARIES} - ${GTEST_MAIN_LIBRARIES} + AgentQueue SQLiteCpp - nlohmann_json::nlohmann_json - fmt::fmt) \ No newline at end of file + GTest::gtest + GTest::gtest_main + GTest::gmock + GTest::gmock_main) diff --git a/src/agent/queue/tests/queue_test.cpp b/src/agent/queue/tests/queue_test.cpp index c706c16b905..944c0e0a27b 100644 --- a/src/agent/queue/tests/queue_test.cpp +++ b/src/agent/queue/tests/queue_test.cpp @@ -578,7 +578,7 @@ TEST_F(QueueTest, FifoOrderCheck) EXPECT_EQ(singleMessage.data.at("data"), R"({"Data" : "for STATEFUL)" + std::to_string(++i) + R"("})"); } - // Kepp the order of the message: FIFO + // Keep the order of the message: FIFO for (int i : {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) { auto messageReceived = queue.getNextN(messageType, 1); From fff83e990fd6cfe957c4db3f0eb71364004ffdff Mon Sep 17 00:00:00 2001 From: LucioDonda Date: Tue, 13 Aug 2024 13:01:37 -0300 Subject: [PATCH 29/31] feat: directory and files renaming --- src/agent/CMakeLists.txt | 2 +- src/agent/{queue => agent_queue}/CMakeLists.txt | 2 +- .../include/queue.hpp => agent_queue/include/agent_queue.hpp} | 0 src/agent/{queue => agent_queue}/include/persistence.h | 0 src/agent/{queue => agent_queue}/include/shared.hpp | 0 src/agent/{queue => agent_queue}/include/sqlitestorage.h | 0 .../{queue/src/queue.cpp => agent_queue/src/agent_queue.cpp} | 2 +- src/agent/{queue => agent_queue}/src/main.cpp | 2 +- src/agent/{queue => agent_queue}/src/sqlitestorage.cpp | 0 src/agent/{queue => agent_queue}/tests/CMakeLists.txt | 0 src/agent/{queue => agent_queue}/tests/main.cpp | 0 src/agent/{queue => agent_queue}/tests/queue_test.cpp | 2 +- src/agent/{queue => agent_queue}/tests/queue_test.hpp | 0 src/agent/{queue => agent_queue}/tests/sqlitestorage_test.cpp | 0 14 files changed, 5 insertions(+), 5 deletions(-) rename src/agent/{queue => agent_queue}/CMakeLists.txt (97%) rename src/agent/{queue/include/queue.hpp => agent_queue/include/agent_queue.hpp} (100%) rename src/agent/{queue => agent_queue}/include/persistence.h (100%) rename src/agent/{queue => agent_queue}/include/shared.hpp (100%) rename src/agent/{queue => agent_queue}/include/sqlitestorage.h (100%) rename src/agent/{queue/src/queue.cpp => agent_queue/src/agent_queue.cpp} (99%) rename src/agent/{queue => agent_queue}/src/main.cpp (81%) rename src/agent/{queue => agent_queue}/src/sqlitestorage.cpp (100%) rename src/agent/{queue => agent_queue}/tests/CMakeLists.txt (100%) rename src/agent/{queue => agent_queue}/tests/main.cpp (100%) rename src/agent/{queue => agent_queue}/tests/queue_test.cpp (99%) rename src/agent/{queue => agent_queue}/tests/queue_test.hpp (100%) rename src/agent/{queue => agent_queue}/tests/sqlitestorage_test.cpp (100%) diff --git a/src/agent/CMakeLists.txt b/src/agent/CMakeLists.txt index af39fd21165..c5743b94543 100644 --- a/src/agent/CMakeLists.txt +++ b/src/agent/CMakeLists.txt @@ -20,7 +20,7 @@ set(SOURCES add_subdirectory(agent_info) add_subdirectory(communicator) add_subdirectory(configuration_parser) -add_subdirectory(queue) +add_subdirectory(agent_queue) find_package(OpenSSL REQUIRED) find_package(Boost REQUIRED COMPONENTS asio beast) diff --git a/src/agent/queue/CMakeLists.txt b/src/agent/agent_queue/CMakeLists.txt similarity index 97% rename from src/agent/queue/CMakeLists.txt rename to src/agent/agent_queue/CMakeLists.txt index bf7c6aa0906..48d48a24430 100644 --- a/src/agent/queue/CMakeLists.txt +++ b/src/agent/agent_queue/CMakeLists.txt @@ -13,7 +13,7 @@ find_package(Boost REQUIRED COMPONENTS asio) add_library(AgentQueue src/sqlitestorage.cpp - src/queue.cpp + src/agent_queue.cpp ) target_include_directories(AgentQueue PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include PRIVATE ${SQLiteCpp_INCLUDE_DIRS}) diff --git a/src/agent/queue/include/queue.hpp b/src/agent/agent_queue/include/agent_queue.hpp similarity index 100% rename from src/agent/queue/include/queue.hpp rename to src/agent/agent_queue/include/agent_queue.hpp diff --git a/src/agent/queue/include/persistence.h b/src/agent/agent_queue/include/persistence.h similarity index 100% rename from src/agent/queue/include/persistence.h rename to src/agent/agent_queue/include/persistence.h diff --git a/src/agent/queue/include/shared.hpp b/src/agent/agent_queue/include/shared.hpp similarity index 100% rename from src/agent/queue/include/shared.hpp rename to src/agent/agent_queue/include/shared.hpp diff --git a/src/agent/queue/include/sqlitestorage.h b/src/agent/agent_queue/include/sqlitestorage.h similarity index 100% rename from src/agent/queue/include/sqlitestorage.h rename to src/agent/agent_queue/include/sqlitestorage.h diff --git a/src/agent/queue/src/queue.cpp b/src/agent/agent_queue/src/agent_queue.cpp similarity index 99% rename from src/agent/queue/src/queue.cpp rename to src/agent/agent_queue/src/agent_queue.cpp index 0b4ea0062cb..791a4373f85 100644 --- a/src/agent/queue/src/queue.cpp +++ b/src/agent/agent_queue/src/agent_queue.cpp @@ -3,7 +3,7 @@ #include #include -#include "queue.hpp" +#include "agent_queue.hpp" int MultiTypeQueue::push(Message message, bool shouldWait) { diff --git a/src/agent/queue/src/main.cpp b/src/agent/agent_queue/src/main.cpp similarity index 81% rename from src/agent/queue/src/main.cpp rename to src/agent/agent_queue/src/main.cpp index 398d4e93ecb..f3a4a0370ca 100644 --- a/src/agent/queue/src/main.cpp +++ b/src/agent/agent_queue/src/main.cpp @@ -2,7 +2,7 @@ #include #include -#include "queue.hpp" +#include "agent_queue.hpp" int main() { diff --git a/src/agent/queue/src/sqlitestorage.cpp b/src/agent/agent_queue/src/sqlitestorage.cpp similarity index 100% rename from src/agent/queue/src/sqlitestorage.cpp rename to src/agent/agent_queue/src/sqlitestorage.cpp diff --git a/src/agent/queue/tests/CMakeLists.txt b/src/agent/agent_queue/tests/CMakeLists.txt similarity index 100% rename from src/agent/queue/tests/CMakeLists.txt rename to src/agent/agent_queue/tests/CMakeLists.txt diff --git a/src/agent/queue/tests/main.cpp b/src/agent/agent_queue/tests/main.cpp similarity index 100% rename from src/agent/queue/tests/main.cpp rename to src/agent/agent_queue/tests/main.cpp diff --git a/src/agent/queue/tests/queue_test.cpp b/src/agent/agent_queue/tests/queue_test.cpp similarity index 99% rename from src/agent/queue/tests/queue_test.cpp rename to src/agent/agent_queue/tests/queue_test.cpp index 944c0e0a27b..364eb152982 100644 --- a/src/agent/queue/tests/queue_test.cpp +++ b/src/agent/agent_queue/tests/queue_test.cpp @@ -12,7 +12,7 @@ #include #include -#include "queue.hpp" +#include "agent_queue.hpp" #include "queue_test.hpp" using json = nlohmann::json; diff --git a/src/agent/queue/tests/queue_test.hpp b/src/agent/agent_queue/tests/queue_test.hpp similarity index 100% rename from src/agent/queue/tests/queue_test.hpp rename to src/agent/agent_queue/tests/queue_test.hpp diff --git a/src/agent/queue/tests/sqlitestorage_test.cpp b/src/agent/agent_queue/tests/sqlitestorage_test.cpp similarity index 100% rename from src/agent/queue/tests/sqlitestorage_test.cpp rename to src/agent/agent_queue/tests/sqlitestorage_test.cpp From ce05ec61807522b943a47c11d2b6672699628dba Mon Sep 17 00:00:00 2001 From: LucioDonda Date: Tue, 13 Aug 2024 13:50:28 -0300 Subject: [PATCH 30/31] fix: pr corrections and cleaning --- src/agent/agent_queue/CMakeLists.txt | 3 ++- src/agent/agent_queue/include/agent_queue.hpp | 1 - src/agent/agent_queue/src/main.cpp | 12 ------------ src/agent/agent_queue/tests/CMakeLists.txt | 3 +++ src/vcpkg.json | 4 ++-- 5 files changed, 7 insertions(+), 16 deletions(-) delete mode 100644 src/agent/agent_queue/src/main.cpp diff --git a/src/agent/agent_queue/CMakeLists.txt b/src/agent/agent_queue/CMakeLists.txt index 48d48a24430..e791d3a8caa 100644 --- a/src/agent/agent_queue/CMakeLists.txt +++ b/src/agent/agent_queue/CMakeLists.txt @@ -3,8 +3,9 @@ cmake_minimum_required(VERSION 3.22) project(AgentQueue LANGUAGES CXX) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) +set(CMAKE_BUILD_TYPE RelWithDebInfo) -set(CMAKE_CXX_FLAGS "-Wall -Wextra -Wunused -pthread") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -Wall -Wextra -Wunused -pthread") find_package(SQLiteCpp REQUIRED) find_package(nlohmann_json REQUIRED) diff --git a/src/agent/agent_queue/include/agent_queue.hpp b/src/agent/agent_queue/include/agent_queue.hpp index 5390fe1d172..039b93da7e2 100644 --- a/src/agent/agent_queue/include/agent_queue.hpp +++ b/src/agent/agent_queue/include/agent_queue.hpp @@ -10,7 +10,6 @@ #include #include -// #include #include #include "shared.hpp" diff --git a/src/agent/agent_queue/src/main.cpp b/src/agent/agent_queue/src/main.cpp deleted file mode 100644 index f3a4a0370ca..00000000000 --- a/src/agent/agent_queue/src/main.cpp +++ /dev/null @@ -1,12 +0,0 @@ -#include -#include -#include - -#include "agent_queue.hpp" - -int main() -{ - MultiTypeQueue queue(100); - - return 0; -} diff --git a/src/agent/agent_queue/tests/CMakeLists.txt b/src/agent/agent_queue/tests/CMakeLists.txt index 8af9802bdc9..3da2988d63a 100644 --- a/src/agent/agent_queue/tests/CMakeLists.txt +++ b/src/agent/agent_queue/tests/CMakeLists.txt @@ -9,6 +9,7 @@ target_link_libraries(test_AgentQueue PUBLIC GTest::gtest_main GTest::gmock GTest::gmock_main) +add_test(NAME AgentQueueTest COMMAND test_AgentQueue) # SQLiteStorage tests @@ -21,3 +22,5 @@ target_link_libraries(test_sqlitestorage GTest::gtest_main GTest::gmock GTest::gmock_main) +add_test(NAME SqliteStorageTest COMMAND test_sqlitestorage) + diff --git a/src/vcpkg.json b/src/vcpkg.json index 474d01a633b..984eb01f8cc 100644 --- a/src/vcpkg.json +++ b/src/vcpkg.json @@ -5,13 +5,13 @@ "boost-asio", "boost-beast", "boost-uuid", + "fmt", "gtest", "jwt-cpp", "nlohmann-json", "openssl", "sqlitecpp", - "toml11", - "fmt" + "toml11" ], "overrides": [ { From 942a9052631242131df1d83cf5091c896d3dc66c Mon Sep 17 00:00:00 2001 From: LucioDonda Date: Tue, 13 Aug 2024 15:41:53 -0300 Subject: [PATCH 31/31] fix: completing method and parameters description --- src/agent/agent_queue/CMakeLists.txt | 2 +- src/agent/agent_queue/include/agent_queue.hpp | 64 ++++++++++--------- src/agent/agent_queue/include/persistence.h | 58 ++++++++--------- src/agent/agent_queue/include/shared.hpp | 2 +- src/agent/agent_queue/include/sqlitestorage.h | 59 ++++++++++++----- src/agent/agent_queue/src/sqlitestorage.cpp | 1 - 6 files changed, 109 insertions(+), 77 deletions(-) diff --git a/src/agent/agent_queue/CMakeLists.txt b/src/agent/agent_queue/CMakeLists.txt index e791d3a8caa..6d9280b30bc 100644 --- a/src/agent/agent_queue/CMakeLists.txt +++ b/src/agent/agent_queue/CMakeLists.txt @@ -21,6 +21,6 @@ target_include_directories(AgentQueue PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include target_link_libraries(AgentQueue PUBLIC nlohmann_json::nlohmann_json Boost::asio PRIVATE SQLiteCpp nlohmann_json::nlohmann_json fmt::fmt Boost::asio) if(BUILD_TESTS) - enable_testing() + enable_testing() add_subdirectory(tests) endif() diff --git a/src/agent/agent_queue/include/agent_queue.hpp b/src/agent/agent_queue/include/agent_queue.hpp index 039b93da7e2..9ce98d5281f 100644 --- a/src/agent/agent_queue/include/agent_queue.hpp +++ b/src/agent/agent_queue/include/agent_queue.hpp @@ -39,10 +39,6 @@ class PersistenceFactory } }; -/** - * @brief - * - */ class MultiTypeQueue { private: @@ -59,7 +55,6 @@ class MultiTypeQueue std::condition_variable m_cv; public: - // Create a vector with 3 PersistedQueue elements MultiTypeQueue(int size = DEFAULT_MAX, int timeout = DEFAULT_TIMEOUT_S) : m_maxItems(size) , m_timeout(timeout) @@ -75,18 +70,29 @@ class MultiTypeQueue } } - // Delete copy constructor + /** + * @brief Delete copy constructor + */ MultiTypeQueue(const MultiTypeQueue&) = delete; - // Delete copy assignment operator + /** + * @brief Delete copy assignment operator + */ MultiTypeQueue& operator=(const MultiTypeQueue&) = delete; - // Delete move constructor + /** + * @brief Delete move constructor + */ MultiTypeQueue(MultiTypeQueue&&) = delete; - // Delete move assignment operator + /** + * @brief Delete move assignment operator + */ MultiTypeQueue& operator=(MultiTypeQueue&&) = delete; + /** + * @brief Destructor. + */ ~MultiTypeQueue() {}; /** @@ -109,16 +115,16 @@ class MultiTypeQueue /** * @brief pushes a vector of messages * - * @param messages to be pushed - * @param shouldWait when true, the function will wait until the message is pushed + * @param messages vector of messages to be pushed * @return int number of messages pushed */ int push(std::vector messages); /** - * @brief Get the Last Message object + * @brief Get the next Message object * * @param type of the queue to be used as source + * @param module module name * @return Message type object taken from the queue */ Message getNext(MessageType type, const std::string module = ""); @@ -127,17 +133,18 @@ class MultiTypeQueue * @brief Get the Next Awaitable object * * @param type of the queue to be used as source - * @param module module name + * @param moduleName module name * @param messageQuantity quantity of messages to return - * @return boost::asio::awaitable + * @return boost::asio::awaitable awaitable object taken from the queue */ boost::asio::awaitable getNextNAwaitable(MessageType type, int messageQuantity, const std::string moduleName = ""); + /** * @brief Returns N messages from a queue * * @param type Of the queue to be used as source - * @param moduleName + * @param moduleName module name * @param messageQuantity quantity of messages to return * @return Message Json data othe messages fetched */ @@ -148,17 +155,17 @@ class MultiTypeQueue * * @param type MessageType queue to pop * @param moduleName - * @return true popped succesfully - * @return false wasn't able to pop message + * @return true when popped succesfully + * @return false if it wasn't able to pop message */ bool pop(MessageType type, const std::string moduleName = ""); /** - * @brief + * @brief deletes N messages from a queue * - * @param type - * @param moduleName - * @param messageQuantity + * @param type MessageType queue to pop + * @param moduleName module name + * @param messageQuantity quantity of messages to pop * @return Number of messages deleted */ int popN(MessageType type, int messageQuantity, const std::string moduleName = ""); @@ -166,8 +173,8 @@ class MultiTypeQueue /** * @brief Checks emptyness of a queue * - * @param type - * @param moduleName + * @param type MessageType + * @param moduleName module name * @return true when queue empty * @return false otherwise */ @@ -176,8 +183,8 @@ class MultiTypeQueue /** * @brief Checks fullness of a queue * - * @param type - * @param moduleName + * @param type MessageType + * @param moduleName module name * @return true when queue is full * @return false otherwise */ @@ -186,10 +193,9 @@ class MultiTypeQueue /** * @brief Get the Items By Type object * - * @param type - * @param moduleName - * @return true - * @return false + * @param type MessageType + * @param moduleName module name + * @return int number of items in the queue. */ int storedItems(MessageType type, const std::string moduleName = ""); }; diff --git a/src/agent/agent_queue/include/persistence.h b/src/agent/agent_queue/include/persistence.h index 8ef69b6dd11..7f859403f6d 100644 --- a/src/agent/agent_queue/include/persistence.h +++ b/src/agent/agent_queue/include/persistence.h @@ -21,61 +21,61 @@ class Persistence virtual ~Persistence() = default; /** - * @brief + * @brief Store a JSON message in the specified queue. * - * @param message - * @param queueName - * @param moduleName - * @return int + * @param message The JSON message to be stored. + * @param queueName The name of the queue. + * @param moduleName The name of the module. + * @return int The number of messages stored. */ virtual int Store(const json& message, const std::string& queueName, const std::string& moduleName = "") = 0; /** - * @brief + * @brief Retrieve a JSON message from the specified queue. * - * @param id - * @param queueName - * @param moduleName - * @return json + * @param id rowid of the message to be retrieved. + * @param queueName The name of the queue. + * @param moduleName The name of the module. + * @return json The retrieved JSON message. */ virtual json Retrieve(int id, const std::string& queueName, const std::string& moduleName = "") = 0; /** - * @brief + * @brief Retrieve multiple JSON messages from the specified queue. * - * @param n - * @param queueName - * @param moduleName - * @return json + * @param n number of messages to be retrieved. + * @param queueName The name of the queue. + * @param moduleName The name of the module. + * @return json The retrieved JSON messages. */ virtual json RetrieveMultiple(int n, const std::string& queueName, const std::string& moduleName = "") = 0; /** - * @brief + * @brief Remove a JSON message from the specified queue. * - * @param id - * @param queueName - * @param moduleName - * @return int + * @param id number of messages to be removed. + * @param queueName The name of the queue. + * @param moduleName The name of the module. + * @return int The number of messages removed. */ virtual int Remove(int id, const std::string& queueName, const std::string& moduleName = "") = 0; /** - * @brief + * @brief Remove multiple JSON messages from the specified queue. * - * @param n - * @param queueName - * @param moduleName - * @return int + * @param n number of messages to be removed. + * @param queueName The name of the queue. + * @param moduleName The name of the module. + * @return int The number of messages removed. */ virtual int RemoveMultiple(int n, const std::string& queueName, const std::string& moduleName = "") = 0; /** - * @brief Get the Element Count object + * @brief Get the quantity of elements stored in the specified queue. * - * @param queueName - * @param moduleName - * @return int + * @param queueName The name of the queue. + * @param moduleName The name of the module. + * @return int The quantity of elements stored in the specified queue. */ virtual int GetElementCount(const std::string& queueName, const std::string& moduleName = "") = 0; }; diff --git a/src/agent/agent_queue/include/shared.hpp b/src/agent/agent_queue/include/shared.hpp index 0561f0db8ea..c2b07d3022e 100644 --- a/src/agent/agent_queue/include/shared.hpp +++ b/src/agent/agent_queue/include/shared.hpp @@ -21,7 +21,7 @@ enum MessageType }; /** - * @brief Wrapper for Message data and type + * @brief Wrapper for Message, contains the message type, the json data and the module name. * */ class Message diff --git a/src/agent/agent_queue/include/sqlitestorage.h b/src/agent/agent_queue/include/sqlitestorage.h index e01d085de0c..aa1bda09c6f 100644 --- a/src/agent/agent_queue/include/sqlitestorage.h +++ b/src/agent/agent_queue/include/sqlitestorage.h @@ -26,16 +26,24 @@ class SQLiteStorage : public Persistence public: SQLiteStorage(const std::string& dbName, const std::vector tableName); - // Delete copy constructor + /** + * @brief Delete copy constructor + */ SQLiteStorage(const SQLiteStorage&) = delete; - // Delete copy assignment operator + /** + * @brief Delete copy assignment operator + */ SQLiteStorage& operator=(const SQLiteStorage&) = delete; - // Delete move constructor + /** + * @brief Delete move constructor + */ SQLiteStorage(SQLiteStorage&&) = delete; - // Delete move assignment operator + /** + * @brief Delete move assignment operator + */ SQLiteStorage& operator=(SQLiteStorage&&) = delete; /** @@ -48,7 +56,7 @@ class SQLiteStorage : public Persistence * * @param message The JSON message to store. * @param tableName The name of the table to store the message in. - * @param moduleName + * @param moduleName The name of the module that created the message. * @return The number of stored elements. */ int Store(const json& message, const std::string& tableName, const std::string& moduleName = "") override; @@ -57,6 +65,8 @@ class SQLiteStorage : public Persistence * @brief Retrieve a JSON message by its ID. * * @param id The ID of the message to retrieve. + * @param tableName The name of the table to retrieve the message from. + * @param moduleName The name of the module that created the message. * @return The retrieved JSON message. */ json Retrieve(int id, const std::string& tableName, const std::string& moduleName = "") override; @@ -65,6 +75,8 @@ class SQLiteStorage : public Persistence * @brief Retrieve multiple JSON messages. * * @param n The number of messages to retrieve. + * @param tableName The name of the table to retrieve the message from. + * @param moduleName The name of the module that created the message. * @return A vector of retrieved JSON messages. */ json RetrieveMultiple(int n, const std::string& tableName, const std::string& moduleName = "") override; @@ -72,8 +84,9 @@ class SQLiteStorage : public Persistence /** * @brief Remove a JSON message by its ID. * - * @param id The ID of the message to remove. - * @param moduleName + * @param id The number the message to remove. + * @param tableName The name of the table to remove the message from. + * @param moduleName The name of the module that created the message. * @return The number of removed elements. */ int Remove(int id, const std::string& tableName, const std::string& moduleName = "") override; @@ -82,14 +95,16 @@ class SQLiteStorage : public Persistence * @brief Remove multiple JSON messages. * * @param n The number of messages to remove. - * @param moduleName + * @param tableName The name of the table to remove the message from. + * @param moduleName The name of the module that created the message. * @return The number of removed elements. */ int RemoveMultiple(int n, const std::string& tableName, const std::string& moduleName = "") override; /** * @brief Get the number of elements in the table. - * + * @param tableName The name of the table to retrieve the message from. + * @param moduleName The name of the module that created the message. * @return The number of elements in the table. */ int GetElementCount(const std::string& tableName, const std::string& moduleName = "") override; @@ -97,8 +112,8 @@ class SQLiteStorage : public Persistence private: /** * @brief Initialize the table in the SQLite database. - * * This method creates the table if it does not already exist. + * @param tableName The name of the table to initialize. */ void InitializeTable(const std::string& tableName); @@ -114,22 +129,34 @@ class SQLiteStorage : public Persistence */ void releaseDatabaseAccess(); - /// The name of the SQLite database file. + /** + * @brief The name of the SQLite database file. + */ const std::string m_dbName; - /// The name of the table to use for storing messages. + /** + * @brief The name of the table to use for storing messages. + */ const std::string m_tableName; - /// Pointer to the SQLite database connection. + /** + * @brief Pointer to the SQLite database connection. + */ std::unique_ptr m_db; - /// Mutex to ensure thread-safe operations. + /** + * @brief Mutex to ensure thread-safe operations. + */ std::mutex m_mutex; - /// @brief condition variable to wait for database access. + /** + * @brief condition variable to wait for database access. + */ std::condition_variable m_cv; - // @brief flag for notifying the use of the db. + /** + * @brief flag for notifying the use of the db. + */ bool m_dbInUse = false; }; diff --git a/src/agent/agent_queue/src/sqlitestorage.cpp b/src/agent/agent_queue/src/sqlitestorage.cpp index 855d209e75f..357c0e52f7a 100644 --- a/src/agent/agent_queue/src/sqlitestorage.cpp +++ b/src/agent/agent_queue/src/sqlitestorage.cpp @@ -1,6 +1,5 @@ #include "sqlitestorage.h" #include -// TODO: only in gcc13, does it worth it? #include #include #include