Skip to content

Commit

Permalink
feat: first approach of measuring size in retrieval method
Browse files Browse the repository at this point in the history
  • Loading branch information
@LucioDonda
LucioDonda committed Dec 5, 2024
1 parent 88f1c95 commit 7635341
Show file tree
Hide file tree
Showing 3 changed files with 119 additions and 55 deletions.
138 changes: 89 additions & 49 deletions src/agent/multitype_queue/src/sqlitestorage.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,55 +201,9 @@ nlohmann::json SQLiteStorage::RetrieveMultiple(int n,
selectQuery = fmt::format(SELECT_MULTIPLE_QUERY, tableName, moduleName);
}

try
{
SQLite::Statement query(*m_db, selectQuery);
query.bind(1, n);
nlohmann::json messages = nlohmann::json::array();
while (query.executeStep())
{
if (query.getColumnCount() == 4 && query.getColumn(3).getType() == SQLite::TEXT &&
query.getColumn(2).getType() == SQLite::TEXT && query.getColumn(1).getType() == SQLite::TEXT &&
query.getColumn(0).getType() == SQLite::TEXT)
{
std::string moduleNameString = query.getColumn(0).getString();
std::string moduleTypeString = query.getColumn(1).getString();
std::string metadataString = query.getColumn(2).getString();
std::string dataString = query.getColumn(3).getString();

nlohmann::json outputJson = {{"moduleName", ""}, {"moduleType", ""}, {"metadata", ""}, {"data", {}}};

if (!dataString.empty())
{
outputJson["data"] = nlohmann::json::parse(dataString);
}

if (!metadataString.empty())
{
outputJson["metadata"] = metadataString;
}

if (!moduleNameString.empty())
{
outputJson["moduleName"] = moduleNameString;
}

if (!moduleTypeString.empty())
{
outputJson["moduleType"] = moduleTypeString;
}

messages.push_back(outputJson);
}
}

return messages;
}
catch (const std::exception& e)
{
LogError("Error during RetrieveMultiple operation: {}.", e.what());
return {};
}
SQLite::Statement query(*m_db, selectQuery);
query.bind(1, n);
return ProcessRequest(query);
}

int SQLiteStorage::Remove(int id, const std::string& tableName, const std::string& moduleName)
Expand Down Expand Up @@ -379,3 +333,89 @@ int SQLiteStorage::GetElementsStoredSize(const std::string& tableName)
return 0;
}
}

nlohmann::json SQLiteStorage::ProcessRequest(SQLite::Statement& sqlStatementQuery, size_t maxSize)
{
try
{
nlohmann::json messages = nlohmann::json::array();
size_t sizeAccum = 0;
while (sqlStatementQuery.executeStep())
{
if (sqlStatementQuery.getColumnCount() == 4 && sqlStatementQuery.getColumn(3).getType() == SQLite::TEXT &&
sqlStatementQuery.getColumn(2).getType() == SQLite::TEXT &&
sqlStatementQuery.getColumn(1).getType() == SQLite::TEXT &&
sqlStatementQuery.getColumn(0).getType() == SQLite::TEXT)
{
std::string moduleNameString = sqlStatementQuery.getColumn(0).getString();
std::string moduleTypeString = sqlStatementQuery.getColumn(1).getString();
std::string metadataString = sqlStatementQuery.getColumn(2).getString();
std::string dataString = sqlStatementQuery.getColumn(3).getString();

nlohmann::json outputJson = {{"moduleName", ""}, {"moduleType", ""}, {"metadata", ""}, {"data", {}}};

if (!dataString.empty())
{
outputJson["data"] = nlohmann::json::parse(dataString);
}

if (!metadataString.empty())
{
outputJson["metadata"] = metadataString;
}

if (!moduleNameString.empty())
{
outputJson["moduleName"] = moduleNameString;
}

if (!moduleTypeString.empty())
{
outputJson["moduleType"] = moduleTypeString;
}

messages.push_back(outputJson);
if (maxSize)
{
size_t messageSize = outputJson.dump().size();
LogError("/*/*/*/*/ Size: {} - '{}'.", messageSize, outputJson.dump());
if (sizeAccum + messageSize > maxSize)
{
break;
}
}
}
}

return messages;
}
catch (const std::exception& e)
{
LogError("Error during RetrieveMultiple operation: {}.", e.what());
return {};
}
}

nlohmann::json SQLiteStorage::RetrieveBySize(size_t n,
const std::string& tableName,
const std::string& moduleName,
[[maybe_unused]] const std::string& moduleType)
{
std::string selectQuery;
if (moduleName.empty())
{
constexpr std::string_view SELECT_MULTIPLE_QUERY {
"SELECT module_name, module_type, metadata, message FROM {} ORDER BY rowid ASC;"};
selectQuery = fmt::format(SELECT_MULTIPLE_QUERY, tableName);
}
else
{
constexpr std::string_view SELECT_MULTIPLE_QUERY {
"SELECT module_name, module_type, metadata, message FROM {} WHERE "
"module_name LIKE \"{}\" ORDER BY rowid ASC;"};
selectQuery = fmt::format(SELECT_MULTIPLE_QUERY, tableName, moduleName);
}

SQLite::Statement query(*m_db, selectQuery);
return ProcessRequest(query, n);
}
12 changes: 10 additions & 2 deletions src/agent/multitype_queue/src/sqlitestorage.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,10 +120,15 @@ class SQLiteStorage : public Persistence
*/
int GetElementCount(const std::string& tableName, const std::string& moduleName = "") override;


//TODO: doc
// TODO: doc
int GetElementsStoredSize(const std::string& tableName) override;

// TODO: doc
nlohmann::json RetrieveBySize(size_t n,
const std::string& tableName,
const std::string& moduleName = "",
[[maybe_unused]] const std::string& moduleType = "");

private:
/**
* @brief Initialize the table in the SQLite database.
Expand All @@ -144,6 +149,9 @@ class SQLiteStorage : public Persistence
*/
void ReleaseDatabaseAccess();

// TODO: fill doc
nlohmann::json ProcessRequest(SQLite::Statement& sqlStatementQuery, size_t maxSize = 0);

/**
* @brief The name of the SQLite database file.
*/
Expand Down
24 changes: 20 additions & 4 deletions src/agent/multitype_queue/tests/sqlitestorage_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,14 +204,30 @@ TEST_F(SQLiteStorageTest, MessagesSizes)
auto val = storage->Store(messages, tableName);
EXPECT_EQ(val, 2);

auto retrievedMessages = storage->GetElementsStoredSize(tableName);
EXPECT_EQ(retrievedMessages, 32);
auto storedSizes = storage->GetElementsStoredSize(tableName);
EXPECT_EQ(storedSizes, 32);

val = storage->Store(messages, tableName);
EXPECT_EQ(val, 2);

retrievedMessages = storage->GetElementsStoredSize(tableName);
EXPECT_EQ(retrievedMessages, 64);
storedSizes = storage->GetElementsStoredSize(tableName);
EXPECT_EQ(storedSizes, 64);
}

TEST_F(SQLiteStorageTest, GetMessagesBySize)
{
auto messages = nlohmann::json::array();
messages.push_back({{"key", "value1"}});
messages.push_back({{"key", "value2"}});
auto val = storage->Store(messages, tableName);
EXPECT_EQ(val, 2);

//Stored: '{"data":{"key":"value1"},"metadata":"","moduleName":"","moduleType":""}' size=72
auto storedSizes = storage->GetElementsStoredSize(tableName);
EXPECT_EQ(storedSizes, 32);

auto retrievedMessages = storage->RetrieveBySize(static_cast<size_t>(storedSizes), tableName);
EXPECT_EQ(retrievedMessages.size(), 2);
}

class SQLiteStorageMultithreadedTest : public ::testing::Test
Expand Down

0 comments on commit 7635341

Please sign in to comment.