Collect memory usage data

[pflanze]

We need checks for controlling memory usage.

There are two possible data sources:

1. whole process usage as per kernel report (Linux)

2. allocator statistics (userspace); this is expected to be much faster to retrieve and more precise and have more info (e.g. also number of allocations/frees, also per-thread?) than the data from the kernel.

And there are two possible logging approaches:

a. as linear log (to a separate file?) via polling, e.g. every 10 ms
b. log as part of probes

Approach b requires an efficient data source. Retrieving from the Kernel is about 150x slower than retrieving the real time (which is also via the kernel); thus that would necessitate to do the memory collection statistically in many places--either use EVOBENCH_SCOPE_EVERY and have all timings be statistical, but then many EVOBENCH_SCOPE probes would have to be changed to _EVERY; or do the memory collection at (yet) a lower frequency than the rest. Neither solution is very palatable. It appears that probes are best combined with a userspace data source.

So, the most sensible approach is to:

1. implement whole process usage (from Kernel) via polling now.

2. implement probe based collection via allocator statistics later.

Tackling step 1 now.

[pflanze]

Here is some testing to find the most efficient way to source memory data from the kernel (the most efficient is get_memory_usage--that is still 150x slower than retrieving the real time as mentioned above) (PS: forgot the close call in that procedure!):

#include <assert.h>
#include <time.h>
#include <stdlib.h>
#include <stdio.h>
#include <errno.h>
#include <sys/sysinfo.h>

#include <optional>
#include <cstdint>
#include <string>
#include <regex>
#include <fstream>
#include <filesystem>

// -----------------------------------------------------------------------------
// in SILO

std::optional<uint32_t> parseVmRSSLine(const std::string& line) {
   static const std::regex vmRssRegex("VmRSS:\\s*(\\d+) kB");
   std::smatch match;

   if (std::regex_search(line, match, vmRssRegex) && match.size() > 1) {
      try {
         return std::stol(match.str(1));
      } catch (const std::out_of_range& oor) {
         // SPDLOG_DEBUG(
         //    "parseVmRSSLine: VmRSS value out of range for long: {} - {}", match.str(1), oor.what()
         // );
         return std::nullopt;
      } catch (const std::invalid_argument& ia) {
         // SPDLOG_DEBUG(
         //    "parseVmRSSLine: Invalid argument for stol: {} - {}", match.str(1), ia.what()
         // );
         return std::nullopt;
      }
   }
   return std::nullopt;
}

std::optional<uint32_t> getResidentSetSize() noexcept {
   std::filesystem::path path = "/proc/self/status";
   std::ifstream file(path);
   if (!file.is_open()) {
      // SPDLOG_DEBUG("getResidentSetSize: Could not open status file {}.", path.string());
      return std::nullopt;
   }

   std::string line;
   while (std::getline(file, line)) {
      if (line.rfind("VmRSS:", 0) == 0) {
         return parseVmRSSLine(line);
      }
   }

   // SPDLOG_DEBUG("getResidentSetSize: VmRSS line not found in {}", path.string());
   return std::nullopt;
}

// -----------------------------------------------------------------------------
// gpt
#include <iostream>
#include <fstream>
#include <string>
#include <unistd.h>

static
void get_memory_usage_kb(double& vm_usage, double& resident_set) {
    vm_usage = 0.0;
    resident_set = 0.0;

    std::ifstream stat_stream("/proc/self/stat", std::ios_base::in);

    std::string dummy;
    unsigned long vsize;
    long rss;

    // Skip the first 22 fields
    for (int i = 0; i < 22; ++i) stat_stream >> dummy;

    stat_stream >> vsize >> rss;
    stat_stream.close();

    long page_size_kb = sysconf(_SC_PAGE_SIZE) / 1024; // in case it's not 4KB
    vm_usage = vsize / 1024.0;
    resident_set = rss * page_size_kb;
}

// -----------------------------------------------------------------------------
// me

#include <fcntl.h>
#include <string_view>

static
void get_memory_usage(std::string& vsize, std::string& rss) {
#define BUFSIZ_ 1000
    char buf[BUFSIZ_];
    int fd = open("/proc/self/stat", O_RDONLY);
    assert(fd >= 0);
    ssize_t result_len = read(fd, buf, BUFSIZ_);
    assert(result_len >= 0);
    assert(result_len < BUFSIZ_);
#undef BUFSIZ_
    // 2596146 (cat) R 2596139 2596146 2596139 34856 2596146 4194304 92 0 0 0 0 0 0 0 20 0 1 0 235093164 6111232 235 18446744073709551615 94182750162944 94182750182825 140736636439104 0 0 0 0 128 0 0 0 0 17 3 0 0 0 0 0 94182750198832 94182750200448 94183773274112 140736636441297 140736636441317 140736636441317 140736636444651 0

    size_t len = (size_t)result_len;
    size_t i = 0;
    
    // Skip the first 22 fields
    int seen_spaces = 0;
    for (; i < len; ++i) {
        if (buf[i] == ' ') {
            seen_spaces++;
            if (seen_spaces == 22) {
                goto capture;
            }
        }
    }
    abort();
    
capture:
    i++;
    size_t start_vsize = i;
    for (; i < len; ++i) {
        if (buf[i] == ' ') {
            goto capture2;
        }
    }
    abort();

capture2:
    size_t end_vsize = i;
    i++;
    size_t start_rss = i;
    for (; i < len; ++i) {
        if (buf[i] == ' ') {
            goto capture3;
        }
    }
    abort();

capture3:
    size_t end_rss = i;

    vsize = std::string_view { &buf[start_vsize], end_vsize - start_vsize };
    rss = std::string_view { &buf[start_rss], end_rss - start_rss };
}

// -----------------------------------------------------------------------------

int main() {
    std::string vsize { "foo" };
    std::string rss { "bar" };
    for (int i = 0; i < 10; i++) {
    {
        struct timespec start;
        struct timespec end;
        assert(clock_gettime(CLOCK_REALTIME, &start) >= 0);
        struct sysinfo si;
        if (sysinfo(&si) < 0) {
            perror("sysinfo");
            exit(1);
        }
        // ACHSOOOOOOOOO das is SYSTEM not PROCESS !!!!!!!
        // Number of current processes  etc too !
        assert(clock_gettime(CLOCK_REALTIME, &end) >= 0);

        printf("(%li + %li / 1000000000) - (%li + %li / 1000000000)\n",
               end.tv_sec, end.tv_nsec,
               start.tv_sec, start.tv_nsec);
    }

    {
        struct timespec start;
        struct timespec end;
        assert(clock_gettime(CLOCK_REALTIME, &start) >= 0);

        auto sz = getResidentSetSize();

        assert(clock_gettime(CLOCK_REALTIME, &end) >= 0);

        printf("getResidentSetSize() = %u\n", *sz);
        
        printf("(%li + %li / 1000000000) - (%li + %li / 1000000000)\n",
               end.tv_sec, end.tv_nsec,
               start.tv_sec, start.tv_nsec);
    }

    {
        struct timespec start;
        struct timespec end;
        assert(clock_gettime(CLOCK_REALTIME, &start) >= 0);

        double vm_usage;
        double resident_set;
        get_memory_usage_kb(vm_usage, resident_set);

        assert(clock_gettime(CLOCK_REALTIME, &end) >= 0);

        printf("get_memory_usage_kb() = %f, %f\n", vm_usage, resident_set);
        
        printf("(%li + %li / 1000000000) - (%li + %li / 1000000000)\n",
               end.tv_sec, end.tv_nsec,
               start.tv_sec, start.tv_nsec);
    }

    {
        struct timespec start;
        struct timespec end;
        assert(clock_gettime(CLOCK_REALTIME, &start) >= 0);

        get_memory_usage(vsize, rss);

        assert(clock_gettime(CLOCK_REALTIME, &end) >= 0);

        printf("get_memory_usage() = %s, %s\n", vsize.c_str(), rss.c_str());
        
        printf("(%li + %li / 1000000000) - (%li + %li / 1000000000)\n",
               end.tv_sec, end.tv_nsec,
               start.tv_sec, start.tv_nsec);
    }
    }
    
    return 0;
}

[pflanze]

These are the steps:

• parse the time numbers from a lines based log
• feed them into my existing Stats
  • add a "Bytes" type (type parameter for Stats) for representing memory
  • later possibly make a type parameter for Stats that represents the data as logarithms
• generate the usual Excel files from those (just with only one row)
• the above points in a separate binary for easy manual use
• use that binary in evobench-run daemon
  • add a config for it (possibly later)

~1-2 d of work.