From 48f3b87ed4291d72c1ecbaccce6df55df1795f87 Mon Sep 17 00:00:00 2001
From: Elise <Elise.Dupont@etu.univ-grenoble-alpes.fr>
Date: Sun, 25 Feb 2024 08:05:17 +0100
Subject: [PATCH 01/11] fonctionne pas mais j'ai peur de me faire voler mon pc
 et de tout perdre

---
 colmet.nix                      | 94 +++++++++++++++++++++++++++++++++
 opensearch-mono/composition.nix | 52 ++----------------
 opensearch-mono/flake.lock      |  6 +--
 3 files changed, 101 insertions(+), 51 deletions(-)
 create mode 100644 colmet.nix

diff --git a/colmet.nix b/colmet.nix
new file mode 100644
index 0000000..0bea2be
--- /dev/null
+++ b/colmet.nix
@@ -0,0 +1,94 @@
+{ lib, pkgs, config,  ... }:
+
+let
+  colmet = pkgs.python3Packages.buildPythonApplication rec {
+    name = "colmet-${version}";
+    version = "0.5.4";
+
+    src = pkgs.fetchFromGitHub {
+      owner = "oar-team";
+      repo = "colmet";
+      rev = "4cc29227fcaf5236d97dde74b9a52e04250a5b77";
+      sha256 = "1g2m6crdmlgk8c57qa1nss20128dnw9x58yg4r5wdc7zliicahqq";
+    };
+
+    buildInputs = [ pkgs.powercap ];
+
+    propagatedBuildInputs = with pkgs.python3Packages; [
+      pyinotify
+      pyzmq
+      tables
+      requests
+    ];
+
+    preBuild = ''
+      mkdir -p $out/lib
+      sed -i "s#/usr/lib/#$out/lib/#g" colmet/node/backends/perfhwstats.py
+      sed -i "s#/usr/lib/#$out/lib/#g" colmet/node/backends/RAPLstats.py
+      sed -i "s#/usr/lib/#$out/lib/#g" colmet/node/backends/lib_perf_hw/makefile
+      sed -i "s#/usr/lib/#$out/lib/#g" colmet/node/backends/lib_rapl/makefile
+    '';
+
+    # Tests do not pass
+    doCheck = false;
+
+    meta = with lib; {
+      description = "Collecting metrics about process running in cpuset and in a distributed environnement";
+      homepage = https://github.com/oar-team/colmet;
+      platforms = pkgs.powercap.meta.platforms;
+      licence = licenses.gpl2;
+      longDescription = ''
+    '';
+    };
+  };
+  cfg-collector = config.services.colmet-collector;
+  cfg-node = config.services.colmet-node;
+in
+with lib;
+{
+  config.environment.systemPackages = [ colmet ];
+
+  #option to enable the colmet collector
+  options.services.colmet-collector = {
+    enable = mkEnableOption (mdDoc "Enable the Colmet collector service");
+  };
+
+  #option to enable colmet nodes
+  options.services.colmet-node = {
+    enable = mkEnableOption (mdDoc "Enable the Colmet node service");
+  };
+
+  #if colmet-collector is enable
+  config.systemd.services.colmet-collector = mkIf cfg-collector.enable {
+    description = "Colmet collector";
+    wantedBy = [ "multi-user.target" ];
+    after = [ "network-online.target" (mkIf config.services.opensearch.enable "opensearch.service") ];
+    serviceConfig = {
+      ExecStart = "${colmet}/bin/colmet-collector -vvv \
+        --zeromq-bind-uri tcp://192.168.0.1:5556 \
+        --buffer-size 5000 \
+        --sample-period 3 \
+        --elastic-host https://127.0.0.1:9200 \
+        --elastic-index-prefix colmet_dahu_ 2>>/var/log/colmet_err.log >> /var/log/colmet.log \
+        --no-check-certificates";
+    };
+  };
+
+  #if colmet-node is enable
+  config.systemd.services.colmet-node = mkIf cfg-node.enable {
+    description = "Colmet node";
+    wantedBy = [ "multi-user.target" ];
+    after = [ "network-online.target" (mkIf config.services.opensearch.enable "opensearch.service") ];
+    serviceConfig = {
+      ExecStart = "${colmet}/bin/colmet-node -vvv --zeromq-uri tcp://127.0.0.1:5556";
+    };
+  };
+
+  config.boot.kernelPatches = [ {
+    name = "crashdump-config";
+    patch = null;
+    extraConfig = "MEMORY_FAILURE y";
+  }];
+
+
+}
diff --git a/opensearch-mono/composition.nix b/opensearch-mono/composition.nix
index 3a8ead9..17f7bdd 100644
--- a/opensearch-mono/composition.nix
+++ b/opensearch-mono/composition.nix
@@ -13,10 +13,10 @@ in
   roles = {
     opensearch = { pkgs, config, lib, ... }:
       {
-        imports = [ ../opensearch-dashboards.nix ];
+        imports = [ ../opensearch-dashboards.nix ../colmet.nix ];
 
         environment.noXlibs = false;
-        environment.systemPackages = with pkgs; [ opensearch-fixed vector jq ];
+        environment.systemPackages = with pkgs; [ opensearch-fixed jq ];
 
         systemd.services.opensearch.serviceConfig.ExecStartPre = [
           "${pkgs.writeShellScript
@@ -105,54 +105,10 @@ in
           ];
         };
 
-        # Vector est système de gestion de logs
-        services.vector = {
-          enable = true;
-          journaldAccess = true;
-          settings = {
-            sources = {
-              "in" = {
-                type = "stdin";
-              };
-              "systemd" = {
-                type = "journald";
-              };
-            };
-            sinks = {
-              out = {
-                inputs = [ "in" ];
-                type = "console";
-                encoding = {
-                  codec = "text";
-                };
-              };
-              opensearch = {
-                inputs = [ "systemd" ];
-                type = "elasticsearch";
-                endpoints = ["https://127.0.0.1:9200"];
-                auth = {
-                  strategy = "basic";
-                  user = "admin";
-                  password = "admin";
-                };
-                tls.verify_certificate = false;
-              };
-            };
-          };
-        };
-
         services.opensearch-dashboards.enable = true;
+        services.colmet-collector.enable = true;
+        services.colmet-node.enable = true;
 
-        environment.variables = {
-          # La variable "VECTOR_CONFIG" défini le chemin de la configuration à utiliser quand on
-          # lance la commande `vector`. Le service Systemd génère une config à partir de `services.vector.settings`
-          # et s'assure que le service utilise bien ce fichier. Mais il faut aussi indiquer où ce trouve
-          # ce fichier de configuration à l'outil en ligne de commande disponible dans le PATH.
-          # On parse la configuration systemd pour récupérer le chemin du fichier.
-          VECTOR_CONFIG = lib.lists.last (
-            builtins.split " " config.systemd.services.vector.serviceConfig.ExecStart
-          );
-        };
       };
   };
 
diff --git a/opensearch-mono/flake.lock b/opensearch-mono/flake.lock
index 50ea78c..3cfd504 100644
--- a/opensearch-mono/flake.lock
+++ b/opensearch-mono/flake.lock
@@ -55,11 +55,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1707091808,
-        "narHash": "sha256-LahKBAfGbY836gtpVNnWwBTIzN7yf/uYM/S0g393r0Y=",
+        "lastModified": 1708294118,
+        "narHash": "sha256-evZzmLW7qoHXf76VCepvun1esZDxHfVRFUJtumD7L2M=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "9f2ee8c91ac42da3ae6c6a1d21555f283458247e",
+        "rev": "e0da498ad77ac8909a980f07eff060862417ccf7",
         "type": "github"
       },
       "original": {

From e4ff3521f5cc091273c3d0a5484463c4b43ae75e Mon Sep 17 00:00:00 2001
From: Elise <Elise.Dupont@etu.univ-grenoble-alpes.fr>
Date: Tue, 5 Mar 2024 13:59:35 +0100
Subject: [PATCH 02/11] collector ok node pas ok

---
 colmet.nix                                  |  10 +-
 hardwarecorrupted.patch                     |  13 ++
 opensearch-mono/colmet_hardwarecorrupted.py | 197 ++++++++++++++++++++
 opensearch-mono/colmet_original.py          | 197 ++++++++++++++++++++
 4 files changed, 409 insertions(+), 8 deletions(-)
 create mode 100644 hardwarecorrupted.patch
 create mode 100644 opensearch-mono/colmet_hardwarecorrupted.py
 create mode 100644 opensearch-mono/colmet_original.py

diff --git a/colmet.nix b/colmet.nix
index 0bea2be..8c06cd0 100644
--- a/colmet.nix
+++ b/colmet.nix
@@ -32,6 +32,8 @@ let
     # Tests do not pass
     doCheck = false;
 
+    patches= [./hardwarecorrupted.patch];
+
     meta = with lib; {
       description = "Collecting metrics about process running in cpuset and in a distributed environnement";
       homepage = https://github.com/oar-team/colmet;
@@ -83,12 +85,4 @@ with lib;
       ExecStart = "${colmet}/bin/colmet-node -vvv --zeromq-uri tcp://127.0.0.1:5556";
     };
   };
-
-  config.boot.kernelPatches = [ {
-    name = "crashdump-config";
-    patch = null;
-    extraConfig = "MEMORY_FAILURE y";
-  }];
-
-
 }
diff --git a/hardwarecorrupted.patch b/hardwarecorrupted.patch
new file mode 100644
index 0000000..31ec7f3
--- /dev/null
+++ b/hardwarecorrupted.patch
@@ -0,0 +1,13 @@
+--- a/colmet/common/metrics/procstats.py
++++ b/colmet/common/metrics/procstats.py
+@@ -53,8 +53,8 @@
+                                 'meminfo_vmallocused'),
+         'meminfo_vmallocchunk': (UInt64(), 'kbytes', 'none',
+                                  'meminfo_vmallocchunk'),
+-        'meminfo_hardwarecorrupted': (UInt64(), 'kbytes', 'none',
+-                                      'meminfo_hardwarecorrupted'),
++        #'meminfo_hardwarecorrupted': (UInt64(), 'kbytes', 'none',
++        #                              'meminfo_hardwarecorrupted'),
+         'meminfo_anonhugepages': (UInt64(), 'kbytes', 'none',
+                                   'meminfo_anonhugepages'),
+         'meminfo_hugepages_total': (UInt64(), 'count', 'none',
diff --git a/opensearch-mono/colmet_hardwarecorrupted.py b/opensearch-mono/colmet_hardwarecorrupted.py
new file mode 100644
index 0000000..9bb26a2
--- /dev/null
+++ b/opensearch-mono/colmet_hardwarecorrupted.py
@@ -0,0 +1,197 @@
+import logging
+
+LOG = logging.getLogger()
+
+from .base import UInt64, UFloat, BaseCounters, String
+
+class ProcstatsCounters(BaseCounters):
+    __metric_name__ = 'procstats_default'
+
+    counters_procstats = {
+        # 'key': ( offset,length, type, repr, acc )
+        'uptime_total': (UInt64(), 'sec', 'none', 'uptime.total'),
+        'uptime_idle': (UInt64(), 'sec', 'none', 'uptime.idle'),
+        'meminfo_memtotal': (UInt64(), 'kbytes', 'none', 'meminfo_memtotal'),
+        'meminfo_memfree': (UInt64(), 'kbytes', 'none', 'meminfo_memfree'),
+        'meminfo_buffers': (UInt64(), 'kbytes', 'none', 'meminfo_buffers'),
+        'meminfo_cached': (UInt64(), 'kbytes', 'none', 'meminfo_cached'),
+        'meminfo_swapcached': (UInt64(), 'kbytes', 'none',
+                               'meminfo_swapcached'),
+        'meminfo_active': (UInt64(), 'kbytes', 'none', 'meminfo_active'),
+        'meminfo_inactive': (UInt64(), 'kbytes', 'none', 'meminfo_inactive'),
+        'meminfo_unevictable': (UInt64(), 'kbytes', 'none',
+                                'meminfo_unevictable'),
+        'meminfo_mlocked': (UInt64(), 'kbytes', 'none', 'meminfo_mlocked'),
+        'meminfo_swaptotal': (UInt64(), 'kbytes', 'none', 'meminfo_swaptotal'),
+        'meminfo_swapfree': (UInt64(), 'kbytes', 'none', 'meminfo_swapfree'),
+        'meminfo_dirty': (UInt64(), 'kbytes', 'none', 'meminfo_dirty'),
+        'meminfo_writeback': (UInt64(), 'kbytes', 'none', 'meminfo_writeback'),
+        'meminfo_anonpages': (UInt64(), 'kbytes', 'none', 'meminfo_anonpages'),
+        'meminfo_mapped': (UInt64(), 'kbytes', 'none', 'meminfo_mapped'),
+        'meminfo_shmem': (UInt64(), 'kbytes', 'none', 'meminfo_shmem'),
+        'meminfo_slab': (UInt64(), 'kbytes', 'none', 'meminfo_slab'),
+        'meminfo_sreclaimable': (UInt64(), 'kbytes', 'none',
+                                 'meminfo_sreclaimable'),
+        'meminfo_sunreclaim': (UInt64(), 'kbytes', 'none',
+                               'meminfo_sunreclaim'),
+        'meminfo_kernelstack': (UInt64(), 'kbytes', 'none',
+                                'meminfo_kernelstack'),
+        'meminfo_pagetables': (UInt64(), 'kbytes', 'none',
+                               'meminfo_pagetables'),
+        'meminfo_nfs_unstable': (UInt64(), 'kbytes', 'none',
+                                 'meminfo_nfs_unstable'),
+        'meminfo_bounce': (UInt64(), 'kbytes', 'none', 'meminfo_bounce'),
+        'meminfo_writebacktmp': (UInt64(), 'kbytes', 'none',
+                                 'meminfo_writebacktmp'),
+        'meminfo_commitlimit': (UInt64(), 'kbytes', 'none',
+                                'meminfo_commitlimit'),
+        'meminfo_committed_as': (UInt64(), 'kbytes', 'none',
+                                 'meminfo_committed_as'),
+        'meminfo_vmalloctotal': (UInt64(), 'kbytes', 'none',
+                                 'meminfo_vmalloctotal'),
+        'meminfo_vmallocused': (UInt64(), 'kbytes', 'none',
+                                'meminfo_vmallocused'),
+        'meminfo_vmallocchunk': (UInt64(), 'kbytes', 'none',
+                                 'meminfo_vmallocchunk'),
+        #'meminfo_hardwarecorrupted': (UInt64(), 'kbytes', 'none',
+        #                              'meminfo_hardwarecorrupted'),
+        'meminfo_anonhugepages': (UInt64(), 'kbytes', 'none',
+                                  'meminfo_anonhugepages'),
+        'meminfo_hugepages_total': (UInt64(), 'count', 'none',
+                                    'meminfo_hugepages_total'),
+        'meminfo_hugepages_free': (UInt64(), 'count', 'none',
+                                   'meminfo_hugepages_free'),
+        'meminfo_hugepages_rsvd': (UInt64(), 'count', 'none',
+                                   'meminfo_hugepages_rsvd'),
+        'meminfo_hugepages_surp': (UInt64(), 'count', 'none',
+                                   'meminfo_hugepages_surp'),
+        'meminfo_hugepagesize': (UInt64(), 'count', 'none',
+                                 'meminfo_hugepagesize'),
+        'meminfo_directmap4k': (UInt64(), 'kbytes', 'none',
+                                'meminfo_directmap4k'),
+        'meminfo_directmap2m': (UInt64(), 'kbytes', 'none',
+                                'meminfo_directmap2m'),
+        'vmstat_pgpgin': (UInt64(), 'count', 'none', 'vmstat_pgpgin'),
+        'vmstat_pgpgout': (UInt64(), 'count', 'none', 'vmstat_pgpgout'),
+        'vmstat_pswpin': (UInt64(), 'count', 'none', 'vmstat_pswpin'),
+        'vmstat_pswpout': (UInt64(), 'count', 'none', 'vmstat_pswpout'),
+        'vmstat_pgfault': (UInt64(), 'count', 'none', 'vmstat_pgfault'),
+        'vmstat_pgmajfault': (UInt64(), 'count', 'none', 'vmstat_pgmajfault'),
+        'stat_cpu_user': (UInt64(), 'n/a', 'none', 'stat_cpu_user'),
+        'stat_cpu_nice': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
+        'stat_cpu_system': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
+        'stat_cpu_idle': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
+        'stat_cpu_iowait': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
+        'stat_cpu_irq': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
+        'stat_cpu_softirq': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
+        'stat_cpu_guest': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
+        'stat_cpu_guest_nice': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
+        'stat_intr': (UInt64(), 'n/a', 'none', 'stat_intr'),
+        'stat_ctxt': (UInt64(), 'n/a', 'none', 'stat_ctxt'),
+        'stat_processes': (UInt64(), 'n/a', 'none', 'stat_processes'),
+        'stat_procs_blocked': (UInt64(), 'n/a', 'none', 'stat_procs_blocked'),
+        'loadavg_1min': (UFloat(), 'n/a', 'none', 'loadavg_1min'),
+        'loadavg_5min': (UFloat(), 'n/a', 'none', 'loadavg_5min'),
+        'loadavg_15min': (UFloat(), 'n/a', 'none', 'loadavg_15min'),
+        'loadavg_runnable': (UFloat(), 'n/a', 'none', 'loadavg_runnable'),
+        'loadavg_total_threads': (UFloat(), 'n/a', 'none',
+                                  'loadavg_total_threads'),
+        'involved_jobs': (String(8192), 'string', 'none', 'involved_jobs')
+    }
+
+    counters_procstats_to_get = [
+        'uptime_total',
+        'uptime_idle',
+        'meminfo_memtotal',
+        'meminfo_memfree',
+        'meminfo_buffers',
+        'meminfo_cached',
+        'meminfo_swapcached',
+        'meminfo_active',
+        'meminfo_inactive',
+        'meminfo_unevictable',
+        'meminfo_mlocked',
+        'meminfo_swaptotal',
+        'meminfo_swapfree',
+        'meminfo_dirty',
+        'meminfo_writeback',
+        'meminfo_anonpages',
+        'meminfo_mapped',
+        'meminfo_shmem',
+        'meminfo_slab',
+        'meminfo_sreclaimable',
+        'meminfo_sunreclaim',
+        'meminfo_kernelstack',
+        'meminfo_pagetables',
+        'meminfo_nfs_unstable',
+        'meminfo_bounce',
+        'meminfo_writebacktmp',
+        'meminfo_commitlimit',
+        'meminfo_committed_as',
+        'meminfo_vmalloctotal',
+        'meminfo_vmallocused',
+        'meminfo_vmallocchunk',
+        #'meminfo_hardwarecorrupted', # not supported on some kernel
+        'meminfo_anonhugepages',
+        'meminfo_hugepages_total',
+        'meminfo_hugepages_free',
+        'meminfo_hugepages_rsvd',
+        'meminfo_hugepages_surp',
+        'meminfo_hugepagesize',
+        'meminfo_directmap4k',
+        'meminfo_directmap2m',
+        'vmstat_pgpgin',
+        'vmstat_pgpgout',
+        'vmstat_pswpin',
+        'vmstat_pswpout',
+        'vmstat_pgfault',
+        'vmstat_pgmajfault',
+        'stat_cpu_user',
+        'stat_cpu_nice',
+        'stat_cpu_system',
+        'stat_cpu_idle',
+        'stat_cpu_iowait',
+        'stat_cpu_irq',
+        'stat_cpu_softirq',
+        'stat_cpu_guest',
+        'stat_cpu_guest_nice',
+        'stat_intr',
+        'stat_ctxt',
+        'stat_processes',
+        'stat_procs_blocked',
+        'loadavg_1min',
+        'loadavg_5min',
+        'loadavg_15min',
+        'loadavg_runnable',
+        'loadavg_total_threads',
+        # 'sys_numa_zoneallocs',
+        # 'sys_numa_zoneallocs',
+        # 'sys_numa_foreign_allocs',
+        # 'sys_numa_allocation',
+        # 'sys_numa_allocation',
+        # 'sys_numa_interleave',
+        'involved_jobs'
+    ]
+
+    _counters = []
+    for c_name in counters_procstats_to_get:
+        (c_type, c_repr, c_acc, c_descr) = counters_procstats[c_name]
+        _counters.append((c_name, c_type, c_repr, c_acc, c_descr))
+
+    @classmethod
+    def get_zero_counters(cls):
+        return cls(procstats_buffer=None, raw=None)
+
+    @classmethod
+    def fetch(cls, procstats_backend):
+        return procstats_backend.get_procstats()
+
+    def __init__(self, procstats_buffer=None, raw=None):
+        BaseCounters.__init__(self, raw=raw)
+        if raw is not None:
+            pass
+        elif procstats_buffer is None:
+            self._empty_fill()
+        else:
+            for name in ProcstatsCounters._counter_definitions:
+                self._counter_values[name] = procstats_buffer[name]
\ No newline at end of file
diff --git a/opensearch-mono/colmet_original.py b/opensearch-mono/colmet_original.py
new file mode 100644
index 0000000..6d22572
--- /dev/null
+++ b/opensearch-mono/colmet_original.py
@@ -0,0 +1,197 @@
+import logging
+
+LOG = logging.getLogger()
+
+from .base import UInt64, UFloat, BaseCounters, String
+
+class ProcstatsCounters(BaseCounters):
+    __metric_name__ = 'procstats_default'
+
+    counters_procstats = {
+        # 'key': ( offset,length, type, repr, acc )
+        'uptime_total': (UInt64(), 'sec', 'none', 'uptime.total'),
+        'uptime_idle': (UInt64(), 'sec', 'none', 'uptime.idle'),
+        'meminfo_memtotal': (UInt64(), 'kbytes', 'none', 'meminfo_memtotal'),
+        'meminfo_memfree': (UInt64(), 'kbytes', 'none', 'meminfo_memfree'),
+        'meminfo_buffers': (UInt64(), 'kbytes', 'none', 'meminfo_buffers'),
+        'meminfo_cached': (UInt64(), 'kbytes', 'none', 'meminfo_cached'),
+        'meminfo_swapcached': (UInt64(), 'kbytes', 'none',
+                               'meminfo_swapcached'),
+        'meminfo_active': (UInt64(), 'kbytes', 'none', 'meminfo_active'),
+        'meminfo_inactive': (UInt64(), 'kbytes', 'none', 'meminfo_inactive'),
+        'meminfo_unevictable': (UInt64(), 'kbytes', 'none',
+                                'meminfo_unevictable'),
+        'meminfo_mlocked': (UInt64(), 'kbytes', 'none', 'meminfo_mlocked'),
+        'meminfo_swaptotal': (UInt64(), 'kbytes', 'none', 'meminfo_swaptotal'),
+        'meminfo_swapfree': (UInt64(), 'kbytes', 'none', 'meminfo_swapfree'),
+        'meminfo_dirty': (UInt64(), 'kbytes', 'none', 'meminfo_dirty'),
+        'meminfo_writeback': (UInt64(), 'kbytes', 'none', 'meminfo_writeback'),
+        'meminfo_anonpages': (UInt64(), 'kbytes', 'none', 'meminfo_anonpages'),
+        'meminfo_mapped': (UInt64(), 'kbytes', 'none', 'meminfo_mapped'),
+        'meminfo_shmem': (UInt64(), 'kbytes', 'none', 'meminfo_shmem'),
+        'meminfo_slab': (UInt64(), 'kbytes', 'none', 'meminfo_slab'),
+        'meminfo_sreclaimable': (UInt64(), 'kbytes', 'none',
+                                 'meminfo_sreclaimable'),
+        'meminfo_sunreclaim': (UInt64(), 'kbytes', 'none',
+                               'meminfo_sunreclaim'),
+        'meminfo_kernelstack': (UInt64(), 'kbytes', 'none',
+                                'meminfo_kernelstack'),
+        'meminfo_pagetables': (UInt64(), 'kbytes', 'none',
+                               'meminfo_pagetables'),
+        'meminfo_nfs_unstable': (UInt64(), 'kbytes', 'none',
+                                 'meminfo_nfs_unstable'),
+        'meminfo_bounce': (UInt64(), 'kbytes', 'none', 'meminfo_bounce'),
+        'meminfo_writebacktmp': (UInt64(), 'kbytes', 'none',
+                                 'meminfo_writebacktmp'),
+        'meminfo_commitlimit': (UInt64(), 'kbytes', 'none',
+                                'meminfo_commitlimit'),
+        'meminfo_committed_as': (UInt64(), 'kbytes', 'none',
+                                 'meminfo_committed_as'),
+        'meminfo_vmalloctotal': (UInt64(), 'kbytes', 'none',
+                                 'meminfo_vmalloctotal'),
+        'meminfo_vmallocused': (UInt64(), 'kbytes', 'none',
+                                'meminfo_vmallocused'),
+        'meminfo_vmallocchunk': (UInt64(), 'kbytes', 'none',
+                                 'meminfo_vmallocchunk'),
+        'meminfo_hardwarecorrupted': (UInt64(), 'kbytes', 'none',
+                                      'meminfo_hardwarecorrupted'),
+        'meminfo_anonhugepages': (UInt64(), 'kbytes', 'none',
+                                  'meminfo_anonhugepages'),
+        'meminfo_hugepages_total': (UInt64(), 'count', 'none',
+                                    'meminfo_hugepages_total'),
+        'meminfo_hugepages_free': (UInt64(), 'count', 'none',
+                                   'meminfo_hugepages_free'),
+        'meminfo_hugepages_rsvd': (UInt64(), 'count', 'none',
+                                   'meminfo_hugepages_rsvd'),
+        'meminfo_hugepages_surp': (UInt64(), 'count', 'none',
+                                   'meminfo_hugepages_surp'),
+        'meminfo_hugepagesize': (UInt64(), 'count', 'none',
+                                 'meminfo_hugepagesize'),
+        'meminfo_directmap4k': (UInt64(), 'kbytes', 'none',
+                                'meminfo_directmap4k'),
+        'meminfo_directmap2m': (UInt64(), 'kbytes', 'none',
+                                'meminfo_directmap2m'),
+        'vmstat_pgpgin': (UInt64(), 'count', 'none', 'vmstat_pgpgin'),
+        'vmstat_pgpgout': (UInt64(), 'count', 'none', 'vmstat_pgpgout'),
+        'vmstat_pswpin': (UInt64(), 'count', 'none', 'vmstat_pswpin'),
+        'vmstat_pswpout': (UInt64(), 'count', 'none', 'vmstat_pswpout'),
+        'vmstat_pgfault': (UInt64(), 'count', 'none', 'vmstat_pgfault'),
+        'vmstat_pgmajfault': (UInt64(), 'count', 'none', 'vmstat_pgmajfault'),
+        'stat_cpu_user': (UInt64(), 'n/a', 'none', 'stat_cpu_user'),
+        'stat_cpu_nice': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
+        'stat_cpu_system': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
+        'stat_cpu_idle': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
+        'stat_cpu_iowait': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
+        'stat_cpu_irq': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
+        'stat_cpu_softirq': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
+        'stat_cpu_guest': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
+        'stat_cpu_guest_nice': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
+        'stat_intr': (UInt64(), 'n/a', 'none', 'stat_intr'),
+        'stat_ctxt': (UInt64(), 'n/a', 'none', 'stat_ctxt'),
+        'stat_processes': (UInt64(), 'n/a', 'none', 'stat_processes'),
+        'stat_procs_blocked': (UInt64(), 'n/a', 'none', 'stat_procs_blocked'),
+        'loadavg_1min': (UFloat(), 'n/a', 'none', 'loadavg_1min'),
+        'loadavg_5min': (UFloat(), 'n/a', 'none', 'loadavg_5min'),
+        'loadavg_15min': (UFloat(), 'n/a', 'none', 'loadavg_15min'),
+        'loadavg_runnable': (UFloat(), 'n/a', 'none', 'loadavg_runnable'),
+        'loadavg_total_threads': (UFloat(), 'n/a', 'none',
+                                  'loadavg_total_threads'),
+        'involved_jobs': (String(8192), 'string', 'none', 'involved_jobs')
+    }
+
+    counters_procstats_to_get = [
+        'uptime_total',
+        'uptime_idle',
+        'meminfo_memtotal',
+        'meminfo_memfree',
+        'meminfo_buffers',
+        'meminfo_cached',
+        'meminfo_swapcached',
+        'meminfo_active',
+        'meminfo_inactive',
+        'meminfo_unevictable',
+        'meminfo_mlocked',
+        'meminfo_swaptotal',
+        'meminfo_swapfree',
+        'meminfo_dirty',
+        'meminfo_writeback',
+        'meminfo_anonpages',
+        'meminfo_mapped',
+        'meminfo_shmem',
+        'meminfo_slab',
+        'meminfo_sreclaimable',
+        'meminfo_sunreclaim',
+        'meminfo_kernelstack',
+        'meminfo_pagetables',
+        'meminfo_nfs_unstable',
+        'meminfo_bounce',
+        'meminfo_writebacktmp',
+        'meminfo_commitlimit',
+        'meminfo_committed_as',
+        'meminfo_vmalloctotal',
+        'meminfo_vmallocused',
+        'meminfo_vmallocchunk',
+        #'meminfo_hardwarecorrupted', # not supported on some kernel
+        'meminfo_anonhugepages',
+        'meminfo_hugepages_total',
+        'meminfo_hugepages_free',
+        'meminfo_hugepages_rsvd',
+        'meminfo_hugepages_surp',
+        'meminfo_hugepagesize',
+        'meminfo_directmap4k',
+        'meminfo_directmap2m',
+        'vmstat_pgpgin',
+        'vmstat_pgpgout',
+        'vmstat_pswpin',
+        'vmstat_pswpout',
+        'vmstat_pgfault',
+        'vmstat_pgmajfault',
+        'stat_cpu_user',
+        'stat_cpu_nice',
+        'stat_cpu_system',
+        'stat_cpu_idle',
+        'stat_cpu_iowait',
+        'stat_cpu_irq',
+        'stat_cpu_softirq',
+        'stat_cpu_guest',
+        'stat_cpu_guest_nice',
+        'stat_intr',
+        'stat_ctxt',
+        'stat_processes',
+        'stat_procs_blocked',
+        'loadavg_1min',
+        'loadavg_5min',
+        'loadavg_15min',
+        'loadavg_runnable',
+        'loadavg_total_threads',
+        # 'sys_numa_zoneallocs',
+        # 'sys_numa_zoneallocs',
+        # 'sys_numa_foreign_allocs',
+        # 'sys_numa_allocation',
+        # 'sys_numa_allocation',
+        # 'sys_numa_interleave',
+        'involved_jobs'
+    ]
+
+    _counters = []
+    for c_name in counters_procstats_to_get:
+        (c_type, c_repr, c_acc, c_descr) = counters_procstats[c_name]
+        _counters.append((c_name, c_type, c_repr, c_acc, c_descr))
+
+    @classmethod
+    def get_zero_counters(cls):
+        return cls(procstats_buffer=None, raw=None)
+
+    @classmethod
+    def fetch(cls, procstats_backend):
+        return procstats_backend.get_procstats()
+
+    def __init__(self, procstats_buffer=None, raw=None):
+        BaseCounters.__init__(self, raw=raw)
+        if raw is not None:
+            pass
+        elif procstats_buffer is None:
+            self._empty_fill()
+        else:
+            for name in ProcstatsCounters._counter_definitions:
+                self._counter_values[name] = procstats_buffer[name]
\ No newline at end of file

From 37b376c77561ec8783a2bd0d2d14995d7d37c934 Mon Sep 17 00:00:00 2001
From: Ana Gelez <ana@gelez.xyz>
Date: Tue, 5 Mar 2024 14:50:48 +0100
Subject: [PATCH 03/11] Make colmet-collector start

---
 colmet.nix | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/colmet.nix b/colmet.nix
index 8c06cd0..89d11e0 100644
--- a/colmet.nix
+++ b/colmet.nix
@@ -3,13 +3,13 @@
 let
   colmet = pkgs.python3Packages.buildPythonApplication rec {
     name = "colmet-${version}";
-    version = "0.5.4";
+    version = "0.6.10.dev0";
 
     src = pkgs.fetchFromGitHub {
       owner = "oar-team";
       repo = "colmet";
-      rev = "4cc29227fcaf5236d97dde74b9a52e04250a5b77";
-      sha256 = "1g2m6crdmlgk8c57qa1nss20128dnw9x58yg4r5wdc7zliicahqq";
+      rev = "a1c1c32e9f6014ddf8253842df27579b46b9a2fa";
+      sha256 = "sha256-wXsJtvml6PdG1V5hE7wmpdEcAvof6U1NPc1rFR15NHU=";
     };
 
     buildInputs = [ pkgs.powercap ];
@@ -66,13 +66,13 @@ with lib;
     wantedBy = [ "multi-user.target" ];
     after = [ "network-online.target" (mkIf config.services.opensearch.enable "opensearch.service") ];
     serviceConfig = {
-      ExecStart = "${colmet}/bin/colmet-collector -vvv \
-        --zeromq-bind-uri tcp://192.168.0.1:5556 \
-        --buffer-size 5000 \
-        --sample-period 3 \
-        --elastic-host https://127.0.0.1:9200 \
-        --elastic-index-prefix colmet_dahu_ 2>>/var/log/colmet_err.log >> /var/log/colmet.log \
-        --no-check-certificates";
+      ExecStart = "${colmet}/bin/colmet-collector -vvv " +
+        "--zeromq-bind-uri tcp://0.0.0.0:5556 " +
+        "--buffer-size 5000 " +
+        "--sample-period 3 " +
+        "--elastic-host https://127.0.0.1:9200 " +
+        "--elastic-index-prefix colmet_dahu_ " +
+        "--no-check-certificates";
     };
   };
 

From a047a0d72a8ae39f4fa1e6501cafb8dbb8b56c1c Mon Sep 17 00:00:00 2001
From: Ana Gelez <ana@gelez.xyz>
Date: Tue, 5 Mar 2024 15:31:54 +0100
Subject: [PATCH 04/11] Patch colmet to make colmet-collector start

---
 hardwarecorrupted.patch | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/hardwarecorrupted.patch b/hardwarecorrupted.patch
index 31ec7f3..3bfa105 100644
--- a/hardwarecorrupted.patch
+++ b/hardwarecorrupted.patch
@@ -11,3 +11,30 @@
          'meminfo_anonhugepages': (UInt64(), 'kbytes', 'none',
                                    'meminfo_anonhugepages'),
          'meminfo_hugepages_total': (UInt64(), 'count', 'none',
+
+--- a/colmet/common/backends/zeromq.py
++++ b/colmet/common/backends/zeromq.py
+@@ -13,12 +13,8 @@
+
+ from colmet.common.backends.base import InputBaseBackend, OutputBaseBackend
+ 
+-try:
+-    _snd_hwm = zmq.HWM
+-    _rcv_hwm = zmq.HWM
+-except AttributeError:
+-    _snd_hwm = zmq.SNDHWM
+-    _rcv_hwm = zmq.RCVHWM
++_snd_hwm = zmq.SNDHWM
++_rcv_hwm = zmq.RCVHWM
+ 
+ 
+ class ZMQInputBackend(InputBaseBackend):
+@@ -69,7 +69,8 @@
+         self.context = zmq.Context()
+         self.hostname = socket.gethostname()
+         self.socket = self.context.socket(zmq.PUSH)
+         self.socket.setsockopt(zmq.LINGER, self.options.zeromq_linger)
++        print(f'{_snd_hwm} / {self.options.zeromq_hwm}')
+         self.socket.setsockopt(_snd_hwm, self.options.zeromq_hwm)
+         self.socket.connect(self.options.zeromq_uri)
+         LOG.debug("Use the URI '%s'" % self.options.zeromq_uri)

From 81729ae931238a605b368ebe42e1055a2739f358 Mon Sep 17 00:00:00 2001
From: Ana Gelez <ana@gelez.xyz>
Date: Tue, 5 Mar 2024 15:43:20 +0100
Subject: [PATCH 05/11] Authenticate colmet when connecting to OpenSearch

---
 colmet.nix | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/colmet.nix b/colmet.nix
index 89d11e0..ac272d9 100644
--- a/colmet.nix
+++ b/colmet.nix
@@ -45,6 +45,7 @@ let
   };
   cfg-collector = config.services.colmet-collector;
   cfg-node = config.services.colmet-node;
+  auth-file = pkgs.writeText "colmet-opensearch-auth" "admin:admin";
 in
 with lib;
 {
@@ -72,6 +73,7 @@ with lib;
         "--sample-period 3 " +
         "--elastic-host https://127.0.0.1:9200 " +
         "--elastic-index-prefix colmet_dahu_ " +
+        "--http-credentials ${auth-file} " +
         "--no-check-certificates";
     };
   };

From 20c8143e35181695b0fca89c64be5df6fffc084f Mon Sep 17 00:00:00 2001
From: Ana Gelez <ana@gelez.xyz>
Date: Wed, 6 Mar 2024 10:52:06 +0100
Subject: [PATCH 06/11] Make the colmet patch more minimal

---
 colmet-hwm-constants.patch | 17 ++++++++++++++++
 colmet.nix                 |  2 +-
 hardwarecorrupted.patch    | 40 --------------------------------------
 3 files changed, 18 insertions(+), 41 deletions(-)
 create mode 100644 colmet-hwm-constants.patch
 delete mode 100644 hardwarecorrupted.patch

diff --git a/colmet-hwm-constants.patch b/colmet-hwm-constants.patch
new file mode 100644
index 0000000..22a8725
--- /dev/null
+++ b/colmet-hwm-constants.patch
@@ -0,0 +1,17 @@
+--- a/colmet/common/backends/zeromq.py
++++ b/colmet/common/backends/zeromq.py
+@@ -13,12 +13,8 @@
+
+ from colmet.common.backends.base import InputBaseBackend, OutputBaseBackend
+ 
+-try:
+-    _snd_hwm = zmq.HWM
+-    _rcv_hwm = zmq.HWM
+-except AttributeError:
+-    _snd_hwm = zmq.SNDHWM
+-    _rcv_hwm = zmq.RCVHWM
++_snd_hwm = zmq.SNDHWM
++_rcv_hwm = zmq.RCVHWM
+ 
+ 
+ class ZMQInputBackend(InputBaseBackend):
diff --git a/colmet.nix b/colmet.nix
index ac272d9..6ec650f 100644
--- a/colmet.nix
+++ b/colmet.nix
@@ -32,7 +32,7 @@ let
     # Tests do not pass
     doCheck = false;
 
-    patches= [./hardwarecorrupted.patch];
+    patches= [ ./colmet-hwm-constants.patch ];
 
     meta = with lib; {
       description = "Collecting metrics about process running in cpuset and in a distributed environnement";
diff --git a/hardwarecorrupted.patch b/hardwarecorrupted.patch
deleted file mode 100644
index 3bfa105..0000000
--- a/hardwarecorrupted.patch
+++ /dev/null
@@ -1,40 +0,0 @@
---- a/colmet/common/metrics/procstats.py
-+++ b/colmet/common/metrics/procstats.py
-@@ -53,8 +53,8 @@
-                                 'meminfo_vmallocused'),
-         'meminfo_vmallocchunk': (UInt64(), 'kbytes', 'none',
-                                  'meminfo_vmallocchunk'),
--        'meminfo_hardwarecorrupted': (UInt64(), 'kbytes', 'none',
--                                      'meminfo_hardwarecorrupted'),
-+        #'meminfo_hardwarecorrupted': (UInt64(), 'kbytes', 'none',
-+        #                              'meminfo_hardwarecorrupted'),
-         'meminfo_anonhugepages': (UInt64(), 'kbytes', 'none',
-                                   'meminfo_anonhugepages'),
-         'meminfo_hugepages_total': (UInt64(), 'count', 'none',
-
---- a/colmet/common/backends/zeromq.py
-+++ b/colmet/common/backends/zeromq.py
-@@ -13,12 +13,8 @@
-
- from colmet.common.backends.base import InputBaseBackend, OutputBaseBackend
- 
--try:
--    _snd_hwm = zmq.HWM
--    _rcv_hwm = zmq.HWM
--except AttributeError:
--    _snd_hwm = zmq.SNDHWM
--    _rcv_hwm = zmq.RCVHWM
-+_snd_hwm = zmq.SNDHWM
-+_rcv_hwm = zmq.RCVHWM
- 
- 
- class ZMQInputBackend(InputBaseBackend):
-@@ -69,7 +69,8 @@
-         self.context = zmq.Context()
-         self.hostname = socket.gethostname()
-         self.socket = self.context.socket(zmq.PUSH)
-         self.socket.setsockopt(zmq.LINGER, self.options.zeromq_linger)
-+        print(f'{_snd_hwm} / {self.options.zeromq_hwm}')
-         self.socket.setsockopt(_snd_hwm, self.options.zeromq_hwm)
-         self.socket.connect(self.options.zeromq_uri)
-         LOG.debug("Use the URI '%s'" % self.options.zeromq_uri)

From c3cb9937ac49be3c7979000ca3604bcfb9207479 Mon Sep 17 00:00:00 2001
From: Ana Gelez <ana@gelez.xyz>
Date: Wed, 6 Mar 2024 14:21:43 +0100
Subject: [PATCH 07/11] colmet: add tests

---
 opensearch-mono/composition.nix | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/opensearch-mono/composition.nix b/opensearch-mono/composition.nix
index 17f7bdd..e623901 100644
--- a/opensearch-mono/composition.nix
+++ b/opensearch-mono/composition.nix
@@ -108,7 +108,6 @@ in
         services.opensearch-dashboards.enable = true;
         services.colmet-collector.enable = true;
         services.colmet-node.enable = true;
-
       };
   };
 
@@ -137,5 +136,13 @@ in
     opensearch.succeed(
       "curl --fail http://localhost:5601/"
     )
+
+    # Check that colmet runs…
+    opensearch.wait_for_unit("colmet-node.service")
+    opensearch.wait_for_unit("colmet-collector.service")
+    # That the collector has set up a ZeroMQ server…
+    opensearch.succeed("netstat -tlnp | grep :5556")
+    # And that the index was created in OpenSearch
+    opensearch.succeed("curl -ku admin:admin https://localhost:9200/_cat/indices | grep colmet")
   '';
 }

From 8dce89067536645ad2f45960c6e8a28c502879ed Mon Sep 17 00:00:00 2001
From: Elise <Elise.Dupont@etu.univ-grenoble-alpes.fr>
Date: Thu, 7 Mar 2024 10:14:36 +0100
Subject: [PATCH 08/11] colmet + vector OK + test OK

---
 opensearch-mono/composition.nix | 49 ++++++++++++++++++++++++++++++++-
 1 file changed, 48 insertions(+), 1 deletion(-)

diff --git a/opensearch-mono/composition.nix b/opensearch-mono/composition.nix
index e623901..4066b3f 100644
--- a/opensearch-mono/composition.nix
+++ b/opensearch-mono/composition.nix
@@ -16,7 +16,7 @@ in
         imports = [ ../opensearch-dashboards.nix ../colmet.nix ];
 
         environment.noXlibs = false;
-        environment.systemPackages = with pkgs; [ opensearch-fixed jq ];
+        environment.systemPackages = with pkgs; [ opensearch-fixed vector jq ];
 
         systemd.services.opensearch.serviceConfig.ExecStartPre = [
           "${pkgs.writeShellScript
@@ -105,9 +105,56 @@ in
           ];
         };
 
+
+        # Vector est système de gestion de logs
+        services.vector = {
+          enable = true;
+          journaldAccess = true;
+          settings = {
+            sources = {
+              "in" = {
+                type = "stdin";
+              };
+              "systemd" = {
+                type = "journald";
+              };
+            };
+            sinks = {
+              out = {
+                inputs = [ "in" ];
+                type = "console";
+                encoding = {
+                  codec = "text";
+                };
+              };
+              opensearch = {
+                inputs = [ "systemd" ];
+                type = "elasticsearch";
+                endpoints = ["https://127.0.0.1:9200"];
+                auth = {
+                  strategy = "basic";
+                  user = "admin";
+                  password = "admin";
+                };
+                tls.verify_certificate = false;
+              };
+            };
+          };
+        };
         services.opensearch-dashboards.enable = true;
         services.colmet-collector.enable = true;
         services.colmet-node.enable = true;
+
+        environment.variables = {
+          # La variable "VECTOR_CONFIG" défini le chemin de la configuration à utiliser quand on
+          # lance la commande `vector`. Le service Systemd génère une config à partir de `services.vector.settings`
+          # et s'assure que le service utilise bien ce fichier. Mais il faut aussi indiquer où ce trouve
+          # ce fichier de configuration à l'outil en ligne de commande disponible dans le PATH.
+          # On parse la configuration systemd pour récupérer le chemin du fichier.
+          VECTOR_CONFIG = lib.lists.last (
+            builtins.split " " config.systemd.services.vector.serviceConfig.ExecStart
+          );
+        };
       };
   };
 

From bf6c74f98f486dc4a8fcbadfdb0e9855f127e3eb Mon Sep 17 00:00:00 2001
From: Elise <Elise.Dupont@etu.univ-grenoble-alpes.fr>
Date: Thu, 7 Mar 2024 11:02:41 +0100
Subject: [PATCH 09/11] Test OK

---
 opensearch-mono/composition.nix | 31 ++++++++++++++++++-------------
 1 file changed, 18 insertions(+), 13 deletions(-)

diff --git a/opensearch-mono/composition.nix b/opensearch-mono/composition.nix
index 4066b3f..02325c0 100644
--- a/opensearch-mono/composition.nix
+++ b/opensearch-mono/composition.nix
@@ -1,4 +1,4 @@
-{ pkgs, ... }:
+{ pkgs, enable-colmet ? true, enable-vector ? true, ... }:
 let
   keystore-password = "usAe#%EX92R7UHSYwJ";
   truststore-password = "*!YWptTiu3&okU%E9a";
@@ -16,7 +16,7 @@ in
         imports = [ ../opensearch-dashboards.nix ../colmet.nix ];
 
         environment.noXlibs = false;
-        environment.systemPackages = with pkgs; [ opensearch-fixed vector jq ];
+        environment.systemPackages = with pkgs; [ opensearch-fixed jq ] ++ (if enable-vector then [ vector ] else []);
 
         systemd.services.opensearch.serviceConfig.ExecStartPre = [
           "${pkgs.writeShellScript
@@ -108,7 +108,7 @@ in
 
         # Vector est système de gestion de logs
         services.vector = {
-          enable = true;
+          enable = enable-vector;
           journaldAccess = true;
           settings = {
             sources = {
@@ -142,10 +142,10 @@ in
           };
         };
         services.opensearch-dashboards.enable = true;
-        services.colmet-collector.enable = true;
-        services.colmet-node.enable = true;
+        services.colmet-collector.enable = enable-colmet;
+        services.colmet-node.enable = enable-colmet;
 
-        environment.variables = {
+        environment.variables = lib.mkIf enable-vector {
           # La variable "VECTOR_CONFIG" défini le chemin de la configuration à utiliser quand on
           # lance la commande `vector`. Le service Systemd génère une config à partir de `services.vector.settings`
           # et s'assure que le service utilise bien ce fichier. Mais il faut aussi indiquer où ce trouve
@@ -161,6 +161,7 @@ in
   dockerPorts.opensearch = [ "5601:5601" "9200:9200" ];
 
   testScript = ''
+    import time
     opensearch.start()
     opensearch.wait_for_unit("opensearch.service")
     opensearch.wait_for_open_port(9200)
@@ -169,6 +170,15 @@ in
       "curl -k -u admin:admin --fail https://localhost:9200"
     )
 
+    opensearch.wait_for_unit("opensearch-dashboards.service")
+    opensearch.wait_for_open_port(5601)
+    # When starting, opensearch-dashboards binds the port but need some time to start
+    time.sleep(10)
+    opensearch.succeed(
+      "curl --fail http://localhost:5601/"
+    )
+  '' + (if enable-vector then ''
+
     opensearch.wait_for_unit("vector.service")
 
     # The inner curl command uses the Opensearch API and JQ to get the name of the Vector index
@@ -178,12 +188,7 @@ in
     opensearch.succeed(
       "curl -k -u admin:admin --fail https://localhost:9200/$(curl -k -u admin:admin --fail https://localhost:9200/_stats | jq -r '.indices | keys[]' | grep vector | tail -n 1)/_search | jq '.hits.hits[0]._source'"
     )
-
-    opensearch.wait_for_unit("opensearch-dashboards.service")
-    opensearch.succeed(
-      "curl --fail http://localhost:5601/"
-    )
-
+  '' else "") + (if enable-colmet then ''
     # Check that colmet runs…
     opensearch.wait_for_unit("colmet-node.service")
     opensearch.wait_for_unit("colmet-collector.service")
@@ -191,5 +196,5 @@ in
     opensearch.succeed("netstat -tlnp | grep :5556")
     # And that the index was created in OpenSearch
     opensearch.succeed("curl -ku admin:admin https://localhost:9200/_cat/indices | grep colmet")
-  '';
+  '' else "");
 }

From 27a7addf2ad94235f186791b3e739f80ba2fc61b Mon Sep 17 00:00:00 2001
From: Ana Gelez <ana@gelez.xyz>
Date: Thu, 7 Mar 2024 11:38:51 +0100
Subject: [PATCH 10/11] Use colmet from nur-kapack

---
 colmet-hwm-constants.patch |  17 ------
 colmet.nix                 |  48 +---------------
 opensearch-mono/flake.lock | 109 +++++++++++++++++++++++++++++++++----
 opensearch-mono/flake.nix  |   6 +-
 4 files changed, 105 insertions(+), 75 deletions(-)
 delete mode 100644 colmet-hwm-constants.patch

diff --git a/colmet-hwm-constants.patch b/colmet-hwm-constants.patch
deleted file mode 100644
index 22a8725..0000000
--- a/colmet-hwm-constants.patch
+++ /dev/null
@@ -1,17 +0,0 @@
---- a/colmet/common/backends/zeromq.py
-+++ b/colmet/common/backends/zeromq.py
-@@ -13,12 +13,8 @@
-
- from colmet.common.backends.base import InputBaseBackend, OutputBaseBackend
- 
--try:
--    _snd_hwm = zmq.HWM
--    _rcv_hwm = zmq.HWM
--except AttributeError:
--    _snd_hwm = zmq.SNDHWM
--    _rcv_hwm = zmq.RCVHWM
-+_snd_hwm = zmq.SNDHWM
-+_rcv_hwm = zmq.RCVHWM
- 
- 
- class ZMQInputBackend(InputBaseBackend):
diff --git a/colmet.nix b/colmet.nix
index 6ec650f..c0cb904 100644
--- a/colmet.nix
+++ b/colmet.nix
@@ -1,55 +1,13 @@
 { lib, pkgs, config,  ... }:
 
 let
-  colmet = pkgs.python3Packages.buildPythonApplication rec {
-    name = "colmet-${version}";
-    version = "0.6.10.dev0";
-
-    src = pkgs.fetchFromGitHub {
-      owner = "oar-team";
-      repo = "colmet";
-      rev = "a1c1c32e9f6014ddf8253842df27579b46b9a2fa";
-      sha256 = "sha256-wXsJtvml6PdG1V5hE7wmpdEcAvof6U1NPc1rFR15NHU=";
-    };
-
-    buildInputs = [ pkgs.powercap ];
-
-    propagatedBuildInputs = with pkgs.python3Packages; [
-      pyinotify
-      pyzmq
-      tables
-      requests
-    ];
-
-    preBuild = ''
-      mkdir -p $out/lib
-      sed -i "s#/usr/lib/#$out/lib/#g" colmet/node/backends/perfhwstats.py
-      sed -i "s#/usr/lib/#$out/lib/#g" colmet/node/backends/RAPLstats.py
-      sed -i "s#/usr/lib/#$out/lib/#g" colmet/node/backends/lib_perf_hw/makefile
-      sed -i "s#/usr/lib/#$out/lib/#g" colmet/node/backends/lib_rapl/makefile
-    '';
-
-    # Tests do not pass
-    doCheck = false;
-
-    patches= [ ./colmet-hwm-constants.patch ];
-
-    meta = with lib; {
-      description = "Collecting metrics about process running in cpuset and in a distributed environnement";
-      homepage = https://github.com/oar-team/colmet;
-      platforms = pkgs.powercap.meta.platforms;
-      licence = licenses.gpl2;
-      longDescription = ''
-    '';
-    };
-  };
   cfg-collector = config.services.colmet-collector;
   cfg-node = config.services.colmet-node;
   auth-file = pkgs.writeText "colmet-opensearch-auth" "admin:admin";
 in
 with lib;
 {
-  config.environment.systemPackages = [ colmet ];
+  config.environment.systemPackages = with pkgs; [ nur.repos.kapack.colmet ];
 
   #option to enable the colmet collector
   options.services.colmet-collector = {
@@ -67,7 +25,7 @@ with lib;
     wantedBy = [ "multi-user.target" ];
     after = [ "network-online.target" (mkIf config.services.opensearch.enable "opensearch.service") ];
     serviceConfig = {
-      ExecStart = "${colmet}/bin/colmet-collector -vvv " +
+      ExecStart = "${pkgs.nur.repos.kapack.colmet}/bin/colmet-collector -vvv " +
         "--zeromq-bind-uri tcp://0.0.0.0:5556 " +
         "--buffer-size 5000 " +
         "--sample-period 3 " +
@@ -84,7 +42,7 @@ with lib;
     wantedBy = [ "multi-user.target" ];
     after = [ "network-online.target" (mkIf config.services.opensearch.enable "opensearch.service") ];
     serviceConfig = {
-      ExecStart = "${colmet}/bin/colmet-node -vvv --zeromq-uri tcp://127.0.0.1:5556";
+      ExecStart = "${pkgs.nur.repos.kapack.colmet}/bin/colmet-node -vvv --zeromq-uri tcp://127.0.0.1:5556";
     };
   };
 }
diff --git a/opensearch-mono/flake.lock b/opensearch-mono/flake.lock
index 3cfd504..299d8a6 100644
--- a/opensearch-mono/flake.lock
+++ b/opensearch-mono/flake.lock
@@ -1,6 +1,24 @@
 {
   "nodes": {
     "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1689068808,
+        "narHash": "sha256-6ixXo3wt24N/melDWjq70UuHQLxGV8jZvooRanIHXw0=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "919d646de7be200f3bf08cb76ae1f09402b6f9b4",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_2": {
       "locked": {
         "lastModified": 1656928814,
         "narHash": "sha256-RIFfgBuKz6Hp89yRr7+NR5tzIAbn52h8vT6vXkYjZoM=",
@@ -15,7 +33,7 @@
         "type": "github"
       }
     },
-    "flake-utils_2": {
+    "flake-utils_3": {
       "locked": {
         "lastModified": 1653893745,
         "narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=",
@@ -32,7 +50,26 @@
     },
     "kapack": {
       "inputs": {
-        "flake-utils": "flake-utils_2",
+        "flake-utils": "flake-utils",
+        "nixpkgs": "nixpkgs"
+      },
+      "locked": {
+        "lastModified": 1709732514,
+        "narHash": "sha256-beQGEh9mgRL4T7tdkSlT/MrhncmfICWt3LmfuydGL1o=",
+        "owner": "oar-team",
+        "repo": "nur-kapack",
+        "rev": "2c3f5d9f5078229de10394eead8f8ed28d17b6a9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "oar-team",
+        "repo": "nur-kapack",
+        "type": "github"
+      }
+    },
+    "kapack_2": {
+      "inputs": {
+        "flake-utils": "flake-utils_3",
         "nixpkgs": [
           "nxc",
           "nixpkgs"
@@ -55,16 +92,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1708294118,
-        "narHash": "sha256-evZzmLW7qoHXf76VCepvun1esZDxHfVRFUJtumD7L2M=",
-        "owner": "NixOS",
+        "lastModified": 1701282334,
+        "narHash": "sha256-MxCVrXY6v4QmfTwIysjjaX0XUhqBbxTWWB4HXtDYsdk=",
+        "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "e0da498ad77ac8909a980f07eff060862417ccf7",
+        "rev": "057f9aecfb71c4437d2b27d3323df7f93c010b7e",
         "type": "github"
       },
       "original": {
-        "owner": "NixOS",
-        "ref": "nixos-23.11",
+        "owner": "nixos",
+        "ref": "23.11",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -86,6 +123,22 @@
       }
     },
     "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1709677081,
+        "narHash": "sha256-tix36Y7u0rkn6mTm0lA45b45oab2cFLqAzDbJxeXS+c=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "880992dcc006a5e00dd0591446fdf723e6a51a64",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-23.11",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_3": {
       "locked": {
         "lastModified": 1685566663,
         "narHash": "sha256-btHN1czJ6rzteeCuE/PNrdssqYD2nIA4w48miQAFloM=",
@@ -101,11 +154,26 @@
         "type": "github"
       }
     },
+    "nur": {
+      "locked": {
+        "lastModified": 1709800027,
+        "narHash": "sha256-iyCUV3wbl+3kYpi5mJZQJRjFo/8YuuVR0UF3Ql5wLFw=",
+        "owner": "nix-community",
+        "repo": "nur",
+        "rev": "895e0855655804508fbeb90ed38bc82b6139c9bc",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "nur",
+        "type": "github"
+      }
+    },
     "nxc": {
       "inputs": {
-        "flake-utils": "flake-utils",
-        "kapack": "kapack",
-        "nixpkgs": "nixpkgs_2"
+        "flake-utils": "flake-utils_2",
+        "kapack": "kapack_2",
+        "nixpkgs": "nixpkgs_3"
       },
       "locked": {
         "lastModified": 1707234688,
@@ -124,9 +192,26 @@
     },
     "root": {
       "inputs": {
-        "nixpkgs": "nixpkgs",
+        "kapack": "kapack",
+        "nixpkgs": "nixpkgs_2",
+        "nur": "nur",
         "nxc": "nxc"
       }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
     }
   },
   "root": "root",
diff --git a/opensearch-mono/flake.nix b/opensearch-mono/flake.nix
index f6e8939..7c36b62 100644
--- a/opensearch-mono/flake.nix
+++ b/opensearch-mono/flake.nix
@@ -4,15 +4,19 @@
   inputs = {
     nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11";
     nxc.url = "github:elegaanz/nixos-compose/fix-vde-switch-groups";
+    kapack.url = "github:oar-team/nur-kapack";
+    nur.url = "github:nix-community/nur";
   };
 
-  outputs = { self, nixpkgs, nxc }:
+  outputs = { self, nixpkgs, nxc, kapack, nur }:
     let
       system = "x86_64-linux";
     in
     {
       packages.${system} = nxc.lib.compose {
         inherit nixpkgs system;
+        NUR = nur;
+        repoOverrides = { inherit kapack; };
         composition = ./composition.nix;
       };
 

From 60b156b5dc72184525e00856e56736a8a2af3dc6 Mon Sep 17 00:00:00 2001
From: Ana Gelez <ana@gelez.xyz>
Date: Thu, 7 Mar 2024 11:42:22 +0100
Subject: [PATCH 11/11] Remove files that were used to generate a patch

---
 opensearch-mono/colmet_hardwarecorrupted.py | 197 --------------------
 opensearch-mono/colmet_original.py          | 197 --------------------
 2 files changed, 394 deletions(-)
 delete mode 100644 opensearch-mono/colmet_hardwarecorrupted.py
 delete mode 100644 opensearch-mono/colmet_original.py

diff --git a/opensearch-mono/colmet_hardwarecorrupted.py b/opensearch-mono/colmet_hardwarecorrupted.py
deleted file mode 100644
index 9bb26a2..0000000
--- a/opensearch-mono/colmet_hardwarecorrupted.py
+++ /dev/null
@@ -1,197 +0,0 @@
-import logging
-
-LOG = logging.getLogger()
-
-from .base import UInt64, UFloat, BaseCounters, String
-
-class ProcstatsCounters(BaseCounters):
-    __metric_name__ = 'procstats_default'
-
-    counters_procstats = {
-        # 'key': ( offset,length, type, repr, acc )
-        'uptime_total': (UInt64(), 'sec', 'none', 'uptime.total'),
-        'uptime_idle': (UInt64(), 'sec', 'none', 'uptime.idle'),
-        'meminfo_memtotal': (UInt64(), 'kbytes', 'none', 'meminfo_memtotal'),
-        'meminfo_memfree': (UInt64(), 'kbytes', 'none', 'meminfo_memfree'),
-        'meminfo_buffers': (UInt64(), 'kbytes', 'none', 'meminfo_buffers'),
-        'meminfo_cached': (UInt64(), 'kbytes', 'none', 'meminfo_cached'),
-        'meminfo_swapcached': (UInt64(), 'kbytes', 'none',
-                               'meminfo_swapcached'),
-        'meminfo_active': (UInt64(), 'kbytes', 'none', 'meminfo_active'),
-        'meminfo_inactive': (UInt64(), 'kbytes', 'none', 'meminfo_inactive'),
-        'meminfo_unevictable': (UInt64(), 'kbytes', 'none',
-                                'meminfo_unevictable'),
-        'meminfo_mlocked': (UInt64(), 'kbytes', 'none', 'meminfo_mlocked'),
-        'meminfo_swaptotal': (UInt64(), 'kbytes', 'none', 'meminfo_swaptotal'),
-        'meminfo_swapfree': (UInt64(), 'kbytes', 'none', 'meminfo_swapfree'),
-        'meminfo_dirty': (UInt64(), 'kbytes', 'none', 'meminfo_dirty'),
-        'meminfo_writeback': (UInt64(), 'kbytes', 'none', 'meminfo_writeback'),
-        'meminfo_anonpages': (UInt64(), 'kbytes', 'none', 'meminfo_anonpages'),
-        'meminfo_mapped': (UInt64(), 'kbytes', 'none', 'meminfo_mapped'),
-        'meminfo_shmem': (UInt64(), 'kbytes', 'none', 'meminfo_shmem'),
-        'meminfo_slab': (UInt64(), 'kbytes', 'none', 'meminfo_slab'),
-        'meminfo_sreclaimable': (UInt64(), 'kbytes', 'none',
-                                 'meminfo_sreclaimable'),
-        'meminfo_sunreclaim': (UInt64(), 'kbytes', 'none',
-                               'meminfo_sunreclaim'),
-        'meminfo_kernelstack': (UInt64(), 'kbytes', 'none',
-                                'meminfo_kernelstack'),
-        'meminfo_pagetables': (UInt64(), 'kbytes', 'none',
-                               'meminfo_pagetables'),
-        'meminfo_nfs_unstable': (UInt64(), 'kbytes', 'none',
-                                 'meminfo_nfs_unstable'),
-        'meminfo_bounce': (UInt64(), 'kbytes', 'none', 'meminfo_bounce'),
-        'meminfo_writebacktmp': (UInt64(), 'kbytes', 'none',
-                                 'meminfo_writebacktmp'),
-        'meminfo_commitlimit': (UInt64(), 'kbytes', 'none',
-                                'meminfo_commitlimit'),
-        'meminfo_committed_as': (UInt64(), 'kbytes', 'none',
-                                 'meminfo_committed_as'),
-        'meminfo_vmalloctotal': (UInt64(), 'kbytes', 'none',
-                                 'meminfo_vmalloctotal'),
-        'meminfo_vmallocused': (UInt64(), 'kbytes', 'none',
-                                'meminfo_vmallocused'),
-        'meminfo_vmallocchunk': (UInt64(), 'kbytes', 'none',
-                                 'meminfo_vmallocchunk'),
-        #'meminfo_hardwarecorrupted': (UInt64(), 'kbytes', 'none',
-        #                              'meminfo_hardwarecorrupted'),
-        'meminfo_anonhugepages': (UInt64(), 'kbytes', 'none',
-                                  'meminfo_anonhugepages'),
-        'meminfo_hugepages_total': (UInt64(), 'count', 'none',
-                                    'meminfo_hugepages_total'),
-        'meminfo_hugepages_free': (UInt64(), 'count', 'none',
-                                   'meminfo_hugepages_free'),
-        'meminfo_hugepages_rsvd': (UInt64(), 'count', 'none',
-                                   'meminfo_hugepages_rsvd'),
-        'meminfo_hugepages_surp': (UInt64(), 'count', 'none',
-                                   'meminfo_hugepages_surp'),
-        'meminfo_hugepagesize': (UInt64(), 'count', 'none',
-                                 'meminfo_hugepagesize'),
-        'meminfo_directmap4k': (UInt64(), 'kbytes', 'none',
-                                'meminfo_directmap4k'),
-        'meminfo_directmap2m': (UInt64(), 'kbytes', 'none',
-                                'meminfo_directmap2m'),
-        'vmstat_pgpgin': (UInt64(), 'count', 'none', 'vmstat_pgpgin'),
-        'vmstat_pgpgout': (UInt64(), 'count', 'none', 'vmstat_pgpgout'),
-        'vmstat_pswpin': (UInt64(), 'count', 'none', 'vmstat_pswpin'),
-        'vmstat_pswpout': (UInt64(), 'count', 'none', 'vmstat_pswpout'),
-        'vmstat_pgfault': (UInt64(), 'count', 'none', 'vmstat_pgfault'),
-        'vmstat_pgmajfault': (UInt64(), 'count', 'none', 'vmstat_pgmajfault'),
-        'stat_cpu_user': (UInt64(), 'n/a', 'none', 'stat_cpu_user'),
-        'stat_cpu_nice': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
-        'stat_cpu_system': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
-        'stat_cpu_idle': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
-        'stat_cpu_iowait': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
-        'stat_cpu_irq': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
-        'stat_cpu_softirq': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
-        'stat_cpu_guest': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
-        'stat_cpu_guest_nice': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
-        'stat_intr': (UInt64(), 'n/a', 'none', 'stat_intr'),
-        'stat_ctxt': (UInt64(), 'n/a', 'none', 'stat_ctxt'),
-        'stat_processes': (UInt64(), 'n/a', 'none', 'stat_processes'),
-        'stat_procs_blocked': (UInt64(), 'n/a', 'none', 'stat_procs_blocked'),
-        'loadavg_1min': (UFloat(), 'n/a', 'none', 'loadavg_1min'),
-        'loadavg_5min': (UFloat(), 'n/a', 'none', 'loadavg_5min'),
-        'loadavg_15min': (UFloat(), 'n/a', 'none', 'loadavg_15min'),
-        'loadavg_runnable': (UFloat(), 'n/a', 'none', 'loadavg_runnable'),
-        'loadavg_total_threads': (UFloat(), 'n/a', 'none',
-                                  'loadavg_total_threads'),
-        'involved_jobs': (String(8192), 'string', 'none', 'involved_jobs')
-    }
-
-    counters_procstats_to_get = [
-        'uptime_total',
-        'uptime_idle',
-        'meminfo_memtotal',
-        'meminfo_memfree',
-        'meminfo_buffers',
-        'meminfo_cached',
-        'meminfo_swapcached',
-        'meminfo_active',
-        'meminfo_inactive',
-        'meminfo_unevictable',
-        'meminfo_mlocked',
-        'meminfo_swaptotal',
-        'meminfo_swapfree',
-        'meminfo_dirty',
-        'meminfo_writeback',
-        'meminfo_anonpages',
-        'meminfo_mapped',
-        'meminfo_shmem',
-        'meminfo_slab',
-        'meminfo_sreclaimable',
-        'meminfo_sunreclaim',
-        'meminfo_kernelstack',
-        'meminfo_pagetables',
-        'meminfo_nfs_unstable',
-        'meminfo_bounce',
-        'meminfo_writebacktmp',
-        'meminfo_commitlimit',
-        'meminfo_committed_as',
-        'meminfo_vmalloctotal',
-        'meminfo_vmallocused',
-        'meminfo_vmallocchunk',
-        #'meminfo_hardwarecorrupted', # not supported on some kernel
-        'meminfo_anonhugepages',
-        'meminfo_hugepages_total',
-        'meminfo_hugepages_free',
-        'meminfo_hugepages_rsvd',
-        'meminfo_hugepages_surp',
-        'meminfo_hugepagesize',
-        'meminfo_directmap4k',
-        'meminfo_directmap2m',
-        'vmstat_pgpgin',
-        'vmstat_pgpgout',
-        'vmstat_pswpin',
-        'vmstat_pswpout',
-        'vmstat_pgfault',
-        'vmstat_pgmajfault',
-        'stat_cpu_user',
-        'stat_cpu_nice',
-        'stat_cpu_system',
-        'stat_cpu_idle',
-        'stat_cpu_iowait',
-        'stat_cpu_irq',
-        'stat_cpu_softirq',
-        'stat_cpu_guest',
-        'stat_cpu_guest_nice',
-        'stat_intr',
-        'stat_ctxt',
-        'stat_processes',
-        'stat_procs_blocked',
-        'loadavg_1min',
-        'loadavg_5min',
-        'loadavg_15min',
-        'loadavg_runnable',
-        'loadavg_total_threads',
-        # 'sys_numa_zoneallocs',
-        # 'sys_numa_zoneallocs',
-        # 'sys_numa_foreign_allocs',
-        # 'sys_numa_allocation',
-        # 'sys_numa_allocation',
-        # 'sys_numa_interleave',
-        'involved_jobs'
-    ]
-
-    _counters = []
-    for c_name in counters_procstats_to_get:
-        (c_type, c_repr, c_acc, c_descr) = counters_procstats[c_name]
-        _counters.append((c_name, c_type, c_repr, c_acc, c_descr))
-
-    @classmethod
-    def get_zero_counters(cls):
-        return cls(procstats_buffer=None, raw=None)
-
-    @classmethod
-    def fetch(cls, procstats_backend):
-        return procstats_backend.get_procstats()
-
-    def __init__(self, procstats_buffer=None, raw=None):
-        BaseCounters.__init__(self, raw=raw)
-        if raw is not None:
-            pass
-        elif procstats_buffer is None:
-            self._empty_fill()
-        else:
-            for name in ProcstatsCounters._counter_definitions:
-                self._counter_values[name] = procstats_buffer[name]
\ No newline at end of file
diff --git a/opensearch-mono/colmet_original.py b/opensearch-mono/colmet_original.py
deleted file mode 100644
index 6d22572..0000000
--- a/opensearch-mono/colmet_original.py
+++ /dev/null
@@ -1,197 +0,0 @@
-import logging
-
-LOG = logging.getLogger()
-
-from .base import UInt64, UFloat, BaseCounters, String
-
-class ProcstatsCounters(BaseCounters):
-    __metric_name__ = 'procstats_default'
-
-    counters_procstats = {
-        # 'key': ( offset,length, type, repr, acc )
-        'uptime_total': (UInt64(), 'sec', 'none', 'uptime.total'),
-        'uptime_idle': (UInt64(), 'sec', 'none', 'uptime.idle'),
-        'meminfo_memtotal': (UInt64(), 'kbytes', 'none', 'meminfo_memtotal'),
-        'meminfo_memfree': (UInt64(), 'kbytes', 'none', 'meminfo_memfree'),
-        'meminfo_buffers': (UInt64(), 'kbytes', 'none', 'meminfo_buffers'),
-        'meminfo_cached': (UInt64(), 'kbytes', 'none', 'meminfo_cached'),
-        'meminfo_swapcached': (UInt64(), 'kbytes', 'none',
-                               'meminfo_swapcached'),
-        'meminfo_active': (UInt64(), 'kbytes', 'none', 'meminfo_active'),
-        'meminfo_inactive': (UInt64(), 'kbytes', 'none', 'meminfo_inactive'),
-        'meminfo_unevictable': (UInt64(), 'kbytes', 'none',
-                                'meminfo_unevictable'),
-        'meminfo_mlocked': (UInt64(), 'kbytes', 'none', 'meminfo_mlocked'),
-        'meminfo_swaptotal': (UInt64(), 'kbytes', 'none', 'meminfo_swaptotal'),
-        'meminfo_swapfree': (UInt64(), 'kbytes', 'none', 'meminfo_swapfree'),
-        'meminfo_dirty': (UInt64(), 'kbytes', 'none', 'meminfo_dirty'),
-        'meminfo_writeback': (UInt64(), 'kbytes', 'none', 'meminfo_writeback'),
-        'meminfo_anonpages': (UInt64(), 'kbytes', 'none', 'meminfo_anonpages'),
-        'meminfo_mapped': (UInt64(), 'kbytes', 'none', 'meminfo_mapped'),
-        'meminfo_shmem': (UInt64(), 'kbytes', 'none', 'meminfo_shmem'),
-        'meminfo_slab': (UInt64(), 'kbytes', 'none', 'meminfo_slab'),
-        'meminfo_sreclaimable': (UInt64(), 'kbytes', 'none',
-                                 'meminfo_sreclaimable'),
-        'meminfo_sunreclaim': (UInt64(), 'kbytes', 'none',
-                               'meminfo_sunreclaim'),
-        'meminfo_kernelstack': (UInt64(), 'kbytes', 'none',
-                                'meminfo_kernelstack'),
-        'meminfo_pagetables': (UInt64(), 'kbytes', 'none',
-                               'meminfo_pagetables'),
-        'meminfo_nfs_unstable': (UInt64(), 'kbytes', 'none',
-                                 'meminfo_nfs_unstable'),
-        'meminfo_bounce': (UInt64(), 'kbytes', 'none', 'meminfo_bounce'),
-        'meminfo_writebacktmp': (UInt64(), 'kbytes', 'none',
-                                 'meminfo_writebacktmp'),
-        'meminfo_commitlimit': (UInt64(), 'kbytes', 'none',
-                                'meminfo_commitlimit'),
-        'meminfo_committed_as': (UInt64(), 'kbytes', 'none',
-                                 'meminfo_committed_as'),
-        'meminfo_vmalloctotal': (UInt64(), 'kbytes', 'none',
-                                 'meminfo_vmalloctotal'),
-        'meminfo_vmallocused': (UInt64(), 'kbytes', 'none',
-                                'meminfo_vmallocused'),
-        'meminfo_vmallocchunk': (UInt64(), 'kbytes', 'none',
-                                 'meminfo_vmallocchunk'),
-        'meminfo_hardwarecorrupted': (UInt64(), 'kbytes', 'none',
-                                      'meminfo_hardwarecorrupted'),
-        'meminfo_anonhugepages': (UInt64(), 'kbytes', 'none',
-                                  'meminfo_anonhugepages'),
-        'meminfo_hugepages_total': (UInt64(), 'count', 'none',
-                                    'meminfo_hugepages_total'),
-        'meminfo_hugepages_free': (UInt64(), 'count', 'none',
-                                   'meminfo_hugepages_free'),
-        'meminfo_hugepages_rsvd': (UInt64(), 'count', 'none',
-                                   'meminfo_hugepages_rsvd'),
-        'meminfo_hugepages_surp': (UInt64(), 'count', 'none',
-                                   'meminfo_hugepages_surp'),
-        'meminfo_hugepagesize': (UInt64(), 'count', 'none',
-                                 'meminfo_hugepagesize'),
-        'meminfo_directmap4k': (UInt64(), 'kbytes', 'none',
-                                'meminfo_directmap4k'),
-        'meminfo_directmap2m': (UInt64(), 'kbytes', 'none',
-                                'meminfo_directmap2m'),
-        'vmstat_pgpgin': (UInt64(), 'count', 'none', 'vmstat_pgpgin'),
-        'vmstat_pgpgout': (UInt64(), 'count', 'none', 'vmstat_pgpgout'),
-        'vmstat_pswpin': (UInt64(), 'count', 'none', 'vmstat_pswpin'),
-        'vmstat_pswpout': (UInt64(), 'count', 'none', 'vmstat_pswpout'),
-        'vmstat_pgfault': (UInt64(), 'count', 'none', 'vmstat_pgfault'),
-        'vmstat_pgmajfault': (UInt64(), 'count', 'none', 'vmstat_pgmajfault'),
-        'stat_cpu_user': (UInt64(), 'n/a', 'none', 'stat_cpu_user'),
-        'stat_cpu_nice': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
-        'stat_cpu_system': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
-        'stat_cpu_idle': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
-        'stat_cpu_iowait': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
-        'stat_cpu_irq': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
-        'stat_cpu_softirq': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
-        'stat_cpu_guest': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
-        'stat_cpu_guest_nice': (UInt64(), 'n/a', 'none', 'stat_cpu_nice'),
-        'stat_intr': (UInt64(), 'n/a', 'none', 'stat_intr'),
-        'stat_ctxt': (UInt64(), 'n/a', 'none', 'stat_ctxt'),
-        'stat_processes': (UInt64(), 'n/a', 'none', 'stat_processes'),
-        'stat_procs_blocked': (UInt64(), 'n/a', 'none', 'stat_procs_blocked'),
-        'loadavg_1min': (UFloat(), 'n/a', 'none', 'loadavg_1min'),
-        'loadavg_5min': (UFloat(), 'n/a', 'none', 'loadavg_5min'),
-        'loadavg_15min': (UFloat(), 'n/a', 'none', 'loadavg_15min'),
-        'loadavg_runnable': (UFloat(), 'n/a', 'none', 'loadavg_runnable'),
-        'loadavg_total_threads': (UFloat(), 'n/a', 'none',
-                                  'loadavg_total_threads'),
-        'involved_jobs': (String(8192), 'string', 'none', 'involved_jobs')
-    }
-
-    counters_procstats_to_get = [
-        'uptime_total',
-        'uptime_idle',
-        'meminfo_memtotal',
-        'meminfo_memfree',
-        'meminfo_buffers',
-        'meminfo_cached',
-        'meminfo_swapcached',
-        'meminfo_active',
-        'meminfo_inactive',
-        'meminfo_unevictable',
-        'meminfo_mlocked',
-        'meminfo_swaptotal',
-        'meminfo_swapfree',
-        'meminfo_dirty',
-        'meminfo_writeback',
-        'meminfo_anonpages',
-        'meminfo_mapped',
-        'meminfo_shmem',
-        'meminfo_slab',
-        'meminfo_sreclaimable',
-        'meminfo_sunreclaim',
-        'meminfo_kernelstack',
-        'meminfo_pagetables',
-        'meminfo_nfs_unstable',
-        'meminfo_bounce',
-        'meminfo_writebacktmp',
-        'meminfo_commitlimit',
-        'meminfo_committed_as',
-        'meminfo_vmalloctotal',
-        'meminfo_vmallocused',
-        'meminfo_vmallocchunk',
-        #'meminfo_hardwarecorrupted', # not supported on some kernel
-        'meminfo_anonhugepages',
-        'meminfo_hugepages_total',
-        'meminfo_hugepages_free',
-        'meminfo_hugepages_rsvd',
-        'meminfo_hugepages_surp',
-        'meminfo_hugepagesize',
-        'meminfo_directmap4k',
-        'meminfo_directmap2m',
-        'vmstat_pgpgin',
-        'vmstat_pgpgout',
-        'vmstat_pswpin',
-        'vmstat_pswpout',
-        'vmstat_pgfault',
-        'vmstat_pgmajfault',
-        'stat_cpu_user',
-        'stat_cpu_nice',
-        'stat_cpu_system',
-        'stat_cpu_idle',
-        'stat_cpu_iowait',
-        'stat_cpu_irq',
-        'stat_cpu_softirq',
-        'stat_cpu_guest',
-        'stat_cpu_guest_nice',
-        'stat_intr',
-        'stat_ctxt',
-        'stat_processes',
-        'stat_procs_blocked',
-        'loadavg_1min',
-        'loadavg_5min',
-        'loadavg_15min',
-        'loadavg_runnable',
-        'loadavg_total_threads',
-        # 'sys_numa_zoneallocs',
-        # 'sys_numa_zoneallocs',
-        # 'sys_numa_foreign_allocs',
-        # 'sys_numa_allocation',
-        # 'sys_numa_allocation',
-        # 'sys_numa_interleave',
-        'involved_jobs'
-    ]
-
-    _counters = []
-    for c_name in counters_procstats_to_get:
-        (c_type, c_repr, c_acc, c_descr) = counters_procstats[c_name]
-        _counters.append((c_name, c_type, c_repr, c_acc, c_descr))
-
-    @classmethod
-    def get_zero_counters(cls):
-        return cls(procstats_buffer=None, raw=None)
-
-    @classmethod
-    def fetch(cls, procstats_backend):
-        return procstats_backend.get_procstats()
-
-    def __init__(self, procstats_buffer=None, raw=None):
-        BaseCounters.__init__(self, raw=raw)
-        if raw is not None:
-            pass
-        elif procstats_buffer is None:
-            self._empty_fill()
-        else:
-            for name in ProcstatsCounters._counter_definitions:
-                self._counter_values[name] = procstats_buffer[name]
\ No newline at end of file