Merge pull request #9 from elegaanz/opensearch-multi

Opensearch multi

docs/src/compositions/opensearch-multi.md

@@ -1,33 +1,42 @@
 # OpenSearch multi-nœud
 
-Composition à faire.
 
-Pour lancer plusieurs même noeud à partir d'un code on utilise la commande : 
+=======
+Quatre rôles existent :
 
-```nxc start -r nom_du_noeud=nombre_de_noeud -f vm ```
+- `vector` qui fait tourner vector, connecté à OpenSearch
+- `manager`, nœud OpenSearch manager
+- `ingest`, nœud OpenSearch ingest (uniquement, pas de data)
+- `data`, nœud OpenSearch data
 
-Par exemple : 
+## Note pour Docker
 
-Avec le code du noeud data :
+Le cluster peut ne pas démarrer avec Docker, avec le message d'erreur
+suivant dans les logs d'OpenSearch (`systemctl status opensearch.service`).
 
 ```
-    data = { pkgs, config, lib, ... }: {
-      boot.kernel.sysctl."vm.max_map_count" = 262144;
+max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]
+```
 
-      environment.noXlibs = false;
-      environment.systemPackages = with pkgs; [ opensearch-fixed ];
+Cette option est une option du noyau Linux qui peut être configurée avec
+`sysctl`. Dans les autres flavours, elle est configurée via NixOS dans
+le code de la composition, mais dans le cas de Docker, comme le noyau
+est partagé avec la machine hôte, on doit changer l'option manuellement
+sur sa machine.
 
-      systemd.services.opensearch.serviceConfig.ExecStartPre =
-        populate-hosts-script;
+En dehors des containers, il faut donc faire :
 
-      services.opensearch = service-config {
-        settings."node.name" = config.networking.hostName;
-        settings."node.roles" = [ "data" ];
-      };
-    };
-  };
+```bash
+sudo sysctl -w vm.max_map_count=262144
 ```
 
-On peut lancer la commande suivante pour avoir 2 noeuds data identique : ``` nxc start -r data=2 -f vm ```
+## Lancer plusieurs nœuds du même type
+
+Pour lancer plusieurs même noeud à partir d'un code on utilise la commande : 
+
+```
+nxc start -r nom_du_noeud=nombre_de_noeud -f vm
+```
 
+Par exemple, on peut lancer la commande suivante pour avoir 2 noeuds data identique : `nxc start -r data=2 -f vm`
 

opensearch-multi/composition.nix

@@ -0,0 +1,176 @@
+{ pkgs, lib, ... }:
+let
+  keystore-password = "usAe#%EX92R7UHSYwJ";
+  truststore-password = "*!YWptTiu3&okU%E9a";
+  opensearch-fixed = pkgs.opensearch.overrideAttrs (final: previous: {
+    installPhase = previous.installPhase + ''
+      chmod +x $out/plugins/opensearch-security/tools/*.sh
+    '';
+  });
+  cluster-name = "boris";
+  service-config = lib.recursiveUpdate {
+    enable = true;
+    package = opensearch-fixed;
+
+    extraJavaOptions = [
+      "-Xmx512m" # Limite maximale de la mémoire utilisée par la machine virtuelle Java à 512 Mo
+      "-Xms512m" # Mémoire initiale allouée par la machine virtuelle Java à 512 Mo
+    ];
+
+    settings = {
+      "node.name" = "localhost";
+      "cluster.name" = cluster-name;
+      "network.bind_host" = "0.0.0.0";
+      "network.host" = "localhost";
+      "plugins.security.disabled" = true; # TODO: for the moment we disable the security plugin
+      "discovery.type" = "zen";
+    };
+  };
+  opensearch-node = role: {
+    imports = [ ../opensearch-dashboards.nix ];
+
+    boot.kernel.sysctl."vm.max_map_count" = 262144;
+
+    environment.noXlibs = false;
+    environment.systemPackages = with pkgs; [ opensearch-fixed jq ];
+
+    systemd.services.opensearch.serviceConfig.ExecStartPre = populate-hosts-script;
+    services.opensearch = service-config {
+      settings."node.roles" = [ role ];
+    };
+  };
+  # On ne connait pas les IP des nœuds à l'avance donc on
+  # génère ça dynamiquement
+  populate-hosts-script = [
+    "${
+      pkgs.writeShellScriptBin "configure-opensearch" ''
+        CONF=/var/lib/opensearch/config/opensearch.yml
+
+        while [ ! -f $CONF ]; do
+          sleep 1
+        done
+
+        chmod +w $CONF
+
+        # Depending on the flavour, the /etc/nxc/deployment.json file
+        # does not contain the same information
+        # We build a temporary file that contains the hostnames of all deployed nodes
+        # regardless of the current flavour
+        # We first see if we are in a VM or on Grid5000
+        declare -a JQ_PIPELINE
+        if grep "ssh_key.pub" /etc/nxc/deployment.json; then
+          JQ_PIPELINE=('"- " + (.deployment | map(.host) | .[])')
+        else
+          # Same as above, but in this case we are in Docker
+          # and the deployment.json format is not exactly the same
+          JQ_PIPELINE=('"- " + (.deployment | keys | .[])')
+        fi
+
+        # All nodes must have discovery.seed_hosts set to the IP of manager nodes
+        echo "discovery.seed_hosts:" >> $CONF
+        ${pkgs.jq}/bin/jq "''${JQ_PIPELINE[@]}" /etc/nxc/deployment.json -r | grep manager >> $CONF
+
+        # On manager nodes, they should also be listed in cluster.initial_cluster_manager_nodes
+        if hostname | grep manager; then
+          echo "cluster.initial_cluster_manager_nodes:" >> $CONF
+          ${pkgs.jq}/bin/jq "''${JQ_PIPELINE[@]}" /etc/nxc/deployment.json -r | grep manager >> $CONF
+        fi
+
+        # Replace localhost with the actual hostname
+        sed -i "s/localhost/$(hostname)/" $CONF
+      ''
+    }/bin/configure-opensearch"
+  ];
+in {
+  roles = {
+    manager = { pkgs, config, lib, ... }: lib.recursiveUpdate (opensearch-node "cluster_manager") {
+      services.opensearch-dashboards.enable = true;
+    };
+
+    # The ingest node is responsible for pre-processing documents before they are indexed
+    ingest = { pkgs, config, lib, ... }: opensearch-node "ingest";
+
+    # The data node stores the data and executes data-related operations such as search and aggregation
+    data = { pkgs, config, lib, ... }: opensearch-node "data";
+
+    vector = { lib, pkgs, config, ... }: {
+      environment.systemPackages = [ pkgs.vector ];
+      environment.noXlibs = false;
+
+      services.vector = {
+        enable = true;
+        journaldAccess = true;
+        settings = {
+          sources = {
+            "in" = { type = "stdin"; };
+            "systemd" = { type = "journald"; };
+          };
+          sinks = {
+            out = {
+              inputs = [ "in" ];
+              type = "console";
+              encoding = { codec = "text"; };
+            };
+            opensearch = {
+              inputs = [ "systemd" ];
+              type = "elasticsearch";
+              endpoints = [ "https://clusterManager:9200" ];
+              auth = {
+                strategy = "basic";
+                user = "admin";
+                password = "admin";
+              };
+              tls.verify_certificate = false;
+            };
+          };
+        };
+      };
+
+      environment.variables = {
+        # La variable "VECTOR_CONFIG" défini le chemin de la configuration à utiliser quand on
+        # lance la commande `vector`. Le service Systemd génère une config à partir de `services.vector.settings`
+        # et s'assure que le service utilise bien ce fichier. Mais il faut aussi indiquer où ce trouve
+        # ce fichier de configuration à l'outil en ligne de commande disponible dans le PATH.
+        # On parse la configuration systemd pour récupérer le chemin du fichier.
+        VECTOR_CONFIG = lib.lists.last (builtins.split " "
+          config.systemd.services.vector.serviceConfig.ExecStart);
+      };
+    };
+  };
+
+  dockerPorts.manager = [ "5601:5601" "9200:9200" ];
+
+  testScript = ''
+    for opensearch in [manager, data, ingest]:
+      opensearch.start()
+      opensearch.wait_for_unit("opensearch.service")
+      opensearch.wait_for_open_port(9200)
+
+      opensearch.succeed(
+        "curl --fail localhost:9200"
+      )
+
+    vector.start()
+    vector.wait_for_unit("vector.service")
+
+    # The inner curl command uses the Opensearch API and JQ to get the name of the Vector index
+    # (this index contains the current date and thus has a different name every day).
+    # The outer curl call just queries the content of the index and checks that it is in the expected
+    # format with JQ
+    manager.succeed(
+      "curl --fail http://localhost:9200/$(curl --fail http://localhost:9200/_stats | jq -r '.indices | keys[]' | grep vector | tail -n 1)/_search | jq '.hits.hits[0]._source'"
+    )
+
+    # This script gets the host name of all nodes in the cluster, and checks that all the expected nodes
+    # are present
+    manager.succeed(
+      "curl -s http://localhost:9200/_nodes/ | jq '.nodes.[] | .host' -r | grep ingest"
+    )
+    manager.succeed(
+      "curl -s http://localhost:9200/_nodes/ | jq '.nodes.[] | .host' -r | grep data"
+    )
+    manager.succeed(
+      "curl -s http://localhost:9200/_nodes/ | jq '.nodes.[] | .host' -r | grep manager"
+    )
+  '';
+}

opensearch-multi/flake.nix

@@ -0,0 +1,24 @@
+{
+  description = "nixos-compose - basic setup";
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11";
+    nxc.url = "github:elegaanz/nixos-compose/fix-vde-switch-groups";
+  };
+
+  outputs = { self, nixpkgs, nxc }:
+    let
+      system = "x86_64-linux";
+    in
+    {
+      packages.${system} = nxc.lib.compose {
+        inherit nixpkgs system;
+        composition = ./composition.nix;
+      };
+
+      defaultPackage.${system} =
+        self.packages.${system}."composition::vm";
+
+      devShell.${system} = nxc.devShells.${system}.nxcShellFull;
+    };
+}

opensearch-multi/nxc.json

@@ -0,0 +1 @@
+{"composition": "composition.nix", "default_flavour": "vm"}