Alloy configuration updates

flake.nix

@@ -7,9 +7,6 @@
 
     srvos.url = "github:nix-community/srvos";
     srvos.inputs.nixpkgs.follows = "nixpkgs";
-
-    pkgs-nix.url = "github:ALT-F4-LLC/pkgs.nix";
-    pkgs-nix.inputs.nixpkgs.follows = "nixpkgs";
   };
 
   outputs = inputs @ {flake-parts, ...}:

modules/mixins/alloy-forwarder/config.alloy

@@ -20,20 +20,29 @@ prometheus.receive_http "forward" {
   ]
 }
 
-prometheus.scrape "linux_node" {
-  targets = prometheus.exporter.unix.node.targets
-  forward_to = [ grafana_cloud.stack.receivers.metrics ]
+// Set instance label to the hostname
+prometheus.relabel "instance" {
+  forward_to = [grafana_cloud.stack.receivers.metrics]
+  rule {
+    target_label = "instance"
+    replacement  = local.file.hostname.content
+  }
 }
 
 prometheus.exporter.unix "node" {
 }
 
+prometheus.scrape "linux_node" {
+  targets = prometheus.exporter.unix.node.targets
+  forward_to = [prometheus.relabel.instance.receiver]
+}
+
 prometheus.exporter.self "agent" {
 }
 
 prometheus.scrape "agent" {
   targets = prometheus.exporter.self.agent.targets
-  forward_to = [ grafana_cloud.stack.receivers.metrics ]
+  forward_to = [prometheus.relabel.instance.receiver]
 }
 
 loki.source.api "receive" {
@@ -46,45 +55,21 @@ loki.source.api "receive" {
     ]
 }
 
-loki.relabel "journal" {
-  forward_to = []
-
-  rule {
-    source_labels = ["__journal__systemd_unit"]
-    target_label  = "unit"
-  }
-  rule {
-    source_labels = ["__journal__boot_id"]
-    target_label  = "boot_id"
-  }
-  rule {
-    source_labels = ["__journal__transport"]
-    target_label  = "transport"
-  }
-  rule {
-    source_labels = ["__journal_priority_keyword"]
-    target_label  = "level"
-  }
-  rule {
-    source_labels = ["__journal__hostname"]
-    target_label  = "instance"
-  }
-}
-
 loki.source.journal "read" {
   forward_to = [
     grafana_cloud.stack.receivers.logs,
   ]
-  relabel_rules = loki.relabel.journal.rules
+  relabel_rules = concat(
+    loki.relabel.journal.rules,
+    loki.relabel.instance.rules,
+  )
   labels = {
     "job" = "integrations/node_exporter",
   }
 }
 
 otelcol.exporter.prometheus "to_prometheus" {
-  forward_to = [
-    grafana_cloud.stack.receivers.metrics,
-  ]
+  forward_to = [grafana_cloud.stack.receivers.metrics]
 }
 
 otelcol.exporter.loki "to_loki" {

modules/mixins/alloy-forwarder/default.nix

@@ -15,12 +15,9 @@
     4317 # OTLP/gRPC
   ];
 
-  services.alloy = {
-    extraArgs = "--stability.level public-preview";
-
-    environmentFiles = [ "/run/keys/grafana-cloud" ];
-    extraEnvironment = {
-      GRAFANA_CLOUD_STACK = "altf4llc";
-    };
+  services.alloy.extraFlags = ["--stability.level=public-preview"];
+  systemd.services.alloy.serviceConfig.EnvironmentFile = [ "/run/keys/grafana-cloud" ];
+  systemd.services.alloy.environment = {
+    GRAFANA_CLOUD_STACK = "altf4llc";
   };
 }

modules/mixins/alloy/base.alloy

@@ -0,0 +1,56 @@
+// Grab hostname from /etc instead of environment variables
+local.file "hostname" {
+  filename = "/etc/hostname"
+}
+
+// Set hostname from /etc
+loki.relabel "instance" {
+  forward_to = []
+
+  rule {
+    target_label = "instance"
+    replacement  = local.file.hostname.content
+  }
+}
+
+// Loki journal relabeller
+loki.relabel "journal" {
+  forward_to = []
+
+  rule {
+    source_labels = ["__journal__systemd_unit"]
+    target_label  = "unit"
+  }
+
+  rule {
+    source_labels = ["__journal__boot_id"]
+    target_label  = "boot_id"
+  }
+
+  rule {
+    source_labels = ["__journal__transport"]
+    target_label  = "transport"
+  }
+
+  rule {
+    source_labels = ["__journal_priority_keyword"]
+    target_label  = "level"
+  }
+
+  rule {
+    source_labels = ["__journal_container_name"]
+    target_label  = "container_name"
+  }
+
+  rule {
+    source_labels = ["__journal_image_name"]
+    target_label  = "container_image"
+  }
+
+  rule {
+    source_labels = ["__journal_container_id"]
+    target_label  = "container_id"
+  }
+}
+
+// vim:ft=hcl

modules/mixins/alloy/config.alloy

@@ -64,7 +64,7 @@ prometheus.relabel "instance" {
   forward_to = [otelcol.receiver.prometheus.default.receiver]
   rule {
     target_label = "instance"
-    replacement  = env("HOSTNAME")
+    replacement  = local.file.hostname.content
   }
 }
 

modules/mixins/alloy/default.nix

@@ -1,18 +1,15 @@
-{ pkgs-nix, pkgs, ... }: {
-  imports = [ pkgs-nix.nixosModules.alloy ];
-
+{ ... }: {
   environment.etc."alloy/config.alloy" = {
     source = ./config.alloy;
     mode = "0440";
     user = "root";
   };
 
-  services.alloy = {
-    enable = true;
-    package = pkgs-nix.packages.${pkgs.system}.alloy;
-    openFirewall = true;
-    configPath = "/etc/alloy";
-    group = "root";
+  environment.etc."alloy/base.alloy" = {
+    source = ./base.alloy;
+    mode = "0440";
     user = "root";
   };
+
+  services.alloy.enable = true;
 }