From 3ed336a56f69b4a06a308ecf28dbf84b97a7b768 Mon Sep 17 00:00:00 2001 From: Hayden Young <22327045+hbjydev@users.noreply.github.com> Date: Wed, 19 Jun 2024 15:47:39 +0100 Subject: [PATCH 1/7] feat: update permissions on alloy config files --- modules/mixins/alloy/default.nix | 6 +----- modules/mixins/docker/default.nix | 6 +----- modules/mixins/ecs-agent/default.nix | 6 +----- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/modules/mixins/alloy/default.nix b/modules/mixins/alloy/default.nix index 1ea3c69..110e8ee 100644 --- a/modules/mixins/alloy/default.nix +++ b/modules/mixins/alloy/default.nix @@ -1,11 +1,7 @@ { pkgs-nix, pkgs, ... }: { imports = [ pkgs-nix.nixosModules.alloy ]; - environment.etc."alloy/config.alloy" = { - source = ./config.alloy; - mode = "0440"; - user = "root"; - }; + environment.etc."alloy/config.alloy".source = ./config.alloy; services.alloy = { enable = true; diff --git a/modules/mixins/docker/default.nix b/modules/mixins/docker/default.nix index 481b969..05695e7 100644 --- a/modules/mixins/docker/default.nix +++ b/modules/mixins/docker/default.nix @@ -3,9 +3,5 @@ virtualisation.oci-containers.backend = "docker"; # Monitoring - environment.etc."alloy/docker.alloy" = { - source = ./config.alloy; - mode = "0440"; - user = "root"; - }; + environment.etc."alloy/docker.alloy".source = ./config.alloy; } diff --git a/modules/mixins/ecs-agent/default.nix b/modules/mixins/ecs-agent/default.nix index 0f12c9f..e198427 100644 --- a/modules/mixins/ecs-agent/default.nix +++ b/modules/mixins/ecs-agent/default.nix @@ -47,9 +47,5 @@ }; # Monitoring - environment.etc."alloy/ecs-agent.alloy" = { - source = ./config.alloy; - mode = "0440"; - user = "root"; - }; + environment.etc."alloy/ecs-agent.alloy".source = ./config.alloy; } From 24a20570bb7cb37b416243b7dadbbc17ba323b87 Mon Sep 17 00:00:00 2001 From: Hayden <22327045+hbjydev@users.noreply.github.com> Date: Sun, 16 Jun 2024 00:46:34 +0100 Subject: [PATCH 2/7] Alloy configuration updates (#21) * feat: switch to nixpkgs alloy package and service, some improvements to alloy ergonomics * feat: nix flake update * fix: update alloy-forwarder nix config for new module --- flake.lock | 70 +++------------------ flake.nix | 3 - modules/mixins/alloy-forwarder/config.alloy | 51 ++++++--------- modules/mixins/alloy-forwarder/default.nix | 11 ++-- modules/mixins/alloy/base.alloy | 56 +++++++++++++++++ modules/mixins/alloy/config.alloy | 2 +- modules/mixins/alloy/default.nix | 15 +---- 7 files changed, 91 insertions(+), 117 deletions(-) create mode 100644 modules/mixins/alloy/base.alloy diff --git a/flake.lock b/flake.lock index 3a09714..3fea64f 100644 --- a/flake.lock +++ b/flake.lock @@ -16,24 +16,6 @@ "type": "indirect" } }, - "flake-parts_2": { - "inputs": { - "nixpkgs-lib": "nixpkgs-lib_2" - }, - "locked": { - "lastModified": 1715865404, - "narHash": "sha256-/GJvTdTpuDjNn84j82cU6bXztE0MSkdnTWClUCRub78=", - "owner": "hercules-ci", - "repo": "flake-parts", - "rev": "8dc45382d5206bd292f9c2768b8058a8fd8311d9", - "type": "github" - }, - "original": { - "owner": "hercules-ci", - "repo": "flake-parts", - "type": "github" - } - }, "nixlib": { "locked": { "lastModified": 1712450863, @@ -57,11 +39,11 @@ ] }, "locked": { - "lastModified": 1716210724, - "narHash": "sha256-iqQa3omRcHGpWb1ds75jS9ruA5R39FTmAkeR3J+ve1w=", + "lastModified": 1718025593, + "narHash": "sha256-WZ1gdKq/9u1Ns/oXuNsDm+W0salonVA0VY1amw8urJ4=", "owner": "nix-community", "repo": "nixos-generators", - "rev": "d14b286322c7f4f897ca4b1726ce38cb68596c94", + "rev": "35c20ba421dfa5059e20e0ef2343c875372bdcf3", "type": "github" }, "original": { @@ -72,11 +54,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1716948383, - "narHash": "sha256-SzDKxseEcHR5KzPXLwsemyTR/kaM9whxeiJohbL04rs=", + "lastModified": 1718318537, + "narHash": "sha256-4Zu0RYRcAY/VWuu6awwq4opuiD//ahpc2aFHg2CWqFY=", "owner": "nixos", "repo": "nixpkgs", - "rev": "ad57eef4ef0659193044870c731987a6df5cf56b", + "rev": "e9ee548d90ff586a6471b4ae80ae9cfcbceb3420", "type": "github" }, "original": { @@ -104,45 +86,11 @@ "type": "github" } }, - "nixpkgs-lib_2": { - "locked": { - "lastModified": 1714640452, - "narHash": "sha256-QBx10+k6JWz6u7VsohfSw8g8hjdBZEf8CFzXH1/1Z94=", - "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/50eb7ecf4cd0a5756d7275c8ba36790e5bd53e33.tar.gz" - }, - "original": { - "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/50eb7ecf4cd0a5756d7275c8ba36790e5bd53e33.tar.gz" - } - }, - "pkgs-nix": { - "inputs": { - "flake-parts": "flake-parts_2", - "nixpkgs": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1717279556, - "narHash": "sha256-msDwm0MHE+zvAfuWXtTBVR4PQhnI/MU9XQzx+4LbUP0=", - "owner": "ALT-F4-LLC", - "repo": "pkgs.nix", - "rev": "3143fc567c8d82edadda31efe90ccc5c2d5d5c64", - "type": "github" - }, - "original": { - "owner": "ALT-F4-LLC", - "repo": "pkgs.nix", - "type": "github" - } - }, "root": { "inputs": { "flake-parts": "flake-parts", "nixos-generators": "nixos-generators", "nixpkgs": "nixpkgs", - "pkgs-nix": "pkgs-nix", "srvos": "srvos" } }, @@ -153,11 +101,11 @@ ] }, "locked": { - "lastModified": 1717058062, - "narHash": "sha256-R8Gb2MlJzfBE76DVWFmfZWODMdAanqxFnK+OOmkoQ7E=", + "lastModified": 1718459800, + "narHash": "sha256-oRkHJbp/jIljo+yXY6sSjMMTBqWNhIjd4qhs0pTjwbs=", "owner": "nix-community", "repo": "srvos", - "rev": "414d1039a58b667e4512ad9f7068aa935ebf8d59", + "rev": "b724a9ad24945a4d6fb11a42f1c2ce072fa3c4c2", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 2e796a1..ab3fab7 100644 --- a/flake.nix +++ b/flake.nix @@ -7,9 +7,6 @@ srvos.url = "github:nix-community/srvos"; srvos.inputs.nixpkgs.follows = "nixpkgs"; - - pkgs-nix.url = "github:ALT-F4-LLC/pkgs.nix"; - pkgs-nix.inputs.nixpkgs.follows = "nixpkgs"; }; outputs = inputs @ {flake-parts, ...}: diff --git a/modules/mixins/alloy-forwarder/config.alloy b/modules/mixins/alloy-forwarder/config.alloy index 49d3f83..74c3f85 100644 --- a/modules/mixins/alloy-forwarder/config.alloy +++ b/modules/mixins/alloy-forwarder/config.alloy @@ -20,20 +20,29 @@ prometheus.receive_http "forward" { ] } -prometheus.scrape "linux_node" { - targets = prometheus.exporter.unix.node.targets - forward_to = [ grafana_cloud.stack.receivers.metrics ] +// Set instance label to the hostname +prometheus.relabel "instance" { + forward_to = [grafana_cloud.stack.receivers.metrics] + rule { + target_label = "instance" + replacement = local.file.hostname.content + } } prometheus.exporter.unix "node" { } +prometheus.scrape "linux_node" { + targets = prometheus.exporter.unix.node.targets + forward_to = [prometheus.relabel.instance.receiver] +} + prometheus.exporter.self "agent" { } prometheus.scrape "agent" { targets = prometheus.exporter.self.agent.targets - forward_to = [ grafana_cloud.stack.receivers.metrics ] + forward_to = [prometheus.relabel.instance.receiver] } loki.source.api "receive" { @@ -46,45 +55,21 @@ loki.source.api "receive" { ] } -loki.relabel "journal" { - forward_to = [] - - rule { - source_labels = ["__journal__systemd_unit"] - target_label = "unit" - } - rule { - source_labels = ["__journal__boot_id"] - target_label = "boot_id" - } - rule { - source_labels = ["__journal__transport"] - target_label = "transport" - } - rule { - source_labels = ["__journal_priority_keyword"] - target_label = "level" - } - rule { - source_labels = ["__journal__hostname"] - target_label = "instance" - } -} - loki.source.journal "read" { forward_to = [ grafana_cloud.stack.receivers.logs, ] - relabel_rules = loki.relabel.journal.rules + relabel_rules = concat( + loki.relabel.journal.rules, + loki.relabel.instance.rules, + ) labels = { "job" = "integrations/node_exporter", } } otelcol.exporter.prometheus "to_prometheus" { - forward_to = [ - grafana_cloud.stack.receivers.metrics, - ] + forward_to = [grafana_cloud.stack.receivers.metrics] } otelcol.exporter.loki "to_loki" { diff --git a/modules/mixins/alloy-forwarder/default.nix b/modules/mixins/alloy-forwarder/default.nix index 6b50c44..ee05c20 100644 --- a/modules/mixins/alloy-forwarder/default.nix +++ b/modules/mixins/alloy-forwarder/default.nix @@ -15,12 +15,9 @@ 4317 # OTLP/gRPC ]; - services.alloy = { - extraArgs = "--stability.level public-preview"; - - environmentFiles = [ "/run/keys/grafana-cloud" ]; - extraEnvironment = { - GRAFANA_CLOUD_STACK = "altf4llc"; - }; + services.alloy.extraFlags = ["--stability.level=public-preview"]; + systemd.services.alloy.serviceConfig.EnvironmentFile = [ "/run/keys/grafana-cloud" ]; + systemd.services.alloy.environment = { + GRAFANA_CLOUD_STACK = "altf4llc"; }; } diff --git a/modules/mixins/alloy/base.alloy b/modules/mixins/alloy/base.alloy new file mode 100644 index 0000000..6310d60 --- /dev/null +++ b/modules/mixins/alloy/base.alloy @@ -0,0 +1,56 @@ +// Grab hostname from /etc instead of environment variables +local.file "hostname" { + filename = "/etc/hostname" +} + +// Set hostname from /etc +loki.relabel "instance" { + forward_to = [] + + rule { + target_label = "instance" + replacement = local.file.hostname.content + } +} + +// Loki journal relabeller +loki.relabel "journal" { + forward_to = [] + + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } + + rule { + source_labels = ["__journal__boot_id"] + target_label = "boot_id" + } + + rule { + source_labels = ["__journal__transport"] + target_label = "transport" + } + + rule { + source_labels = ["__journal_priority_keyword"] + target_label = "level" + } + + rule { + source_labels = ["__journal_container_name"] + target_label = "container_name" + } + + rule { + source_labels = ["__journal_image_name"] + target_label = "container_image" + } + + rule { + source_labels = ["__journal_container_id"] + target_label = "container_id" + } +} + +// vim:ft=hcl diff --git a/modules/mixins/alloy/config.alloy b/modules/mixins/alloy/config.alloy index 0edad40..554b4a6 100644 --- a/modules/mixins/alloy/config.alloy +++ b/modules/mixins/alloy/config.alloy @@ -64,7 +64,7 @@ prometheus.relabel "instance" { forward_to = [otelcol.receiver.prometheus.default.receiver] rule { target_label = "instance" - replacement = env("HOSTNAME") + replacement = local.file.hostname.content } } diff --git a/modules/mixins/alloy/default.nix b/modules/mixins/alloy/default.nix index 110e8ee..2ac2c4e 100644 --- a/modules/mixins/alloy/default.nix +++ b/modules/mixins/alloy/default.nix @@ -1,14 +1,5 @@ -{ pkgs-nix, pkgs, ... }: { - imports = [ pkgs-nix.nixosModules.alloy ]; - +{ ... }: { environment.etc."alloy/config.alloy".source = ./config.alloy; - - services.alloy = { - enable = true; - package = pkgs-nix.packages.${pkgs.system}.alloy; - openFirewall = true; - configPath = "/etc/alloy"; - group = "root"; - user = "root"; - }; + environment.etc."alloy/base.alloy".source = ./base.alloy; + services.alloy.enable = true; } From d53ea4cc1af33e31cdb65d5225ba51ce04771a64 Mon Sep 17 00:00:00 2001 From: Hayden Young <22327045+hbjydev@users.noreply.github.com> Date: Wed, 19 Jun 2024 17:10:50 +0100 Subject: [PATCH 3/7] feat: update alloy configs --- modules/mixins/alloy-forwarder/config.alloy | 13 +++---------- modules/mixins/alloy/base.alloy | 9 ++------- modules/mixins/alloy/config.alloy | 18 +++++++++--------- modules/mixins/ecs-agent/config.alloy | 5 +---- 4 files changed, 15 insertions(+), 30 deletions(-) diff --git a/modules/mixins/alloy-forwarder/config.alloy b/modules/mixins/alloy-forwarder/config.alloy index 74c3f85..1101c20 100644 --- a/modules/mixins/alloy-forwarder/config.alloy +++ b/modules/mixins/alloy-forwarder/config.alloy @@ -50,19 +50,12 @@ loki.source.api "receive" { listen_address = "0.0.0.0" listen_port = 3100 } - forward_to = [ - grafana_cloud.stack.receivers.logs, - ] + forward_to = [grafana_cloud.stack.receivers.logs] } loki.source.journal "read" { - forward_to = [ - grafana_cloud.stack.receivers.logs, - ] - relabel_rules = concat( - loki.relabel.journal.rules, - loki.relabel.instance.rules, - ) + forward_to = [grafana_cloud.stack.receivers.logs] + relabel_rules = loki.relabel.omnibus.rules labels = { "job" = "integrations/node_exporter", } diff --git a/modules/mixins/alloy/base.alloy b/modules/mixins/alloy/base.alloy index 6310d60..f4db4fd 100644 --- a/modules/mixins/alloy/base.alloy +++ b/modules/mixins/alloy/base.alloy @@ -3,19 +3,14 @@ local.file "hostname" { filename = "/etc/hostname" } -// Set hostname from /etc -loki.relabel "instance" { +// Loki journal relabeller +loki.relabel "omnibus" { forward_to = [] rule { target_label = "instance" replacement = local.file.hostname.content } -} - -// Loki journal relabeller -loki.relabel "journal" { - forward_to = [] rule { source_labels = ["__journal__systemd_unit"] diff --git a/modules/mixins/alloy/config.alloy b/modules/mixins/alloy/config.alloy index 554b4a6..3f341ff 100644 --- a/modules/mixins/alloy/config.alloy +++ b/modules/mixins/alloy/config.alloy @@ -32,30 +32,30 @@ loki.relabel "journal" { forward_to = [] rule { - source_labels = ["__journal__systemd_unit"] - target_label = "unit" + target_label = "instance" + replacement = local.file.hostname.content } rule { - source_labels = ["__journal_container_name"] - target_label = "container_name" + source_labels = ["__journal__systemd_unit"] + target_label = "unit" } rule { - source_labels = ["__journal_image_name"] - target_label = "container_image" + source_labels = ["__journal__boot_id"] + target_label = "boot_id" } rule { - source_labels = ["__journal_container_id"] - target_label = "container_id" + source_labels = ["__journal__transport"] + target_label = "transport" } } // Fetch journal entries loki.source.journal "journal" { forward_to = [otelcol.receiver.loki.default.receiver] - relabel_rules = loki.relabel.journal.rules + relabel_rules = loki.relabel.omnibus.rules labels = {component = "loki.source.journal"} } diff --git a/modules/mixins/ecs-agent/config.alloy b/modules/mixins/ecs-agent/config.alloy index ede3c60..70bc7f1 100644 --- a/modules/mixins/ecs-agent/config.alloy +++ b/modules/mixins/ecs-agent/config.alloy @@ -1,8 +1,5 @@ prometheus.scrape "ecs_agent" { - targets = [ - {"__address__" = "127.0.0.1:51680", instance = env("HOSTNAME")}, - ] - + targets = [{"__address__" = "127.0.0.1:51680"}] forward_to = [prometheus.relabel.instance.receiver] scrape_interval = "30s" } From d2141927fec156dfad9c17a4e18df273e4792d7e Mon Sep 17 00:00:00 2001 From: Hayden Young <22327045+hbjydev@users.noreply.github.com> Date: Thu, 20 Jun 2024 17:16:59 +0100 Subject: [PATCH 4/7] feat: alloy unit tests --- .github/workflows/flake.yaml | 15 ++++++--------- flake.nix | 2 ++ justfile | 3 +++ modules/tests/alloy/default.nix | 15 +++++++++++++++ modules/tests/default.nix | 4 ++++ 5 files changed, 30 insertions(+), 9 deletions(-) create mode 100644 modules/tests/alloy/default.nix create mode 100644 modules/tests/default.nix diff --git a/.github/workflows/flake.yaml b/.github/workflows/flake.yaml index 9912afd..7c47e61 100644 --- a/.github/workflows/flake.yaml +++ b/.github/workflows/flake.yaml @@ -10,22 +10,22 @@ env: CACHIX_BINARY_CACHE: altf4llc-os jobs: - check: + test: runs-on: ubuntu-latest + strategy: + matrix: + profile: + - alloy steps: - uses: cachix/install-nix-action@v27 - with: - enable_kvm: true - uses: cachix/cachix-action@v15 with: authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} name: ${{ env.CACHIX_BINARY_CACHE }} - uses: actions/checkout@v4 - - run: nix develop -c just check + - run: nix develop -c just test "${{ matrix.profile }}" build: - needs: - - check runs-on: ubuntu-latest permissions: contents: read @@ -38,9 +38,6 @@ jobs: - actions-runner steps: - uses: cachix/install-nix-action@v27 - with: - enable_kvm: true - - uses: cachix/cachix-action@v15 with: authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} diff --git a/flake.nix b/flake.nix index ab3fab7..d2bfd86 100644 --- a/flake.nix +++ b/flake.nix @@ -24,6 +24,8 @@ buildInputs = [awscli2 just]; }; + checks = import ./modules/tests { inherit pkgs; }; + formatter = pkgs.alejandra; packages = import ./nix/images.nix { inherit system inputs; }; diff --git a/justfile b/justfile index a6bf87e..4d07041 100644 --- a/justfile +++ b/justfile @@ -8,3 +8,6 @@ build profile: publish-ami profile: bash ./ci-build-publish.sh "{{ami_bucket}}" "{{profile}}" + +test profile arch='x86_64-linux': + just build 'checks.{{arch}}.{{profile}}' diff --git a/modules/tests/alloy/default.nix b/modules/tests/alloy/default.nix new file mode 100644 index 0000000..a610e42 --- /dev/null +++ b/modules/tests/alloy/default.nix @@ -0,0 +1,15 @@ +{ pkgs, ... }: +pkgs.testers.runNixOSTest { + name = "alloy-test"; + + nodes.machine = { ... }: { + networking.firewall.allowedTCPPorts = [ 12345 ]; + imports = [ ../../mixins/alloy ]; + }; + + testScript = '' + machine.wait_for_unit("alloy.service", timeout=10) + machine.wait_for_open_port(12345) + machine.succeed("curl http://localhost:12345 | grep -o \"Grafana Alloy\"") + ''; +} diff --git a/modules/tests/default.nix b/modules/tests/default.nix new file mode 100644 index 0000000..14b5306 --- /dev/null +++ b/modules/tests/default.nix @@ -0,0 +1,4 @@ +{ pkgs, ... }: +{ + alloy = pkgs.callPackage ./alloy/default.nix { }; +} From 3bbd2fefcf6d7e3b0b098611de2d8c655f5159f2 Mon Sep 17 00:00:00 2001 From: Hayden Young <22327045+hbjydev@users.noreply.github.com> Date: Thu, 20 Jun 2024 17:31:36 +0100 Subject: [PATCH 5/7] feat: intentionally break test --- modules/mixins/alloy/base.alloy | 2 +- modules/tests/alloy/default.nix | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/mixins/alloy/base.alloy b/modules/mixins/alloy/base.alloy index f4db4fd..de43152 100644 --- a/modules/mixins/alloy/base.alloy +++ b/modules/mixins/alloy/base.alloy @@ -9,7 +9,7 @@ loki.relabel "omnibus" { rule { target_label = "instance" - replacement = local.file.hostname.content + replacement = local.file.hostname.conten } rule { diff --git a/modules/tests/alloy/default.nix b/modules/tests/alloy/default.nix index a610e42..3317c58 100644 --- a/modules/tests/alloy/default.nix +++ b/modules/tests/alloy/default.nix @@ -8,8 +8,9 @@ pkgs.testers.runNixOSTest { }; testScript = '' + import time machine.wait_for_unit("alloy.service", timeout=10) - machine.wait_for_open_port(12345) + machine.wait_for_open_port(12345, timeout=10) machine.succeed("curl http://localhost:12345 | grep -o \"Grafana Alloy\"") ''; } From b69604cf39e1bc7199a10b0dbf7b1559da528b47 Mon Sep 17 00:00:00 2001 From: Hayden Young <22327045+hbjydev@users.noreply.github.com> Date: Thu, 20 Jun 2024 17:33:20 +0100 Subject: [PATCH 6/7] fix: remove unused import from test --- modules/tests/alloy/default.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/tests/alloy/default.nix b/modules/tests/alloy/default.nix index 3317c58..1caaf3f 100644 --- a/modules/tests/alloy/default.nix +++ b/modules/tests/alloy/default.nix @@ -8,7 +8,6 @@ pkgs.testers.runNixOSTest { }; testScript = '' - import time machine.wait_for_unit("alloy.service", timeout=10) machine.wait_for_open_port(12345, timeout=10) machine.succeed("curl http://localhost:12345 | grep -o \"Grafana Alloy\"") From b8efbf36895097e360bcb770bfc4cd9d0f228061 Mon Sep 17 00:00:00 2001 From: Hayden Young <22327045+hbjydev@users.noreply.github.com> Date: Thu, 20 Jun 2024 17:36:31 +0100 Subject: [PATCH 7/7] fix: fix config for test run --- modules/mixins/alloy/base.alloy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/mixins/alloy/base.alloy b/modules/mixins/alloy/base.alloy index de43152..f4db4fd 100644 --- a/modules/mixins/alloy/base.alloy +++ b/modules/mixins/alloy/base.alloy @@ -9,7 +9,7 @@ loki.relabel "omnibus" { rule { target_label = "instance" - replacement = local.file.hostname.conten + replacement = local.file.hostname.content } rule {