clanService/prometheus: init monitoring system

2026-06-23 15:27:31 +07:00
parent 77b487a709
commit d09f67a757
32 changed files with 580 additions and 0 deletions
@@ -0,0 +1,101 @@
+{
+  self,
+  hostPkgs,
+  config,
+  lib,
+  ...
+}:
+{
+  name = "service-prometheus";
+  result.update-vars =
+    let
+      relativeDir = lib.removePrefix "${self}/" (toString config.clan.directory);
+    in
+    hostPkgs.writeShellScriptBin "update-vars" ''
+      set -x
+      export PRJ_ROOT=$(git rev-parse --show-toplevel)
+      ${
+        self.inputs.clan-core.packages.${hostPkgs.system}.clan-cli
+      }/bin/clan-generate-test-vars $PRJ_ROOT/${relativeDir} ${config.name}
+    '';
+
+  clan = {
+    test.useContainers = false;
+    directory = ./.;
+    inventory = {
+      machines.server = { };
+      machines.nodeA = { };
+
+      instances = {
+        yggdrasil = {
+          module.name = "yggdrasil";
+          roles.default.machines.server = { };
+          roles.default.machines.nodeA = { };
+        };
+
+        prometheus = {
+          module.name = "@clan/prometheus";
+          module.input = "self";
+          roles.nodes.machines."nodeA".settings = {
+            exporters.smartctl = { };
+          };
+          roles.server.machines."server".settings = {
+            extra_rules = [
+              {
+                alert = "test";
+                expr = "zfs_pool_health > 0";
+                for = "5m";
+                labels = {
+                  severity = "critical";
+                };
+                annotations.summary = ''
+                  Unhealthy Pool at {{ $labels.job }}
+                  Pool {{ $labels.pool }} value {{ $value }}
+                '';
+              }
+            ];
+
+            matrix-alertmanager = {
+              enable = true;
+              matrixUser = "test@matrixtest.org";
+              matrixRooms = [
+                {
+                  roomId = "!testroom";
+                  receivers = [ "matrix" ];
+                }
+              ];
+            };
+
+          };
+        };
+      };
+    };
+  };
+
+  nodes = {
+    server = { };
+    nodeA = { };
+  };
+
+  testScript =
+    { nodes, ... }:
+    ''
+      start_all()
+
+      server.wait_for_unit("prometheus.service")
+
+      nodeA.wait_for_unit("prometheus-smartctl-exporter.service")
+      nodeA.wait_for_open_port(9633)
+
+      nodeA.succeed("systemctl status prometheus-smartctl-exporter.service")
+      nodeA.succeed("curl http://localhost:9633/metrics")
+
+
+      server_ip = server.succeed("ip -4 addr show eth1 | grep -oP '(?<=inet\s)\d+(\.\d+){3}'").strip()
+      nodeA_ip = nodeA.succeed("ip -4 addr show eth1 | grep -oP '(?<=inet\s)\d+(\.\d+){3}'").strip()
+
+      server.succeed(f"ping -c 3 {nodeA_ip}")
+      server.succeed(f"curl -v http://{nodeA_ip}:9633/metrics")
+
+    '';
+}