mob next [ci-skip] [ci skip] [skip ci]

lastFile:modules/clan/prometheus/default.nix

inventories/default.nix

@@ -360,23 +360,23 @@
                 };
 
                 services.prometheus = {
-                  rules = [
+                  # rules = [
-                    (builtins.toJSON {
+                  #   (builtins.toJSON {
-                      groups = [
+                  #     groups = [
-                        {
+                  #       {
-                          name = "default";
+                  #         name = "default";
-                          rules = [
+                  #         rules = [
-                            {
+                  #           {
-                              alert = "NodeDown";
+                  #             alert = "NodeDown";
-                              expr = "up == 0";
+                  #             expr = "up == 0";
-                              for = "1m";
+                  #             for = "1m";
-                              annotations.summary = "Node {{ $labels.job }} has been down for more than 1 minutes. {{ $labels.instance }}";
+                  #             annotations.summary = "Node {{ $labels.job }} has been down for more than 1 minutes. {{ $labels.instance }}";
-                            }
+                  #           }
-                          ];
+                  #         ];
-                        }
+                  #       }
-                      ];
+                  #     ];
-                    })
+                  #   })
-                  ];
+                  # ];
 
                   alertmanager = {
                     enable = true;

modules/clan/prometheus/default.nix

@@ -18,6 +18,11 @@
             default = "1m";
             description = "How often to scrape targets. Default is 1 minutes";
           };
+          extraRules = lib.mkOption {
+            type = with lib.types; listOf attrs;
+            default = [ ];
+            description = "Additional rules for Prometheus";
+          };
         };
       };
 
@@ -78,6 +83,41 @@
                 ) machineVal.settings.exporters;
               }) roles.nodes.machines;
 
+              rules = [
+                (builtins.toJSON {
+                  groups = [
+                    {
+                      name = "default";
+                      rules = [
+                        {
+                          alert = "NodesDown";
+                          expr = "up == 0";
+                          for = "1m";
+                          labels = {
+                            severity = "critical";
+                          };
+                          annotations.summary = "Node {{ $labels.job }} has been down for more than 1 minutes. {{ $labels.instance }}";
+                        }
+                        {
+                          alert = "SmartCtlErrors";
+                          expr = "smartctl_device_error_log_count > 0";
+                          for = "5m";
+                          labels = {
+                            severity = "medium";
+                          };
+                          annotations.summary = ''
+                            Errors occur on {{ $labels.job }}
+                            Disk {{ $labels.device }} 
+                            {{ $labels.instance }}
+                          '';
+                        }
+                      ]
+                      ++ settings.extraRules;
+                    }
+                  ];
+                })
+              ];
+
             };
 
           };