diff --git a/modules/clan/prometheus/default.nix b/modules/clan/prometheus/default.nix index 7b817b8..71451c5 100644 --- a/modules/clan/prometheus/default.nix +++ b/modules/clan/prometheus/default.nix @@ -101,6 +101,7 @@ { config, lib, + pkgs, ... }: let @@ -233,38 +234,35 @@ }; } - // lib.optionalAttrs settings.matrix-alertmanager.enable ( - { pkgs, ... }: - { - clan.core.vars.generators.prometheus = { - files.matrix-alertmanager-token.secret = true; - files.matrix-alertmanager-secret.secret = true; - files.matrix-alertmanager-urlfile = { - secret = true; - owner = "alertmanager"; - group = "alertmanager"; - }; - script = '' - echo "" > $out/matrix-alertmanager-token - openssl rand -hex 32 > "$out"/matrix-alertmanager-secret - - echo "http://localhost:3000/alerts?secret=$(cat $out/matrix-alertmanager-secret)" > $out/matrix-alertmanager-urlfile - ''; - runtimeInputs = [ - pkgs.openssl - ]; + // lib.optionalAttrs settings.matrix-alertmanager.enable { + clan.core.vars.generators.prometheus = { + files.matrix-alertmanager-token.secret = true; + files.matrix-alertmanager-secret.secret = true; + files.matrix-alertmanager-urlfile = { + secret = true; + owner = "alertmanager"; + group = "alertmanager"; }; + script = '' + echo "" > $out/matrix-alertmanager-token + openssl rand -hex 32 > "$out"/matrix-alertmanager-secret - services.matrix-alertmanager = lib.mkIf settings.matrix-alertmanager.enable { - enable = true; - tokenFile = config.clan.core.vars.generators.prometheus.files.matrix-alertmanager-token.path; - secretFile = config.clan.core.vars.generators.prometheus.files.matrix-alertmanager-secret.path; - homeserverUrl = settings.matrix-alertmanager.homeserverUrl; - matrixUser = settings.matrix-alertmanager.matrixUser; - matrixRooms = settings.matrix-alertmanager.matrixRooms; - }; - } - ); + echo "http://localhost:3000/alerts?secret=$(cat $out/matrix-alertmanager-secret)" > $out/matrix-alertmanager-urlfile + ''; + runtimeInputs = [ + pkgs.openssl + ]; + }; + + services.matrix-alertmanager = lib.mkIf settings.matrix-alertmanager.enable { + enable = true; + tokenFile = config.clan.core.vars.generators.prometheus.files.matrix-alertmanager-token.path; + secretFile = config.clan.core.vars.generators.prometheus.files.matrix-alertmanager-secret.path; + homeserverUrl = settings.matrix-alertmanager.homeserverUrl; + matrixUser = settings.matrix-alertmanager.matrixUser; + matrixRooms = settings.matrix-alertmanager.matrixRooms; + }; + }; }; }; diff --git a/modules/clan/prometheus/tests/vm/default.nix b/modules/clan/prometheus/tests/vm/default.nix index ca828ee..90b97d1 100644 --- a/modules/clan/prometheus/tests/vm/default.nix +++ b/modules/clan/prometheus/tests/vm/default.nix @@ -39,7 +39,34 @@ roles.nodes.machines."nodeA".settings = { exporters.smartctl = { }; }; - roles.server.machines."server".settings = { }; + roles.server.machines."server".settings = { + extra_rules = [ + { + alert = "test"; + expr = "zfs_pool_health > 0"; + for = "5m"; + labels = { + severity = "critical"; + }; + annotations.summary = '' + Unhealthy Pool at {{ $labels.job }} + Pool {{ $labels.pool }} value {{ $value }} + ''; + } + ]; + + matrix-alertmanager = { + enable = true; + matrixUser = "test@matrixtest.org"; + matrixRooms = [ + { + roomId = "!testroom"; + receivers = [ "matrix" ]; + } + ]; + }; + + }; }; }; }; diff --git a/modules/clan/prometheus/tests/vm/vars/per-machine/server/prometheus/matrix-alertmanager-secret/machines/server b/modules/clan/prometheus/tests/vm/vars/per-machine/server/prometheus/matrix-alertmanager-secret/machines/server new file mode 120000 index 0000000..2bd819e --- /dev/null +++ b/modules/clan/prometheus/tests/vm/vars/per-machine/server/prometheus/matrix-alertmanager-secret/machines/server @@ -0,0 +1 @@ +../../../../../../sops/machines/server \ No newline at end of file diff --git a/modules/clan/prometheus/tests/vm/vars/per-machine/server/prometheus/matrix-alertmanager-secret/secret b/modules/clan/prometheus/tests/vm/vars/per-machine/server/prometheus/matrix-alertmanager-secret/secret new file mode 100644 index 0000000..04ff108 --- /dev/null +++ b/modules/clan/prometheus/tests/vm/vars/per-machine/server/prometheus/matrix-alertmanager-secret/secret @@ -0,0 +1,18 @@ +{ + "data": "ENC[AES256_GCM,data:VszBHfdaNOOYYa6tNUPq9CsJHp+KMBTnZOdHnJz6v3pZQl1zCeYdW3ExvDfNY85tUAZ3YAHthD9JhuR1D+VVVn8=,iv:zbMmaTDZ5mL9IzRTEzuTSPkfwrwOlOIFJtLQyTzGkPw=,tag:Ez68y6gMIj0e/RYQ/Z+s8Q==,type:str]", + "sops": { + "age": [ + { + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBlenlYOCt2RkJOK1hDdUti\ncUpxS1F1RTBnZGo1Njg0Y0EwbzM1dWM5b1hJClFKQ1NDRVVpRXRpOGx3SU52MDZZ\nVVh2NDg5TDgzckFKZ0lNaG1tTEk0MmsKLS0tIHRvR0IzWFZUVkJEM0dwRFZ2SFRz\nNHpCYkI4dUx6YXJSd0xreUN1aUtKNEUK/SJqs5pbFipbp9P7ASUMby7H5ProXknF\nGMvHcIxa6OLLOCRA39YZBVEUlRd03j3rVFILZqVq47CwfaeHj0WBdw==\n-----END AGE ENCRYPTED FILE-----\n", + "recipient": "age1dkrf438z3337d2qnc7ugkggua99xkh55wuf9zgun35fjrxdpnf5qkg4z6j" + }, + { + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA4cWMzaElhYmZaRGhmMXJQ\nbWk2WVl0UmtidjdzYXM0enB3MGdTMWRMVDBjCmdiNEx2RURGL0ZtWCtkcHlabUs0\nVis2d3JieC8yOXV5OW9sN1l5ZWs5Sk0KLS0tIFNmaVpDQklaUDFQK1JnZWZzMDF6\nY0g1M2NHNTEvSkRsTVJSODcxcVVrV0EK8FLzflXqPcooAPh38L7oVliUY8WbB97W\naQYvGf/yo9Izmm8Pa0/ZUGSRnCVRAXtQ1IeR1uPNyuy47mHXO7n7Bw==\n-----END AGE ENCRYPTED FILE-----\n", + "recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg" + } + ], + "lastmodified": "2026-06-22T07:46:31Z", + "mac": "ENC[AES256_GCM,data:ewR8kGgrAj7i6b5UUwh4Fn4CbtRcsDSHhHzrBwGBi9S0XWaatVTQAAmsAVm7DEiJ+a3SQLIAyx6Ef7uqCsZagmzs7LBq0YXNxWtxv62EWPwx8Vihzz3gscDJo1DM3ictX7yi6EiipQ0aYoPCh1veqw8AspLdwnkBxdUF2C+0muc=,iv:bo7vq8BfL437ZI63Os96pAg8EKi8NnrqhABz4Jft9YI=,tag:8krOKra/Z3MJdlmZFBZ7YQ==,type:str]", + "version": "3.13.1" + } +} diff --git a/modules/clan/prometheus/tests/vm/vars/per-machine/server/prometheus/matrix-alertmanager-secret/users/admin b/modules/clan/prometheus/tests/vm/vars/per-machine/server/prometheus/matrix-alertmanager-secret/users/admin new file mode 120000 index 0000000..ca714e1 --- /dev/null +++ b/modules/clan/prometheus/tests/vm/vars/per-machine/server/prometheus/matrix-alertmanager-secret/users/admin @@ -0,0 +1 @@ +../../../../../../sops/users/admin \ No newline at end of file diff --git a/modules/clan/prometheus/tests/vm/vars/per-machine/server/prometheus/matrix-alertmanager-token/machines/server b/modules/clan/prometheus/tests/vm/vars/per-machine/server/prometheus/matrix-alertmanager-token/machines/server new file mode 120000 index 0000000..2bd819e --- /dev/null +++ b/modules/clan/prometheus/tests/vm/vars/per-machine/server/prometheus/matrix-alertmanager-token/machines/server @@ -0,0 +1 @@ +../../../../../../sops/machines/server \ No newline at end of file diff --git a/modules/clan/prometheus/tests/vm/vars/per-machine/server/prometheus/matrix-alertmanager-token/secret b/modules/clan/prometheus/tests/vm/vars/per-machine/server/prometheus/matrix-alertmanager-token/secret new file mode 100644 index 0000000..038c0e4 --- /dev/null +++ b/modules/clan/prometheus/tests/vm/vars/per-machine/server/prometheus/matrix-alertmanager-token/secret @@ -0,0 +1,18 @@ +{ + "data": "ENC[AES256_GCM,data:Mw==,iv:ylmBzsJVBD2pcQNkLcdthT9FX7YW84yZk0u7SlJUdaY=,tag:O1oT/MVijlrQDQG1ddFKlg==,type:str]", + "sops": { + "age": [ + { + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBRd2ZIMElrK2ZacXFTTHFl\nWDkwbGROS1d3WlhzcHhQK21Fc1pmWFZ1VkJRCjlrY1E0cndsZUR1dVQ2L0dud3RQ\nbDlNa3NQZjBPQTAxdUVkUk9lYkgyTGcKLS0tIEUyMVE5Y25BOFJyUWdkdWI1L3VQ\nSE5ubkMvWU9YbE94VTN2VXFUc2F0ajQKKz5VJEtEQcKggoO89ZSfpB3KLBHCnMf+\no8llbCm5bZ39S3qA2Q8spOK4AlkW/NiaCQE4G1LSkvvT6tYEMkwbyQ==\n-----END AGE ENCRYPTED FILE-----\n", + "recipient": "age1dkrf438z3337d2qnc7ugkggua99xkh55wuf9zgun35fjrxdpnf5qkg4z6j" + }, + { + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBnNHhVMWNhZGR6LzgrcG05\nMmVUYU1zcC93YklZUkVETFhZL1BISU13MEFNCjFSZ25EQUQrZTNIcmliTG5UV2xp\nKzQ3MzhkdzcxeGgyV3oxbXo5Y0ZMcmMKLS0tIDVMZjdYWjRkUE50dmE5dm42alpn\nbk1JN1poZWp2bEZNQ3VIdm9PS3Z1ZlEKYOTa7L9tVKq3gZbAeKmCifIxs/sqaPoj\nqdUlsPkwBPjSvlv1QLdRbjBICPdyfH+GiHCmj78DitzZ+KUnRKYqSQ==\n-----END AGE ENCRYPTED FILE-----\n", + "recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg" + } + ], + "lastmodified": "2026-06-22T07:46:31Z", + "mac": "ENC[AES256_GCM,data:NqKlCAKKPF0OTesGozt0GSSd/HT8+h4meiO57EBzD7vwLc9mobG0rLn5C2i3e7tBM13VYzR66qPzQtaWI/jVA7BpJ0PNa2u9MHA2JV6nshRdhMtYgxVCBy8Had7IixAZEs1lLE2zHcWRvLMJPOvUp7tpghb34RddmF/Po/Mkm2s=,iv:9XCsu+rO/DbtaLt13O0/PUo/yV2eUjNP+GGmkYjOIfY=,tag:7tzJE5XBR5PfOqdIh7IKAQ==,type:str]", + "version": "3.13.1" + } +} diff --git a/modules/clan/prometheus/tests/vm/vars/per-machine/server/prometheus/matrix-alertmanager-token/users/admin b/modules/clan/prometheus/tests/vm/vars/per-machine/server/prometheus/matrix-alertmanager-token/users/admin new file mode 120000 index 0000000..ca714e1 --- /dev/null +++ b/modules/clan/prometheus/tests/vm/vars/per-machine/server/prometheus/matrix-alertmanager-token/users/admin @@ -0,0 +1 @@ +../../../../../../sops/users/admin \ No newline at end of file diff --git a/modules/clan/prometheus/tests/vm/vars/per-machine/server/prometheus/matrix-alertmanager-urlfile/machines/server b/modules/clan/prometheus/tests/vm/vars/per-machine/server/prometheus/matrix-alertmanager-urlfile/machines/server new file mode 120000 index 0000000..2bd819e --- /dev/null +++ b/modules/clan/prometheus/tests/vm/vars/per-machine/server/prometheus/matrix-alertmanager-urlfile/machines/server @@ -0,0 +1 @@ +../../../../../../sops/machines/server \ No newline at end of file diff --git a/modules/clan/prometheus/tests/vm/vars/per-machine/server/prometheus/matrix-alertmanager-urlfile/secret b/modules/clan/prometheus/tests/vm/vars/per-machine/server/prometheus/matrix-alertmanager-urlfile/secret new file mode 100644 index 0000000..28188b3 --- /dev/null +++ b/modules/clan/prometheus/tests/vm/vars/per-machine/server/prometheus/matrix-alertmanager-urlfile/secret @@ -0,0 +1,18 @@ +{ + "data": "ENC[AES256_GCM,data:LetAgJg1TgcJL+W7dX8k8MlvpS3PPwVGdco3Z6a8fGhGeQARcuHWV57K4lLQzPpJ7Cruxc6XGQn1U/t3cubdp2NPtwQsP9jaIqPnlZblrq6foHaUmBLaRzc4ed7HOo94ErfV5ZY=,iv:jdt3jMNlK3QvJP8i3OlGydkRPRd2rVybnmUxCDCxfz4=,tag:imNVcalMwnpuaLCdaoaegg==,type:str]", + "sops": { + "age": [ + { + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA4WU9IT3g4TVpiSWdYMUFo\nZGpSUG5xekZLaGMzNTZlcHFaSm1kbTBxUHhjClRIMzlhWW9ub2JFZHhVY3I3TkF3\nU0t5eHJVeFVHRStoNTFkT0lpYnNoMFkKLS0tIEM3aWdIL2RrSGx0ZkdheVRtYUhm\nUHZxeGZvUlBybWJFTHIrZDNxZVloemMKvpt+hkFaRUEXNp1dcfnIWD1i6fyVkaZm\neTn6RBxl1idVN1XlXAwrHHTekuZIobST5kGTV0uR3nLk5Cmhe2x93g==\n-----END AGE ENCRYPTED FILE-----\n", + "recipient": "age1dkrf438z3337d2qnc7ugkggua99xkh55wuf9zgun35fjrxdpnf5qkg4z6j" + }, + { + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBjTDlTM0JBaEowaXJOV1hE\nZWZ2UmUzQmlyek5NRWlqWDZiY0FXVHpKL1d3CitreGc3OUFzS0tYYm5UZ0tUb2pX\nb0pZZ3VacVBma09pSDEyalc0VU1HTVUKLS0tIE5YcnY2RnFCVk13dDZJQ3NMZDQ4\nTGgrY3FwMW5ybjM0a0FmNllrRWZYNWMK2BklSFSm1jT1SsdaMtFWZX4uu4JT2kGi\njyD9E/G0yGl5JH8xfKO/x7vIPuow96WW8bx9aqGRnshXqbe6WzvbIQ==\n-----END AGE ENCRYPTED FILE-----\n", + "recipient": "age1qm0p4vf9jvcnn43s6l4prk8zn6cx0ep9gzvevxecv729xz540v8qa742eg" + } + ], + "lastmodified": "2026-06-22T07:46:31Z", + "mac": "ENC[AES256_GCM,data:bO9VvnIcjXDSyTdEHm1l5Dqm4umLC7FCEaJIbuC+M776q+GR3crq1FWm7J6tinlHDNFX/WmcS417b5WY5VJlP3jqvCalQdttg0EzlhwT65vATvJHoYEp8uqahyLzA9tj9ncQ9LL2XGFeIsvWnU9OcZ5s/42v2DtVdS1/32PT+7U=,iv:W29qp/zHP367rkwiMmpPQcKS/5g6HR5CZGkGCIacwD4=,tag:3DHc5kdj1Ar+7TcaSOnj+Q==,type:str]", + "version": "3.13.1" + } +} diff --git a/modules/clan/prometheus/tests/vm/vars/per-machine/server/prometheus/matrix-alertmanager-urlfile/users/admin b/modules/clan/prometheus/tests/vm/vars/per-machine/server/prometheus/matrix-alertmanager-urlfile/users/admin new file mode 120000 index 0000000..ca714e1 --- /dev/null +++ b/modules/clan/prometheus/tests/vm/vars/per-machine/server/prometheus/matrix-alertmanager-urlfile/users/admin @@ -0,0 +1 @@ +../../../../../../sops/users/admin \ No newline at end of file