From 4bafc64b8460eb6c55d76b00d7385619631960e0 Mon Sep 17 00:00:00 2001 From: Berwn Date: Fri, 26 Jun 2026 11:34:08 +0700 Subject: [PATCH] fix(monitoring): restore DMARC dashboard under Grafana 13 Grafana 13 externalised its core data sources; nixpkgs no longer bundles the Elasticsearch datasource, so every DMARC panel failed with "Plugin not registered". Load the official Grafana-signed elasticsearch plugin via declarativePlugins, pin the dmarc-ag/dmarc-fo datasource UIDs (Grafana 11+ resolves datasource template variables by UID, not name), and clear stale uid-less records left by an earlier deploy. --- docs/src/monitoring.md | 19 ++++++++- modules/monitoring/parsedmarc.nix | 66 ++++++++++++++++++++++++++++++- 2 files changed, 82 insertions(+), 3 deletions(-) diff --git a/docs/src/monitoring.md b/docs/src/monitoring.md index a8f8858..db47414 100644 --- a/docs/src/monitoring.md +++ b/docs/src/monitoring.md @@ -62,7 +62,24 @@ The `cnx.email` DMARC record (`rua`/`ruf`) points at the `dmarc@cnx.email` mailbox on `mx1`. **parsedmarc** on `control` (`modules/monitoring/parsedmarc.nix`) polls that mailbox over IMAPS, parses the XML reports, and stores them in a local **Elasticsearch** (`127.0.0.1:9200`, loopback-only); Grafana renders them via the -auto-provisioned parsedmarc dashboard + Elasticsearch datasource. +parsedmarc dashboard + two Elasticsearch datasources (`dmarc-ag`/`dmarc-fo`). + +> Two Grafana-13 gotchas, both handled in `modules/monitoring/parsedmarc.nix`: +> +> 1. **The Elasticsearch data source plugin is no longer bundled.** Grafana 13 +> externalised its core data sources and nixpkgs ships no `grafanaPlugins` +> entry for Elasticsearch, so a bare ES datasource fails with "Plugin not +> registered" and every panel errors (and Explore silently falls back to the +> default datasource). We pull the official, Grafana-signed `elasticsearch` +> plugin from the catalog via `services.grafana.declarativePlugins`. +> 2. **Datasource variables now resolve by UID, not name.** The bundled dashboard +> binds its datasource through template variables whose stored value is the +> datasource _name_ (`dmarc-ag`/`dmarc-fo`); Grafana 11+ matches that against +> the datasource **UID**. So we provision the two datasources ourselves with +> their **UID pinned to their name** (and leave the module's own +> `provision.grafana.datasource` off to avoid a duplicate, uid-less pair). +> `deleteDatasources` clears any stale uid-less records left by an earlier +> deploy so the pinned-UID versions can be re-created. The IMAP fetch rides the **mesh**, not the public net: `control` pins `mx1.cnx.email` to mx1's mesh address in `/etc/hosts`, so TLS still validates diff --git a/modules/monitoring/parsedmarc.nix b/modules/monitoring/parsedmarc.nix index b2bc944..6ab58c2 100644 --- a/modules/monitoring/parsedmarc.nix +++ b/modules/monitoring/parsedmarc.nix @@ -9,9 +9,25 @@ # Let's Encrypt cert (primary domain mx1.cnx.email) while the bytes stay on the # overlay. The mailbox passphrase is the shared mail-dmarc-cred secret; parsedmarc # reads it as root in its ExecStartPre, so root-owned (clan default) is fine. -{ config, lib, ... }: +{ + config, + lib, + pkgs, + ... +}: let mesh = import ../mesh-hosts.nix { inherit config lib; }; + + # Grafana 13 (since the externalisation of core data sources) no longer bundles + # the Elasticsearch data source, and nixpkgs has no grafanaPlugins entry for it, + # so a bare elasticsearch datasource reports "Plugin not registered" and every + # DMARC panel fails. Pull the official, Grafana-signed plugin from the catalog + # and load it declaratively. + elasticsearchPlugin = pkgs.grafanaPlugins.grafanaPlugin { + pname = "elasticsearch"; + version = "12.6.4"; + zipHash.x86_64-linux = "sha256-xyAUprdWyQM0IJSg/oBVZ0ltFAAffgPrphln9+IKcUY="; + }; in { imports = [ ../mail-dmarc-cred.nix ]; @@ -20,6 +36,8 @@ in # package rather than opening allowUnfree globally. nixpkgs.config.allowUnfreePredicate = pkg: lib.getName pkg == "elasticsearch"; + services.grafana.declarativePlugins = [ elasticsearchPlugin ]; + # Keep mx1's IMAP traffic on the mesh while presenting the public cert name. networking.hosts.${mesh.hosts.mx1} = [ "mx1.cnx.email" ]; @@ -33,7 +51,10 @@ in # just without source-IP geolocation). geoIp = false; grafana = { - datasource = true; + # We provision the two Elasticsearch datasources ourselves (below) so we + # can pin their UIDs. Leaving this on would add a second, uid-less pair + # and collide on the dmarc-ag/dmarc-fo names. + datasource = false; dashboard = true; }; }; @@ -57,4 +78,45 @@ in }; }; }; + + # The bundled parsedmarc dashboard selects its datasource through template + # variables ($datasourceag/$datasourcefo) whose stored value is the string + # "dmarc-ag"/"dmarc-fo". Grafana 11+ resolves a datasource template variable + # by UID, not by name, so without a matching UID every panel renders + # "datasource was not found". Pin the UIDs to those names so it resolves. + services.grafana.provision.datasources.settings = { + # These two were previously provisioned uid-less, so they exist in Grafana's + # DB with random UIDs. Provisioning a *new* UID onto an existing datasource + # makes Grafana abort with "data source not found" and crash-loop, so delete + # the stale records first; the entries below re-create them with pinned UIDs. + deleteDatasources = [ + { + name = "dmarc-ag"; + orgId = 1; + } + { + name = "dmarc-fo"; + orgId = 1; + } + ]; + datasources = + let + esVersion = lib.getVersion config.services.elasticsearch.package; + es = name: { + inherit name; + uid = name; + type = "elasticsearch"; + access = "proxy"; + url = "http://localhost:9200"; + jsonData = { + timeField = "date_range"; + inherit esVersion; + }; + }; + in + [ + (es "dmarc-ag") + (es "dmarc-fo") + ]; + }; }