fix(monitoring): restore DMARC dashboard under Grafana 13

Grafana 13 externalised its core data sources; nixpkgs no longer bundles
the Elasticsearch datasource, so every DMARC panel failed with "Plugin not
registered". Load the official Grafana-signed elasticsearch plugin via
declarativePlugins, pin the dmarc-ag/dmarc-fo datasource UIDs (Grafana 11+
resolves datasource template variables by UID, not name), and clear stale
uid-less records left by an earlier deploy.

docs/src/monitoring.md

@@ -62,7 +62,24 @@ The `cnx.email` DMARC record (`rua`/`ruf`) points at the `dmarc@cnx.email`
 mailbox on `mx1`. **parsedmarc** on `control` (`modules/monitoring/parsedmarc.nix`)
 polls that mailbox over IMAPS, parses the XML reports, and stores them in a local
 **Elasticsearch** (`127.0.0.1:9200`, loopback-only); Grafana renders them via the
-auto-provisioned parsedmarc dashboard + Elasticsearch datasource.
+parsedmarc dashboard + two Elasticsearch datasources (`dmarc-ag`/`dmarc-fo`).
+
+> Two Grafana-13 gotchas, both handled in `modules/monitoring/parsedmarc.nix`:
+>
+> 1. **The Elasticsearch data source plugin is no longer bundled.** Grafana 13
+>    externalised its core data sources and nixpkgs ships no `grafanaPlugins`
+>    entry for Elasticsearch, so a bare ES datasource fails with "Plugin not
+>    registered" and every panel errors (and Explore silently falls back to the
+>    default datasource). We pull the official, Grafana-signed `elasticsearch`
+>    plugin from the catalog via `services.grafana.declarativePlugins`.
+> 2. **Datasource variables now resolve by UID, not name.** The bundled dashboard
+>    binds its datasource through template variables whose stored value is the
+>    datasource _name_ (`dmarc-ag`/`dmarc-fo`); Grafana 11+ matches that against
+>    the datasource **UID**. So we provision the two datasources ourselves with
+>    their **UID pinned to their name** (and leave the module's own
+>    `provision.grafana.datasource` off to avoid a duplicate, uid-less pair).
+>    `deleteDatasources` clears any stale uid-less records left by an earlier
+>    deploy so the pinned-UID versions can be re-created.
 
 The IMAP fetch rides the **mesh**, not the public net: `control` pins
 `mx1.cnx.email` to mx1's mesh address in `/etc/hosts`, so TLS still validates

modules/monitoring/parsedmarc.nix

@@ -9,9 +9,25 @@
 # Let's Encrypt cert (primary domain mx1.cnx.email) while the bytes stay on the
 # overlay. The mailbox passphrase is the shared mail-dmarc-cred secret; parsedmarc
 # reads it as root in its ExecStartPre, so root-owned (clan default) is fine.
-{ config, lib, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
 let
   mesh = import ../mesh-hosts.nix { inherit config lib; };
+
+  # Grafana 13 (since the externalisation of core data sources) no longer bundles
+  # the Elasticsearch data source, and nixpkgs has no grafanaPlugins entry for it,
+  # so a bare elasticsearch datasource reports "Plugin not registered" and every
+  # DMARC panel fails. Pull the official, Grafana-signed plugin from the catalog
+  # and load it declaratively.
+  elasticsearchPlugin = pkgs.grafanaPlugins.grafanaPlugin {
+    pname = "elasticsearch";
+    version = "12.6.4";
+    zipHash.x86_64-linux = "sha256-xyAUprdWyQM0IJSg/oBVZ0ltFAAffgPrphln9+IKcUY=";
+  };
 in
 {
   imports = [ ../mail-dmarc-cred.nix ];
@@ -20,6 +36,8 @@ in
   # package rather than opening allowUnfree globally.
   nixpkgs.config.allowUnfreePredicate = pkg: lib.getName pkg == "elasticsearch";
 
+  services.grafana.declarativePlugins = [ elasticsearchPlugin ];
+
   # Keep mx1's IMAP traffic on the mesh while presenting the public cert name.
   networking.hosts.${mesh.hosts.mx1} = [ "mx1.cnx.email" ];
 
@@ -33,7 +51,10 @@ in
       # just without source-IP geolocation).
       geoIp = false;
       grafana = {
-        datasource = true;
+        # We provision the two Elasticsearch datasources ourselves (below) so we
+        # can pin their UIDs. Leaving this on would add a second, uid-less pair
+        # and collide on the dmarc-ag/dmarc-fo names.
+        datasource = false;
         dashboard = true;
       };
     };
@@ -57,4 +78,45 @@ in
       };
     };
   };
+
+  # The bundled parsedmarc dashboard selects its datasource through template
+  # variables ($datasourceag/$datasourcefo) whose stored value is the string
+  # "dmarc-ag"/"dmarc-fo". Grafana 11+ resolves a datasource template variable
+  # by UID, not by name, so without a matching UID every panel renders
+  # "datasource was not found". Pin the UIDs to those names so it resolves.
+  services.grafana.provision.datasources.settings = {
+    # These two were previously provisioned uid-less, so they exist in Grafana's
+    # DB with random UIDs. Provisioning a *new* UID onto an existing datasource
+    # makes Grafana abort with "data source not found" and crash-loop, so delete
+    # the stale records first; the entries below re-create them with pinned UIDs.
+    deleteDatasources = [
+      {
+        name = "dmarc-ag";
+        orgId = 1;
+      }
+      {
+        name = "dmarc-fo";
+        orgId = 1;
+      }
+    ];
+    datasources =
+      let
+        esVersion = lib.getVersion config.services.elasticsearch.package;
+        es = name: {
+          inherit name;
+          uid = name;
+          type = "elasticsearch";
+          access = "proxy";
+          url = "http://localhost:9200";
+          jsonData = {
+            timeField = "date_range";
+            inherit esVersion;
+          };
+        };
+      in
+      [
+        (es "dmarc-ag")
+        (es "dmarc-fo")
+      ];
+  };
 }