cnx-network-clan/modules/monitoring/server.nix

# Monitoring server, imported by control only: VictoriaMetrics (TSDB + scraper)
# and Grafana. VictoriaMetrics binds loopback (only Grafana, on the same host,
# reads it). Grafana is reachable over the ZeroTier mesh, scoped by the firewall
# rule at the bottom; the Hetzner cloud firewall keeps it off the public net.
{
  config,
  lib,
  pkgs,
  ...
}:
let
  mesh = import ../mesh-hosts.nix { inherit config lib; };
  probes = import ./blackbox-probes.nix { inherit lib; };
  vmPort = 8428;
  logsPort = 9428;
  grafanaPort = 3000;
  controlV6 = mesh.hosts.control;

  # A single scrape target with a friendly instance label. IPv6 mesh addresses
  # must be bracketed for Prometheus-style targets.
  target = name: addr: port: {
    targets = [ "${addr}:${toString port}" ];
    labels.instance = name;
  };
  v6 = addr: "[${addr}]";

  adminPasswordFile = config.clan.core.vars.generators.grafana-admin.files."password".path;
  secretKeyFile = config.clan.core.vars.generators.grafana-secret-key.files."secret-key".path;
in
{
  services.victoriametrics = {
    enable = true;
    listenAddress = "127.0.0.1:${toString vmPort}";
    retentionPeriod = "180d";
    # The scraper dials IPv4-only by default; our ns1/ns2 targets are mesh ULAs,
    # so without this VM drops them with "no suitable address found (try -enableTCP6)".
    extraOptions = [ "-enableTCP6" ];
    prometheusConfig = {
      global.scrape_interval = "30s";
      scrape_configs = [
        {
          job_name = "node";
          static_configs = [
            # control scrapes its own node_exporter over loopback so host metrics
            # survive even if the mesh is down; ns1/ns2 are scraped over the mesh.
            (target "control" "127.0.0.1" 9100)
            (target "ns1" (v6 mesh.hosts.ns1) 9100)
            (target "ns2" (v6 mesh.hosts.ns2) 9100)
            (target "mx1" (v6 mesh.hosts.mx1) 9100)
            (target "web01" (v6 mesh.hosts.web01) 9100)
            (target "gw-cnx-1" (v6 mesh.hosts.gw-cnx-1) 9100)
          ];
        }
        {
          job_name = "blocky";
          static_configs = [
            (target "gw-cnx-1" (v6 mesh.hosts.gw-cnx-1) 4000)
          ];
        }
        {
          job_name = "knot";
          static_configs = [
            (target "ns1" (v6 mesh.hosts.ns1) 9433)
            (target "ns2" (v6 mesh.hosts.ns2) 9433)
          ];
        }
      ]
      # Outside-in DNS probes via the blackbox exporter (blackbox.nix). The job
      # list is generated from the same probe definitions the exporter uses.
      ++ probes.scrapeConfigs;
    };
  };

  # Centralized logs: VictoriaLogs ingests journald from all three hosts, each
  # of which runs systemd-journal-upload against /insert/journald (exporters.nix).
  # Binds all interfaces because ns1/ns2 push over the mesh; the firewall rule at
  # the bottom scopes 9428 to the mesh subnet and the Hetzner firewall closes the
  # public side. Retention is set via extraOptions (no dedicated NixOS option).
  services.victorialogs = {
    enable = true;
    listenAddress = ":${toString logsPort}";
    # -enableTCP6: like the scraper above, VictoriaLogs is IPv4-only by default
    # for *listening* too — ":9428" binds 0.0.0.0 only, so ns1/ns2 pushing over
    # the IPv6 mesh get "connection refused". This makes it bind [::] (dual-stack)
    # so the mesh can reach it. Retention has no dedicated NixOS option.
    extraOptions = [
      "-retentionPeriod=30d"
      "-enableTCP6"
    ];
  };

  # Admin password generated once and stored as a clan secret. Retrieve with:
  #   clan vars get control grafana-admin/password
  clan.core.vars.generators.grafana-admin = {
    files."password" = {
      secret = true;
      owner = "grafana";
      group = "grafana";
    };
    runtimeInputs = [ pkgs.openssl ];
    script = ''
      openssl rand -base64 24 | tr -d "\n" > "$out"/password
    '';
  };

  # NixOS 26.05 removed the built-in default for services.grafana's secret_key
  # (used to sign/encrypt DB secrets). Mint our own instead of hardcoding the old
  # upstream default. Safe to generate fresh here: the provisioned datasources
  # carry no credentials, so nothing in Grafana's DB is encrypted with it.
  clan.core.vars.generators.grafana-secret-key = {
    files."secret-key" = {
      secret = true;
      owner = "grafana";
      group = "grafana";
    };
    runtimeInputs = [ pkgs.openssl ];
    script = ''
      openssl rand -base64 32 | tr -d "\n" > "$out"/secret-key
    '';
  };

  services.grafana = {
    enable = true;
    # VictoriaLogs datasource plugin so journald is greppable from Grafana,
    # alongside the metrics datasource.
    declarativePlugins = [ pkgs.grafanaPlugins.victoriametrics-logs-datasource ];
    settings = {
      server = {
        http_addr = "::";
        http_port = grafanaPort;
        root_url = "http://${v6 controlV6}:${toString grafanaPort}/";
      };
      security = {
        admin_user = "admin";
        admin_password = "$__file{${adminPasswordFile}}";
        secret_key = "$__file{${secretKeyFile}}";
      };
      "auth.anonymous".enabled = false;
      users.allow_sign_up = false;
    };
    provision = {
      enable = true;
      datasources.settings = {
        apiVersion = 1;
        datasources = [
          {
            name = "VictoriaMetrics";
            type = "prometheus";
            uid = "victoriametrics";
            access = "proxy";
            url = "http://127.0.0.1:${toString vmPort}";
            isDefault = true;
          }
          {
            name = "VictoriaLogs";
            type = "victoriametrics-logs-datasource";
            uid = "victorialogs";
            access = "proxy";
            url = "http://127.0.0.1:${toString logsPort}";
          }
        ];
      };
      dashboards.settings = {
        apiVersion = 1;
        providers = [
          {
            name = "cnx";
            options.path = ./dashboards;
            options.foldersFromFilesStructure = false;
          }
        ];
      };
    };
  };

  # Grafana (admin laptops + servers) and VictoriaLogs ingestion (ns1/ns2 push
  # journald over the mesh) reachable only from the ZeroTier mesh.
  networking.firewall.extraInputRules = ''
    ip6 saddr ${mesh.subnet} tcp dport { ${toString grafanaPort}, ${toString logsPort} } accept
  '';
}