Berwn
33ac7e106b
control runs VictoriaMetrics (loopback) and Grafana; every machine exports node metrics and the nameservers export Knot stats (mod-stats + knot-exporter). Scraping and the Grafana UI ride the ZeroTier mesh only, scoped by nftables to the mesh /88; the public side stays closed by the Hetzner cloud firewall. The provisioned DNS dashboard includes a per-zone SOA serial table to catch primary/secondary drift. ZeroTier ULAs are centralised in mesh-hosts.nix.
118 lines
3.3 KiB
Nix
118 lines
3.3 KiB
Nix
# Monitoring server, imported by control only: VictoriaMetrics (TSDB + scraper)
|
|
# and Grafana. VictoriaMetrics binds loopback (only Grafana, on the same host,
|
|
# reads it). Grafana is reachable over the ZeroTier mesh, scoped by the firewall
|
|
# rule at the bottom; the Hetzner cloud firewall keeps it off the public net.
|
|
{
|
|
config,
|
|
lib,
|
|
pkgs,
|
|
...
|
|
}:
|
|
let
|
|
mesh = import ../mesh-hosts.nix;
|
|
vmPort = 8428;
|
|
grafanaPort = 3000;
|
|
controlV6 = mesh.hosts.control;
|
|
|
|
# A single scrape target with a friendly instance label. IPv6 mesh addresses
|
|
# must be bracketed for Prometheus-style targets.
|
|
target = name: addr: port: {
|
|
targets = [ "${addr}:${toString port}" ];
|
|
labels.instance = name;
|
|
};
|
|
v6 = addr: "[${addr}]";
|
|
|
|
adminPasswordFile = config.clan.core.vars.generators.grafana-admin.files."password".path;
|
|
in
|
|
{
|
|
services.victoriametrics = {
|
|
enable = true;
|
|
listenAddress = "127.0.0.1:${toString vmPort}";
|
|
retentionPeriod = "180d";
|
|
prometheusConfig = {
|
|
global.scrape_interval = "30s";
|
|
scrape_configs = [
|
|
{
|
|
job_name = "node";
|
|
static_configs = [
|
|
# control scrapes its own node_exporter over loopback so host metrics
|
|
# survive even if the mesh is down; ns1/ns2 are scraped over the mesh.
|
|
(target "control" "127.0.0.1" 9100)
|
|
(target "ns1" (v6 mesh.hosts.ns1) 9100)
|
|
(target "ns2" (v6 mesh.hosts.ns2) 9100)
|
|
];
|
|
}
|
|
{
|
|
job_name = "knot";
|
|
static_configs = [
|
|
(target "ns1" (v6 mesh.hosts.ns1) 9433)
|
|
(target "ns2" (v6 mesh.hosts.ns2) 9433)
|
|
];
|
|
}
|
|
];
|
|
};
|
|
};
|
|
|
|
# Admin password generated once and stored as a clan secret. Retrieve with:
|
|
# clan vars get control grafana-admin/password
|
|
clan.core.vars.generators.grafana-admin = {
|
|
files."password" = {
|
|
secret = true;
|
|
owner = "grafana";
|
|
group = "grafana";
|
|
};
|
|
runtimeInputs = [ pkgs.openssl ];
|
|
script = ''
|
|
openssl rand -base64 24 | tr -d "\n" > "$out"/password
|
|
'';
|
|
};
|
|
|
|
services.grafana = {
|
|
enable = true;
|
|
settings = {
|
|
server = {
|
|
http_addr = "::";
|
|
http_port = grafanaPort;
|
|
root_url = "http://${v6 controlV6}:${toString grafanaPort}/";
|
|
};
|
|
security = {
|
|
admin_user = "admin";
|
|
admin_password = "$__file{${adminPasswordFile}}";
|
|
};
|
|
"auth.anonymous".enabled = false;
|
|
users.allow_sign_up = false;
|
|
};
|
|
provision = {
|
|
enable = true;
|
|
datasources.settings = {
|
|
apiVersion = 1;
|
|
datasources = [
|
|
{
|
|
name = "VictoriaMetrics";
|
|
type = "prometheus";
|
|
uid = "victoriametrics";
|
|
access = "proxy";
|
|
url = "http://127.0.0.1:${toString vmPort}";
|
|
isDefault = true;
|
|
}
|
|
];
|
|
};
|
|
dashboards.settings = {
|
|
apiVersion = 1;
|
|
providers = [
|
|
{
|
|
name = "cnx";
|
|
options.path = ./dashboards;
|
|
options.foldersFromFilesStructure = false;
|
|
}
|
|
];
|
|
};
|
|
};
|
|
};
|
|
|
|
# Grafana reachable only from the ZeroTier mesh (admin laptops + servers).
|
|
networking.firewall.extraInputRules = ''
|
|
ip6 saddr ${mesh.subnet} tcp dport ${toString grafanaPort} accept
|
|
'';
|
|
}
|