Berwn
d4a171640b
control runs VictoriaLogs (:9428, 30d, mesh-scoped) with a matching Grafana datasource. Each host ships journald via systemd's own journald.upload to the /insert/journald endpoint -- no extra agent. control uploads over loopback so its logs survive a mesh outage; ns1 and ns2 push over the mesh.
111 lines
3.7 KiB
Nix
111 lines
3.7 KiB
Nix
# Per-host observability agents, imported by every machine. Host metrics
|
|
# everywhere; Knot DNS metrics on the nameservers; journald shipped to
|
|
# VictoriaLogs on control. Everything is reachable only over the ZeroTier mesh
|
|
# (see the firewall rule at the bottom); the public side is already closed by the
|
|
# Hetzner cloud firewall.
|
|
{
|
|
config,
|
|
lib,
|
|
pkgs,
|
|
...
|
|
}:
|
|
let
|
|
mesh = import ../mesh-hosts.nix { inherit config lib; };
|
|
knotEnabled = config.services.knot.enable;
|
|
# node_exporter on every host; knot-exporter only where Knot runs.
|
|
ports = [ 9100 ] ++ lib.optional knotEnabled 9433;
|
|
in
|
|
{
|
|
# extraInputRules (the mesh-scoped accept below) needs the nftables firewall
|
|
# backend. allowedTCP/UDPPorts used elsewhere (53, 9993) translate unchanged.
|
|
networking.nftables.enable = true;
|
|
|
|
# Host metrics: CPU, memory, disk, network, systemd unit state.
|
|
services.prometheus.exporters.node = {
|
|
enable = true;
|
|
# Listen on all interfaces (incl. the v6 mesh). We deliberately do NOT bind
|
|
# to the ZeroTier ULA: the node module renders --web.listen-address without
|
|
# IPv6 brackets, and binding a single ULA would also race ZeroTier bring-up
|
|
# at boot. Reachability is constrained by the firewall rule instead.
|
|
listenAddress = "";
|
|
port = 9100;
|
|
enabledCollectors = [ "systemd" ];
|
|
};
|
|
|
|
# Knot DNS metrics. The exporter reads Knot's control socket, so it runs as the
|
|
# knot user; mod-stats (below) populates the query/response counters it exports.
|
|
systemd.services.knot-exporter = lib.mkIf knotEnabled {
|
|
description = "Prometheus exporter for Knot DNS";
|
|
after = [ "knot.service" ];
|
|
wants = [ "knot.service" ];
|
|
wantedBy = [ "multi-user.target" ];
|
|
serviceConfig = {
|
|
ExecStart = lib.concatStringsSep " " [
|
|
"${pkgs.prometheus-knot-exporter}/bin/knot-exporter"
|
|
"--web-listen-addr ::"
|
|
"--web-listen-port 9433"
|
|
"--knot-library-path ${pkgs.knot-dns.out}/lib/libknot.so"
|
|
"--knot-socket-path /run/knot/knot.sock"
|
|
];
|
|
User = "knot";
|
|
Group = "knot";
|
|
Restart = "on-failure";
|
|
RestartSec = 5;
|
|
NoNewPrivileges = true;
|
|
ProtectSystem = "strict";
|
|
ProtectHome = true;
|
|
PrivateTmp = true;
|
|
ProtectKernelTunables = true;
|
|
ProtectControlGroups = true;
|
|
RestrictAddressFamilies = [
|
|
"AF_INET"
|
|
"AF_INET6"
|
|
"AF_UNIX"
|
|
];
|
|
};
|
|
};
|
|
|
|
# mod-stats: per-query/zone counters exposed over the control socket and read
|
|
# by knot-exporter. Loaded as a global module on the default template so it
|
|
# applies to every zone. (Merges with the zone/acl/policy settings elsewhere.)
|
|
services.knot.settings = lib.mkIf knotEnabled {
|
|
"mod-stats" = [
|
|
{
|
|
id = "default";
|
|
"request-protocol" = true;
|
|
"server-operation" = true;
|
|
"response-code" = true;
|
|
"query-type" = true;
|
|
"reply-nodata" = true;
|
|
}
|
|
];
|
|
template = [
|
|
{
|
|
id = "default";
|
|
global-module = [ "mod-stats/default" ];
|
|
}
|
|
];
|
|
};
|
|
|
|
# Ship journald to VictoriaLogs on control (services.victorialogs in
|
|
# server.nix). control uploads to loopback so its own logs survive a mesh
|
|
# outage; ns1/ns2 push over the mesh to control's ZeroTier address.
|
|
services.journald.upload = {
|
|
enable = true;
|
|
settings.Upload.URL =
|
|
let
|
|
dest =
|
|
if config.networking.hostName == "control" then
|
|
"127.0.0.1:9428"
|
|
else
|
|
"[${mesh.hosts.control}]:9428";
|
|
in
|
|
"http://${dest}/insert/journald";
|
|
};
|
|
|
|
# Scrape ports reachable only from the ZeroTier mesh.
|
|
networking.firewall.extraInputRules = ''
|
|
ip6 saddr ${mesh.subnet} tcp dport { ${lib.concatMapStringsSep ", " toString ports} } accept
|
|
'';
|
|
}
|