Add VictoriaLogs for centralized journald across all hosts
control runs VictoriaLogs (:9428, 30d, mesh-scoped) with a matching Grafana datasource. Each host ships journald via systemd's own journald.upload to the /insert/journald endpoint -- no extra agent. control uploads over loopback so its logs survive a mesh outage; ns1 and ns2 push over the mesh.
This commit is contained in:
@@ -12,6 +12,7 @@ let
|
||||
mesh = import ../mesh-hosts.nix { inherit config lib; };
|
||||
probes = import ./blackbox-probes.nix { inherit lib; };
|
||||
vmPort = 8428;
|
||||
logsPort = 9428;
|
||||
grafanaPort = 3000;
|
||||
controlV6 = mesh.hosts.control;
|
||||
|
||||
@@ -60,6 +61,17 @@ in
|
||||
};
|
||||
};
|
||||
|
||||
# Centralized logs: VictoriaLogs ingests journald from all three hosts, each
|
||||
# of which runs systemd-journal-upload against /insert/journald (exporters.nix).
|
||||
# Binds all interfaces because ns1/ns2 push over the mesh; the firewall rule at
|
||||
# the bottom scopes 9428 to the mesh subnet and the Hetzner firewall closes the
|
||||
# public side. Retention is set via extraOptions (no dedicated NixOS option).
|
||||
services.victorialogs = {
|
||||
enable = true;
|
||||
listenAddress = ":${toString logsPort}";
|
||||
extraOptions = [ "-retentionPeriod=30d" ];
|
||||
};
|
||||
|
||||
# Admin password generated once and stored as a clan secret. Retrieve with:
|
||||
# clan vars get control grafana-admin/password
|
||||
clan.core.vars.generators.grafana-admin = {
|
||||
@@ -76,6 +88,9 @@ in
|
||||
|
||||
services.grafana = {
|
||||
enable = true;
|
||||
# VictoriaLogs datasource plugin so journald is greppable from Grafana,
|
||||
# alongside the metrics datasource.
|
||||
declarativePlugins = [ pkgs.grafanaPlugins.victoriametrics-logs-datasource ];
|
||||
settings = {
|
||||
server = {
|
||||
http_addr = "::";
|
||||
@@ -102,6 +117,13 @@ in
|
||||
url = "http://127.0.0.1:${toString vmPort}";
|
||||
isDefault = true;
|
||||
}
|
||||
{
|
||||
name = "VictoriaLogs";
|
||||
type = "victoriametrics-logs-datasource";
|
||||
uid = "victorialogs";
|
||||
access = "proxy";
|
||||
url = "http://127.0.0.1:${toString logsPort}";
|
||||
}
|
||||
];
|
||||
};
|
||||
dashboards.settings = {
|
||||
@@ -117,8 +139,9 @@ in
|
||||
};
|
||||
};
|
||||
|
||||
# Grafana reachable only from the ZeroTier mesh (admin laptops + servers).
|
||||
# Grafana (admin laptops + servers) and VictoriaLogs ingestion (ns1/ns2 push
|
||||
# journald over the mesh) reachable only from the ZeroTier mesh.
|
||||
networking.firewall.extraInputRules = ''
|
||||
ip6 saddr ${mesh.subnet} tcp dport ${toString grafanaPort} accept
|
||||
ip6 saddr ${mesh.subnet} tcp dport { ${toString grafanaPort}, ${toString logsPort} } accept
|
||||
'';
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user