add health check service and example that uses it

This commit is contained in:
Daniel Barlow
2024-07-30 22:37:43 +01:00
parent 8fa3443923
commit c4d00e062a
4 changed files with 102 additions and 10 deletions

View File

@@ -0,0 +1,43 @@
## Health check
##
## Runs a service and a separate periodic health process. When the
## health check starts failing over a period of time, kill the service.
## (Usually that means the supervisor will restart it, but you can
## have other behaviours by e.g. combining this service with a round-robin
## for failover)
{ lib, pkgs, config, ...}:
let
inherit (lib) mkOption types;
inherit (pkgs) liminix;
# inherit (pkgs.liminix.services) longrun;
in {
options = {
system.service.health-check = mkOption {
description = "run a service while periodically checking it is healthy";
type = liminix.lib.types.serviceDefn;
};
};
config.system.service.health-check = config.system.callService ./service.nix {
service = mkOption {
type = liminix.lib.types.service;
};
interval = mkOption {
description = "interval between checks, in seconds";
type = types.int;
default = 10;
example = 10;
};
threshold = mkOption {
description = "number of consecutive failures required for the service to be kicked";
type = types.int;
example = 3;
};
healthCheck = mkOption {
description = "health check command or script. Expected to exit 0 if the service is healthy or any other exit status otherwise";
type = types.path;
};
};
config.programs.busybox.applets = ["expr"];
}

View File

@@ -0,0 +1,37 @@
{
liminix, lib, lim, s6
}:
{ service, interval, threshold, healthCheck } :
let
inherit (liminix.services) oneshot longrun;
inherit (builtins) toString;
inherit (service) name;
checker = let name' = "check-${name}"; in longrun {
name = name';
run = ''
fails=0
echo waiting for /run/service/${name}
${s6}/bin/s6-svwait -U /run/service/${name} || exit
while sleep ${toString interval} ; do
${healthCheck}
if test $? -gt 0; then
fails=$(expr $fails + 1)
else
fails=0
fi
echo fails $fails/${toString threshold} for ${name}
if test "$fails" -gt "${toString threshold}" ; then
echo time to die
${s6}/bin/s6-svc -r /run/service/${name}
echo bounced
fails=0
echo waiting for /run/service/${name}
${s6}/bin/s6-svwait -U /run/service/${name}
fi
done
'';
};
in service.overrideAttrs(o: {
buildInputs = (lim.orEmpty o.buildInputs) ++ [ checker ];
dependencies = (lim.orEmpty o.dependencies) ++ [ checker ];
})