332 lines
12 KiB
Nix
332 lines
12 KiB
Nix
# End-to-end test: Stalwart 0.15 (old) -> Simple NixOS Mailserver (new),
|
|
# driven by mailmig on a third machine. Exercises: REST inventory, password
|
|
# hash carry-over (sha512-crypt, bcrypt, plaintext, none), the generated Nix
|
|
# (imported by the SNM node), Stalwart master-user login over STARTTLS,
|
|
# Dovecot master-user login over implicit TLS, folder mapping, flags/dates,
|
|
# de-duplication, idempotence, delta and flag passes, and DKIM -> zone output.
|
|
{ mailmig, nixos-mailserver }:
|
|
{ pkgs, lib, ... }:
|
|
let
|
|
# Python 3.13 verifies strictly (VERIFY_X509_STRICT): the CA needs keyUsage,
|
|
# leaves need SKI/AKI.
|
|
certs = pkgs.runCommand "mailmig-test-certs" { nativeBuildInputs = [ pkgs.openssl ]; } ''
|
|
mkdir $out && cd $out
|
|
cat > ca.cnf <<EOF
|
|
[req]
|
|
distinguished_name = dn
|
|
x509_extensions = v3_ca
|
|
prompt = no
|
|
[dn]
|
|
CN = mailmig test CA
|
|
[v3_ca]
|
|
basicConstraints = critical,CA:TRUE
|
|
keyUsage = critical,keyCertSign,cRLSign
|
|
subjectKeyIdentifier = hash
|
|
EOF
|
|
openssl req -x509 -config ca.cnf -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \
|
|
-keyout ca.key -out ca.crt -days 3650
|
|
for h in old new; do
|
|
openssl req -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \
|
|
-keyout $h.key -out $h.csr -subj "/CN=$h.test"
|
|
cat > $h.ext <<EOF
|
|
subjectAltName = DNS:$h.test
|
|
extendedKeyUsage = serverAuth
|
|
keyUsage = critical,digitalSignature
|
|
basicConstraints = critical,CA:FALSE
|
|
subjectKeyIdentifier = hash
|
|
authorityKeyIdentifier = keyid
|
|
EOF
|
|
openssl x509 -req -in $h.csr -CA ca.crt -CAkey ca.key -CAcreateserial \
|
|
-out $h.crt -days 3650 -extfile $h.ext
|
|
done
|
|
chmod 644 *
|
|
'';
|
|
|
|
masterUsers = pkgs.runCommand "dovecot-master-users" { nativeBuildInputs = [ pkgs.mkpasswd ]; } ''
|
|
printf 'mailmig:{CRYPT}%s\n' "$(mkpasswd -m sha-512 masterpass)" > $out
|
|
'';
|
|
|
|
migrated = import ./expected-mail-accounts.nix;
|
|
hashFile = login: "/var/lib/mailmig-hashes/${login}.hash";
|
|
|
|
hosts =
|
|
nodes:
|
|
{
|
|
${nodes.old.networking.primaryIPAddress} = [ "old.test" ];
|
|
${nodes.new.networking.primaryIPAddress} = [ "new.test" ];
|
|
};
|
|
|
|
mailmigToml = pkgs.writeText "mailmig.toml" ''
|
|
[source]
|
|
host = "old.test"
|
|
port = 143
|
|
tls = "starttls"
|
|
[source.auth]
|
|
mode = "master"
|
|
user = "migrator"
|
|
password = "migratorpass"
|
|
login_format = "{user}%{master}"
|
|
|
|
[stalwart]
|
|
url = "http://old.test:8080"
|
|
user = "admin"
|
|
password = "adminpass"
|
|
|
|
[destination]
|
|
host = "new.test"
|
|
tls = "implicit"
|
|
ca_file = "${certs}/ca.crt"
|
|
[destination.auth]
|
|
mode = "master"
|
|
user = "mailmig"
|
|
password_env = "DST_MASTER"
|
|
login_format = "{user}*{master}"
|
|
|
|
[folders]
|
|
exclude = ["Shared Folders/*"]
|
|
|
|
[sync]
|
|
jobs = 2
|
|
batch_messages = 3
|
|
|
|
[dns]
|
|
provider = "zonefile"
|
|
mx = ["10 new.test"]
|
|
dmarc = "v=DMARC1; p=quarantine"
|
|
old_hosts = ["old.test"]
|
|
client_host = "new.test"
|
|
'';
|
|
in
|
|
{
|
|
name = "mailmig-e2e";
|
|
|
|
nodes.old =
|
|
{ nodes, ... }:
|
|
{
|
|
virtualisation.memorySize = 1024;
|
|
networking.hosts = hosts nodes;
|
|
services.stalwart = {
|
|
enable = true;
|
|
stateVersion = "26.05";
|
|
openFirewall = true;
|
|
settings = {
|
|
server.hostname = "old.test";
|
|
certificate.test = {
|
|
cert = "%{file:${certs}/old.crt}%";
|
|
private-key = "%{file:${certs}/old.key}%";
|
|
};
|
|
server.tls = {
|
|
certificate = "test";
|
|
enable = true;
|
|
implicit = false;
|
|
};
|
|
server.listener = {
|
|
imap = {
|
|
bind = [ "[::]:143" ];
|
|
protocol = "imap";
|
|
};
|
|
http = {
|
|
bind = [ "[::]:8080" ];
|
|
protocol = "http";
|
|
};
|
|
};
|
|
authentication.fallback-admin = {
|
|
user = "admin";
|
|
secret = "adminpass";
|
|
};
|
|
# Stalwart 0.15 master user: "<account>%migrator" + this secret.
|
|
authentication.master = {
|
|
user = "migrator";
|
|
secret = "migratorpass";
|
|
};
|
|
};
|
|
};
|
|
};
|
|
|
|
nodes.new =
|
|
{ nodes, ... }:
|
|
{
|
|
imports = [
|
|
nixos-mailserver.nixosModules.default
|
|
./expected-dovecot-master-user.nix
|
|
];
|
|
virtualisation.memorySize = 1536;
|
|
virtualisation.cores = 2;
|
|
networking.hosts = hosts nodes;
|
|
mailserver = {
|
|
enable = true;
|
|
stateVersion = 3;
|
|
fqdn = "new.test";
|
|
inherit (migrated) domains forwards;
|
|
localDnsResolver = false;
|
|
x509 = {
|
|
certificateFile = "${certs}/new.crt";
|
|
privateKeyFile = "${certs}/new.key";
|
|
};
|
|
# Same mapping the clan module uses, with plain files instead of clan vars.
|
|
accounts = lib.mapAttrs (login: a: {
|
|
inherit (a) aliases;
|
|
quota = a.quota or null;
|
|
hashedPasswordFile = hashFile login;
|
|
}) migrated.accounts;
|
|
};
|
|
# Placeholder hashes ("!" never verifies) until mailmig's are copied in.
|
|
systemd.tmpfiles.settings."10-mailmig" =
|
|
lib.mapAttrs' (login: _: lib.nameValuePair (hashFile login) { f.argument = "!"; }) migrated.accounts
|
|
// {
|
|
"/var/lib/dovecot-master-users"."L+".argument = "${masterUsers}";
|
|
};
|
|
services.rspamd.locals."options.inc".text = ''
|
|
dns {
|
|
nameservers = ["127.0.0.1"];
|
|
timeout = 0.0s;
|
|
retransmits = 0;
|
|
}
|
|
'';
|
|
services.rspamd.overrides."options.inc".enable = false;
|
|
services.postfix-tlspol.configurePostfix = lib.mkForce false;
|
|
};
|
|
|
|
nodes.client =
|
|
{ nodes, ... }:
|
|
{
|
|
networking.hosts = hosts nodes;
|
|
security.pki.certificateFiles = [ "${certs}/ca.crt" ];
|
|
environment.systemPackages = [
|
|
mailmig
|
|
pkgs.python3
|
|
pkgs.mkpasswd
|
|
pkgs.jq
|
|
pkgs.diffutils
|
|
];
|
|
};
|
|
|
|
testScript = ''
|
|
import json
|
|
|
|
def mig(args, ok=True):
|
|
cmd = f"cd /root/mig && DST_MASTER=masterpass mailmig {args} 2>&1"
|
|
return client.succeed(cmd) if ok else client.fail(cmd)
|
|
|
|
def fixture(args):
|
|
return client.succeed(f"python3 ${./imap_fixture.py} {args}")
|
|
|
|
def report(path):
|
|
return json.loads(client.succeed(f"cat /root/mig/{path}"))
|
|
|
|
def total(rep, key):
|
|
return sum(f[key] for a in rep for f in a["folders"])
|
|
|
|
start_all()
|
|
old.wait_for_unit("stalwart.service")
|
|
old.wait_for_open_port(8080)
|
|
old.wait_for_open_port(143)
|
|
new.wait_for_unit("dovecot.service")
|
|
new.wait_for_open_port(993)
|
|
client.wait_for_unit("multi-user.target")
|
|
|
|
with subtest("provision and seed Stalwart"):
|
|
alice = client.succeed("mkpasswd -m sha-512 alicepass").strip()
|
|
carol = client.succeed("mkpasswd -m bcrypt carolpass").strip()
|
|
client.succeed(f"python3 ${./provision.py} '{alice}' '{carol}'")
|
|
fixture("seed")
|
|
|
|
with subtest("1. inventory"):
|
|
client.succeed("mkdir -p /root/mig && cp ${mailmigToml} /root/mig/mailmig.toml")
|
|
print(mig("inventory --raw raw.json"))
|
|
plan = client.succeed("cat /root/mig/plan.toml")
|
|
print(plan)
|
|
secrets = json.loads(client.succeed("cat /root/mig/secrets.json"))["accounts"]
|
|
assert secrets["alice@b4l.test"]["hash"] == alice, secrets["alice@b4l.test"]
|
|
assert secrets["carol@b4l.test"]["source_scheme"] == "BLF-CRYPT"
|
|
assert secrets["dave@poy.test"]["hash"] is None
|
|
assert client.succeed("stat -c %a /root/mig/secrets.json").strip() == "600"
|
|
mig("inventory", ok=False) # refuses to clobber a reviewed plan
|
|
|
|
with subtest("2. users"):
|
|
print(mig("users --generate-missing"))
|
|
client.succeed(
|
|
"diff -u <(grep -v '^ *#' ${./expected-mail-accounts.nix}) <(grep -v '^ *#' /root/mig/out/mail-accounts.nix)"
|
|
)
|
|
client.succeed("diff -u ${./expected-dovecot-master-user.nix} /root/mig/out/dovecot-master-user.nix")
|
|
for login in ["alice@b4l.test", "bob@b4l.test", "carol@b4l.test", "dave@poy.test"]:
|
|
h = client.succeed(f"cat '/root/mig/out/hashes/{login}.hash'").strip()
|
|
new.succeed(f"printf '%s\\n' '{h}' > '/var/lib/mailmig-hashes/{login}.hash'")
|
|
new.succeed("systemctl restart dovecot.service")
|
|
new.wait_for_open_port(993)
|
|
# Users keep their Stalwart passwords; dave gets the generated one.
|
|
fixture("login-new alice@b4l.test alicepass")
|
|
fixture("login-new bob@b4l.test bobpass")
|
|
fixture("login-new carol@b4l.test carolpass")
|
|
dave_pw = client.succeed("grep ^dave@poy.test: /root/mig/out/generated-passwords.txt | cut -d: -f2-").strip()
|
|
fixture(f"login-new dave@poy.test '{dave_pw}'")
|
|
client.fail("python3 ${./imap_fixture.py} login-new alice@b4l.test wrongpass")
|
|
print(client.succeed("cat /root/mig/out/clan-vars-set.sh"))
|
|
|
|
with subtest("3. check"):
|
|
print(mig("check"))
|
|
|
|
with subtest("4. sync"):
|
|
print(mig("sync --dry-run --report dry.json"))
|
|
assert total(report("dry.json"), "copied") == 0
|
|
mig("verify --report v0.json", ok=False) # missing mail -> exit 1
|
|
print(mig("sync --report s1.json"))
|
|
s1 = report("s1.json")
|
|
assert all(a["ok"] for a in s1), s1
|
|
# 6 in alice's INBOX (\Deleted skipped) + 2 sent + 3 folders + bob 2 + carol 1 + dave 1
|
|
assert total(s1, "copied") == 15, total(s1, "copied")
|
|
assert total(s1, "skipped_deleted") == 1
|
|
print(mig("verify --report v1.json"))
|
|
|
|
box = json.loads(fixture("inspect alice@b4l.test alicepass"))
|
|
print(json.dumps(box, indent=1))
|
|
f = box["folders"]
|
|
for name in ["INBOX", "Sent", "Junk", "Drafts", "Projects.2024", "Projects.v1_2", "Caf&AOk-"]:
|
|
assert name in box["boxes"], (name, box["boxes"])
|
|
# Stalwart's special-use folders land on SNM's names.
|
|
mapped = {fr["src"]: fr["dst"] for a in s1 for fr in a["folders"] if a["login"] == "alice@b4l.test"}
|
|
print("folder mapping:", mapped)
|
|
assert mapped["Sent Items"] == "Sent"
|
|
for src, dst in [("Deleted Items", "Trash"), ("Junk Mail", "Junk")]:
|
|
if src in mapped:
|
|
assert mapped[src] == dst, (src, mapped[src])
|
|
inbox = {m["mid"]: m for m in f["INBOX"]}
|
|
assert len(f["INBOX"]) == 6, f["INBOX"]
|
|
assert "del@test" not in inbox
|
|
assert [m["mid"] for m in f["INBOX"]].count("dup@test") == 2
|
|
assert None in inbox # message without Message-ID
|
|
assert inbox["m1@test"]["flags"] == ["\\Flagged", "\\Seen"], inbox["m1@test"]
|
|
assert inbox["m1@test"]["date"] == "01-Jan-2020 10:00:00 +0000", inbox["m1@test"]
|
|
assert "$Forwarded" in inbox["m2@test"]["flags"]
|
|
assert inbox["big@test"]["size"] > 3000000
|
|
assert [m["mid"] for m in f["Sent"]] == ["s1@test", "s2@test"]
|
|
assert [m["mid"] for m in f["Caf&AOk-"]] == ["c1@test"]
|
|
dave = json.loads(fixture(f"inspect dave@poy.test '{dave_pw}'"))
|
|
assert [m["mid"] for m in dave["folders"]["INBOX"]] == ["dave1@test"]
|
|
|
|
with subtest("5. idempotent, delta and flag passes"):
|
|
print(mig("sync --report s2.json"))
|
|
assert total(report("s2.json"), "copied") == 0
|
|
fixture("late")
|
|
print(mig("sync --sync-flags --report s3.json"))
|
|
s3 = report("s3.json")
|
|
assert total(s3, "copied") == 1, s3
|
|
assert total(s3, "flags_updated") == 1, s3
|
|
inbox = {m["mid"]: m for m in json.loads(fixture("inspect alice@b4l.test alicepass"))["folders"]["INBOX"]}
|
|
assert "late@test" in inbox
|
|
assert "\\Seen" in inbox["m2@test"]["flags"]
|
|
print(mig("verify --report v2.json"))
|
|
|
|
with subtest("6. dns zone from SNM's DKIM keys"):
|
|
client.succeed("mkdir -p /root/mig/dkim")
|
|
for d in ["b4l.test", "poy.test"]:
|
|
key = new.succeed(f"cat /var/dkim/{d}.mail.txt")
|
|
client.succeed(f"cat > /root/mig/dkim/{d}.mail.txt <<'EOF'\n{key}\nEOF")
|
|
zone = mig("dns zone")
|
|
print(zone)
|
|
assert "@ IN MX 10 new.test." in zone
|
|
assert "mail._domainkey IN TXT" in zone and "v=DKIM1" in zone
|
|
assert "_imaps._tcp IN SRV 0 1 993 new.test." in zone
|
|
'';
|
|
}
|