B4L/mailmig
8
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mailmig: Stalwart -> SNM IMAP migration tool (initial)

Browse Source
This commit is contained in:
grabowski committed 2026-10-23 10:10:57 +07:00
commit be16176b6e
24 files changed
+7804
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/e2e.nix
+310
View File
@@ -0,0 +1,310 @@
# End-to-end test: Stalwart 0.15 (old) -> Simple NixOS Mailserver (new),
# driven by mailmig on a third machine. Exercises: REST inventory, password
# hash carry-over (sha512-crypt, bcrypt, plaintext, none), the generated Nix
# (imported by the SNM node), Stalwart master-user login over STARTTLS,
# Dovecot master-user login over implicit TLS, folder mapping, flags/dates,
# de-duplication, idempotence, delta and flag passes, and DKIM -> zone output.
{ mailmig, nixos-mailserver }:
{ pkgs, lib, ... }:
let
certs = pkgs.runCommand "mailmig-test-certs" { nativeBuildInputs = [ pkgs.openssl ]; } ''
mkdir $out && cd $out
openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \
-keyout ca.key -out ca.crt -days 3650 -subj "/CN=mailmig test CA"
for h in old new; do
openssl req -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \
-keyout $h.key -out $h.csr -subj "/CN=$h.test"
printf 'subjectAltName=DNS:%s.test\nextendedKeyUsage=serverAuth\nkeyUsage=digitalSignature\nbasicConstraints=CA:FALSE\n' $h > $h.ext
openssl x509 -req -in $h.csr -CA ca.crt -CAkey ca.key -CAcreateserial \
-out $h.crt -days 3650 -extfile $h.ext
done
chmod 644 *
'';
masterUsers = pkgs.runCommand "dovecot-master-users" { nativeBuildInputs = [ pkgs.mkpasswd ]; } ''
printf 'mailmig:{CRYPT}%s\n' "$(mkpasswd -m sha-512 masterpass)" > $out
'';
migrated = import ./expected-mail-accounts.nix;
hashFile = login: "/var/lib/mailmig-hashes/${login}.hash";
hosts =
nodes:
{
${nodes.old.networking.primaryIPAddress} = [ "old.test" ];
${nodes.new.networking.primaryIPAddress} = [ "new.test" ];
};
mailmigToml = pkgs.writeText "mailmig.toml" ''
[source]
host = "old.test"
port = 143
tls = "starttls"
[source.auth]
mode = "master"
user = "migrator"
password = "migratorpass"
login_format = "{user}%{master}"
[stalwart]
url = "http://old.test:8080"
user = "admin"
password = "adminpass"
[destination]
host = "new.test"
tls = "implicit"
ca_file = "${certs}/ca.crt"
[destination.auth]
mode = "master"
user = "mailmig"
password_env = "DST_MASTER"
login_format = "{user}*{master}"
[folders]
exclude = ["Shared Folders/*"]
[sync]
jobs = 2
batch_messages = 3
[dns]
provider = "zonefile"
mx = ["10 new.test"]
dmarc = "v=DMARC1; p=quarantine"
old_hosts = ["old.test"]
client_host = "new.test"
'';
in
{
name = "mailmig-e2e";
nodes.old =
{ nodes, ... }:
{
virtualisation.memorySize = 1024;
networking.hosts = hosts nodes;
services.stalwart = {
enable = true;
stateVersion = "26.05";
openFirewall = true;
settings = {
server.hostname = "old.test";
certificate.test = {
cert = "%{file:${certs}/old.crt}%";
private-key = "%{file:${certs}/old.key}%";
};
server.tls = {
certificate = "test";
enable = true;
implicit = false;
};
server.listener = {
imap = {
bind = [ "[::]:143" ];
protocol = "imap";
};
http = {
bind = [ "[::]:8080" ];
protocol = "http";
};
};
authentication.fallback-admin = {
user = "admin";
secret = "adminpass";
};
# Stalwart 0.15 master user: "<account>%migrator" + this secret.
authentication.master = {
user = "migrator";
secret = "migratorpass";
};
};
};
};
nodes.new =
{ nodes, ... }:
{
imports = [
nixos-mailserver.nixosModules.default
./expected-dovecot-master-user.nix
];
virtualisation.memorySize = 1536;
virtualisation.cores = 2;
networking.hosts = hosts nodes;
mailserver = {
enable = true;
stateVersion = 3;
fqdn = "new.test";
inherit (migrated) domains forwards;
localDnsResolver = false;
x509 = {
certificateFile = "${certs}/new.crt";
privateKeyFile = "${certs}/new.key";
};
# Same mapping the clan module uses, with plain files instead of clan vars.
accounts = lib.mapAttrs (login: a: {
inherit (a) aliases;
quota = a.quota or null;
hashedPasswordFile = hashFile login;
}) migrated.accounts;
};
# Placeholder hashes ("!" never verifies) until mailmig's are copied in.
systemd.tmpfiles.settings."10-mailmig" =
lib.mapAttrs' (login: _: lib.nameValuePair (hashFile login) { f.argument = "!"; }) migrated.accounts
// {
"/var/lib/dovecot-master-users"."L+".argument = "${masterUsers}";
};
services.rspamd.locals."options.inc".text = ''
dns {
nameservers = ["127.0.0.1"];
timeout = 0.0s;
retransmits = 0;
}
'';
services.rspamd.overrides."options.inc".enable = false;
services.postfix-tlspol.configurePostfix = lib.mkForce false;
};
nodes.client =
{ nodes, ... }:
{
networking.hosts = hosts nodes;
security.pki.certificateFiles = [ "${certs}/ca.crt" ];
environment.systemPackages = [
mailmig
pkgs.python3
pkgs.mkpasswd
pkgs.jq
pkgs.diffutils
];
};
testScript = ''
import json
def mig(args, ok=True):
cmd = f"cd /root/mig && DST_MASTER=masterpass mailmig {args} 2>&1"
return client.succeed(cmd) if ok else client.fail(cmd)
def fixture(args):
return client.succeed(f"python3 ${./imap_fixture.py} {args}")
def report(path):
return json.loads(client.succeed(f"cat /root/mig/{path}"))
def total(rep, key):
return sum(f[key] for a in rep for f in a["folders"])
start_all()
old.wait_for_unit("stalwart.service")
old.wait_for_open_port(8080)
old.wait_for_open_port(143)
new.wait_for_unit("dovecot.service")
new.wait_for_open_port(993)
client.wait_for_unit("multi-user.target")
with subtest("provision and seed Stalwart"):
alice = client.succeed("mkpasswd -m sha-512 alicepass").strip()
carol = client.succeed("mkpasswd -m bcrypt carolpass").strip()
client.succeed(f"python3 ${./provision.py} '{alice}' '{carol}'")
fixture("seed")
with subtest("1. inventory"):
client.succeed("mkdir -p /root/mig && cp ${mailmigToml} /root/mig/mailmig.toml")
print(mig("inventory --raw raw.json"))
plan = client.succeed("cat /root/mig/plan.toml")
print(plan)
secrets = json.loads(client.succeed("cat /root/mig/secrets.json"))["accounts"]
assert secrets["alice@b4l.test"]["hash"] == alice, secrets["alice@b4l.test"]
assert secrets["carol@b4l.test"]["source_scheme"] == "BLF-CRYPT"
assert secrets["dave@poy.test"]["hash"] is None
assert client.succeed("stat -c %a /root/mig/secrets.json").strip() == "600"
mig("inventory", ok=False) # refuses to clobber a reviewed plan
with subtest("2. users"):
print(mig("users --generate-missing"))
client.succeed(
"diff -u <(grep -v '^ *#' ${./expected-mail-accounts.nix}) <(grep -v '^ *#' /root/mig/out/mail-accounts.nix)"
)
client.succeed("diff -u ${./expected-dovecot-master-user.nix} /root/mig/out/dovecot-master-user.nix")
for login in ["alice@b4l.test", "bob@b4l.test", "carol@b4l.test", "dave@poy.test"]:
h = client.succeed(f"cat '/root/mig/out/hashes/{login}.hash'").strip()
new.succeed(f"printf '%s\\n' '{h}' > '/var/lib/mailmig-hashes/{login}.hash'")
new.succeed("systemctl restart dovecot.service")
new.wait_for_open_port(993)
# Users keep their Stalwart passwords; dave gets the generated one.
fixture("login-new alice@b4l.test alicepass")
fixture("login-new bob@b4l.test bobpass")
fixture("login-new carol@b4l.test carolpass")
dave_pw = client.succeed("grep ^dave@poy.test: /root/mig/out/generated-passwords.txt | cut -d: -f2-").strip()
fixture(f"login-new dave@poy.test '{dave_pw}'")
client.fail("python3 ${./imap_fixture.py} login-new alice@b4l.test wrongpass")
print(client.succeed("cat /root/mig/out/clan-vars-set.sh"))
with subtest("3. check"):
print(mig("check"))
with subtest("4. sync"):
print(mig("sync --dry-run --report dry.json"))
assert total(report("dry.json"), "copied") == 0
mig("verify --report v0.json", ok=False) # missing mail -> exit 1
print(mig("sync --report s1.json"))
s1 = report("s1.json")
assert all(a["ok"] for a in s1), s1
# 6 in alice's INBOX (\Deleted skipped) + 2 sent + 3 folders + bob 2 + carol 1 + dave 1
assert total(s1, "copied") == 15, total(s1, "copied")
assert total(s1, "skipped_deleted") == 1
print(mig("verify --report v1.json"))
box = json.loads(fixture("inspect alice@b4l.test alicepass"))
print(json.dumps(box, indent=1))
f = box["folders"]
for name in ["INBOX", "Sent", "Junk", "Drafts", "Projects.2024", "Projects.v1_2", "Caf&AOk-"]:
assert name in box["boxes"], (name, box["boxes"])
# Stalwart's special-use folders land on SNM's names.
mapped = {fr["src"]: fr["dst"] for a in s1 for fr in a["folders"] if a["login"] == "alice@b4l.test"}
print("folder mapping:", mapped)
assert mapped["Sent Items"] == "Sent"
for src, dst in [("Deleted Items", "Trash"), ("Junk Mail", "Junk")]:
if src in mapped:
assert mapped[src] == dst, (src, mapped[src])
inbox = {m["mid"]: m for m in f["INBOX"]}
assert len(f["INBOX"]) == 6, f["INBOX"]
assert "del@test" not in inbox
assert [m["mid"] for m in f["INBOX"]].count("dup@test") == 2
assert None in inbox # message without Message-ID
assert inbox["m1@test"]["flags"] == ["\\Flagged", "\\Seen"], inbox["m1@test"]
assert inbox["m1@test"]["date"] == "01-Jan-2020 10:00:00 +0000", inbox["m1@test"]
assert "$Forwarded" in inbox["m2@test"]["flags"]
assert inbox["big@test"]["size"] > 3000000
assert [m["mid"] for m in f["Sent"]] == ["s1@test", "s2@test"]
assert [m["mid"] for m in f["Caf&AOk-"]] == ["c1@test"]
dave = json.loads(fixture(f"inspect dave@poy.test '{dave_pw}'"))
assert [m["mid"] for m in dave["folders"]["INBOX"]] == ["dave1@test"]
with subtest("5. idempotent, delta and flag passes"):
print(mig("sync --report s2.json"))
assert total(report("s2.json"), "copied") == 0
fixture("late")
print(mig("sync --sync-flags --report s3.json"))
s3 = report("s3.json")
assert total(s3, "copied") == 1, s3
assert total(s3, "flags_updated") == 1, s3
inbox = {m["mid"]: m for m in json.loads(fixture("inspect alice@b4l.test alicepass"))["folders"]["INBOX"]}
assert "late@test" in inbox
assert "\\Seen" in inbox["m2@test"]["flags"]
print(mig("verify --report v2.json"))
with subtest("6. dns zone from SNM's DKIM keys"):
client.succeed("mkdir -p /root/mig/dkim")
for d in ["b4l.test", "poy.test"]:
key = new.succeed(f"cat /var/dkim/{d}.mail.txt")
client.succeed(f"cat > /root/mig/dkim/{d}.mail.txt <<'EOF'\n{key}\nEOF")
zone = mig("dns zone")
print(zone)
assert "@ IN MX 10 new.test." in zone
assert "mail._domainkey IN TXT" in zone and "v=DKIM1" in zone
assert "_imaps._tcp IN SRV 0 1 993 new.test." in zone
'';
}