mailmig: Stalwart -> SNM IMAP migration tool (initial)

tests/e2e.nix

@@ -0,0 +1,310 @@
+# End-to-end test: Stalwart 0.15 (old) -> Simple NixOS Mailserver (new),
+# driven by mailmig on a third machine. Exercises: REST inventory, password
+# hash carry-over (sha512-crypt, bcrypt, plaintext, none), the generated Nix
+# (imported by the SNM node), Stalwart master-user login over STARTTLS,
+# Dovecot master-user login over implicit TLS, folder mapping, flags/dates,
+# de-duplication, idempotence, delta and flag passes, and DKIM -> zone output.
+{ mailmig, nixos-mailserver }:
+{ pkgs, lib, ... }:
+let
+  certs = pkgs.runCommand "mailmig-test-certs" { nativeBuildInputs = [ pkgs.openssl ]; } ''
+    mkdir $out && cd $out
+    openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \
+      -keyout ca.key -out ca.crt -days 3650 -subj "/CN=mailmig test CA"
+    for h in old new; do
+      openssl req -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \
+        -keyout $h.key -out $h.csr -subj "/CN=$h.test"
+      printf 'subjectAltName=DNS:%s.test\nextendedKeyUsage=serverAuth\nkeyUsage=digitalSignature\nbasicConstraints=CA:FALSE\n' $h > $h.ext
+      openssl x509 -req -in $h.csr -CA ca.crt -CAkey ca.key -CAcreateserial \
+        -out $h.crt -days 3650 -extfile $h.ext
+    done
+    chmod 644 *
+  '';
+
+  masterUsers = pkgs.runCommand "dovecot-master-users" { nativeBuildInputs = [ pkgs.mkpasswd ]; } ''
+    printf 'mailmig:{CRYPT}%s\n' "$(mkpasswd -m sha-512 masterpass)" > $out
+  '';
+
+  migrated = import ./expected-mail-accounts.nix;
+  hashFile = login: "/var/lib/mailmig-hashes/${login}.hash";
+
+  hosts =
+    nodes:
+    {
+      ${nodes.old.networking.primaryIPAddress} = [ "old.test" ];
+      ${nodes.new.networking.primaryIPAddress} = [ "new.test" ];
+    };
+
+  mailmigToml = pkgs.writeText "mailmig.toml" ''
+    [source]
+    host = "old.test"
+    port = 143
+    tls = "starttls"
+    [source.auth]
+    mode = "master"
+    user = "migrator"
+    password = "migratorpass"
+    login_format = "{user}%{master}"
+
+    [stalwart]
+    url = "http://old.test:8080"
+    user = "admin"
+    password = "adminpass"
+
+    [destination]
+    host = "new.test"
+    tls = "implicit"
+    ca_file = "${certs}/ca.crt"
+    [destination.auth]
+    mode = "master"
+    user = "mailmig"
+    password_env = "DST_MASTER"
+    login_format = "{user}*{master}"
+
+    [folders]
+    exclude = ["Shared Folders/*"]
+
+    [sync]
+    jobs = 2
+    batch_messages = 3
+
+    [dns]
+    provider = "zonefile"
+    mx = ["10 new.test"]
+    dmarc = "v=DMARC1; p=quarantine"
+    old_hosts = ["old.test"]
+    client_host = "new.test"
+  '';
+in
+{
+  name = "mailmig-e2e";
+
+  nodes.old =
+    { nodes, ... }:
+    {
+      virtualisation.memorySize = 1024;
+      networking.hosts = hosts nodes;
+      services.stalwart = {
+        enable = true;
+        stateVersion = "26.05";
+        openFirewall = true;
+        settings = {
+          server.hostname = "old.test";
+          certificate.test = {
+            cert = "%{file:${certs}/old.crt}%";
+            private-key = "%{file:${certs}/old.key}%";
+          };
+          server.tls = {
+            certificate = "test";
+            enable = true;
+            implicit = false;
+          };
+          server.listener = {
+            imap = {
+              bind = [ "[::]:143" ];
+              protocol = "imap";
+            };
+            http = {
+              bind = [ "[::]:8080" ];
+              protocol = "http";
+            };
+          };
+          authentication.fallback-admin = {
+            user = "admin";
+            secret = "adminpass";
+          };
+          # Stalwart 0.15 master user: "<account>%migrator" + this secret.
+          authentication.master = {
+            user = "migrator";
+            secret = "migratorpass";
+          };
+        };
+      };
+    };
+
+  nodes.new =
+    { nodes, ... }:
+    {
+      imports = [
+        nixos-mailserver.nixosModules.default
+        ./expected-dovecot-master-user.nix
+      ];
+      virtualisation.memorySize = 1536;
+      virtualisation.cores = 2;
+      networking.hosts = hosts nodes;
+      mailserver = {
+        enable = true;
+        stateVersion = 3;
+        fqdn = "new.test";
+        inherit (migrated) domains forwards;
+        localDnsResolver = false;
+        x509 = {
+          certificateFile = "${certs}/new.crt";
+          privateKeyFile = "${certs}/new.key";
+        };
+        # Same mapping the clan module uses, with plain files instead of clan vars.
+        accounts = lib.mapAttrs (login: a: {
+          inherit (a) aliases;
+          quota = a.quota or null;
+          hashedPasswordFile = hashFile login;
+        }) migrated.accounts;
+      };
+      # Placeholder hashes ("!" never verifies) until mailmig's are copied in.
+      systemd.tmpfiles.settings."10-mailmig" =
+        lib.mapAttrs' (login: _: lib.nameValuePair (hashFile login) { f.argument = "!"; }) migrated.accounts
+        // {
+          "/var/lib/dovecot-master-users"."L+".argument = "${masterUsers}";
+        };
+      services.rspamd.locals."options.inc".text = ''
+        dns {
+          nameservers = ["127.0.0.1"];
+          timeout = 0.0s;
+          retransmits = 0;
+        }
+      '';
+      services.rspamd.overrides."options.inc".enable = false;
+      services.postfix-tlspol.configurePostfix = lib.mkForce false;
+    };
+
+  nodes.client =
+    { nodes, ... }:
+    {
+      networking.hosts = hosts nodes;
+      security.pki.certificateFiles = [ "${certs}/ca.crt" ];
+      environment.systemPackages = [
+        mailmig
+        pkgs.python3
+        pkgs.mkpasswd
+        pkgs.jq
+        pkgs.diffutils
+      ];
+    };
+
+  testScript = ''
+    import json
+
+    def mig(args, ok=True):
+        cmd = f"cd /root/mig && DST_MASTER=masterpass mailmig {args} 2>&1"
+        return client.succeed(cmd) if ok else client.fail(cmd)
+
+    def fixture(args):
+        return client.succeed(f"python3 ${./imap_fixture.py} {args}")
+
+    def report(path):
+        return json.loads(client.succeed(f"cat /root/mig/{path}"))
+
+    def total(rep, key):
+        return sum(f[key] for a in rep for f in a["folders"])
+
+    start_all()
+    old.wait_for_unit("stalwart.service")
+    old.wait_for_open_port(8080)
+    old.wait_for_open_port(143)
+    new.wait_for_unit("dovecot.service")
+    new.wait_for_open_port(993)
+    client.wait_for_unit("multi-user.target")
+
+    with subtest("provision and seed Stalwart"):
+        alice = client.succeed("mkpasswd -m sha-512 alicepass").strip()
+        carol = client.succeed("mkpasswd -m bcrypt carolpass").strip()
+        client.succeed(f"python3 ${./provision.py} '{alice}' '{carol}'")
+        fixture("seed")
+
+    with subtest("1. inventory"):
+        client.succeed("mkdir -p /root/mig && cp ${mailmigToml} /root/mig/mailmig.toml")
+        print(mig("inventory --raw raw.json"))
+        plan = client.succeed("cat /root/mig/plan.toml")
+        print(plan)
+        secrets = json.loads(client.succeed("cat /root/mig/secrets.json"))["accounts"]
+        assert secrets["alice@b4l.test"]["hash"] == alice, secrets["alice@b4l.test"]
+        assert secrets["carol@b4l.test"]["source_scheme"] == "BLF-CRYPT"
+        assert secrets["dave@poy.test"]["hash"] is None
+        assert client.succeed("stat -c %a /root/mig/secrets.json").strip() == "600"
+        mig("inventory", ok=False)  # refuses to clobber a reviewed plan
+
+    with subtest("2. users"):
+        print(mig("users --generate-missing"))
+        client.succeed(
+            "diff -u <(grep -v '^ *#' ${./expected-mail-accounts.nix}) <(grep -v '^ *#' /root/mig/out/mail-accounts.nix)"
+        )
+        client.succeed("diff -u ${./expected-dovecot-master-user.nix} /root/mig/out/dovecot-master-user.nix")
+        for login in ["alice@b4l.test", "bob@b4l.test", "carol@b4l.test", "dave@poy.test"]:
+            h = client.succeed(f"cat '/root/mig/out/hashes/{login}.hash'").strip()
+            new.succeed(f"printf '%s\\n' '{h}' > '/var/lib/mailmig-hashes/{login}.hash'")
+        new.succeed("systemctl restart dovecot.service")
+        new.wait_for_open_port(993)
+        # Users keep their Stalwart passwords; dave gets the generated one.
+        fixture("login-new alice@b4l.test alicepass")
+        fixture("login-new bob@b4l.test bobpass")
+        fixture("login-new carol@b4l.test carolpass")
+        dave_pw = client.succeed("grep ^dave@poy.test: /root/mig/out/generated-passwords.txt | cut -d: -f2-").strip()
+        fixture(f"login-new dave@poy.test '{dave_pw}'")
+        client.fail("python3 ${./imap_fixture.py} login-new alice@b4l.test wrongpass")
+        print(client.succeed("cat /root/mig/out/clan-vars-set.sh"))
+
+    with subtest("3. check"):
+        print(mig("check"))
+
+    with subtest("4. sync"):
+        print(mig("sync --dry-run --report dry.json"))
+        assert total(report("dry.json"), "copied") == 0
+        mig("verify --report v0.json", ok=False)  # missing mail -> exit 1
+        print(mig("sync --report s1.json"))
+        s1 = report("s1.json")
+        assert all(a["ok"] for a in s1), s1
+        # 6 in alice's INBOX (\Deleted skipped) + 2 sent + 3 folders + bob 2 + carol 1 + dave 1
+        assert total(s1, "copied") == 15, total(s1, "copied")
+        assert total(s1, "skipped_deleted") == 1
+        print(mig("verify --report v1.json"))
+
+        box = json.loads(fixture("inspect alice@b4l.test alicepass"))
+        print(json.dumps(box, indent=1))
+        f = box["folders"]
+        for name in ["INBOX", "Sent", "Junk", "Drafts", "Projects.2024", "Projects.v1_2", "Caf&AOk-"]:
+            assert name in box["boxes"], (name, box["boxes"])
+        # Stalwart's special-use folders land on SNM's names.
+        mapped = {fr["src"]: fr["dst"] for a in s1 for fr in a["folders"] if a["login"] == "alice@b4l.test"}
+        print("folder mapping:", mapped)
+        assert mapped["Sent Items"] == "Sent"
+        for src, dst in [("Deleted Items", "Trash"), ("Junk Mail", "Junk")]:
+            if src in mapped:
+                assert mapped[src] == dst, (src, mapped[src])
+        inbox = {m["mid"]: m for m in f["INBOX"]}
+        assert len(f["INBOX"]) == 6, f["INBOX"]
+        assert "del@test" not in inbox
+        assert [m["mid"] for m in f["INBOX"]].count("dup@test") == 2
+        assert None in inbox  # message without Message-ID
+        assert inbox["m1@test"]["flags"] == ["\\Flagged", "\\Seen"], inbox["m1@test"]
+        assert inbox["m1@test"]["date"] == "01-Jan-2020 10:00:00 +0000", inbox["m1@test"]
+        assert "$Forwarded" in inbox["m2@test"]["flags"]
+        assert inbox["big@test"]["size"] > 3000000
+        assert [m["mid"] for m in f["Sent"]] == ["s1@test", "s2@test"]
+        assert [m["mid"] for m in f["Caf&AOk-"]] == ["c1@test"]
+        dave = json.loads(fixture(f"inspect dave@poy.test '{dave_pw}'"))
+        assert [m["mid"] for m in dave["folders"]["INBOX"]] == ["dave1@test"]
+
+    with subtest("5. idempotent, delta and flag passes"):
+        print(mig("sync --report s2.json"))
+        assert total(report("s2.json"), "copied") == 0
+        fixture("late")
+        print(mig("sync --sync-flags --report s3.json"))
+        s3 = report("s3.json")
+        assert total(s3, "copied") == 1, s3
+        assert total(s3, "flags_updated") == 1, s3
+        inbox = {m["mid"]: m for m in json.loads(fixture("inspect alice@b4l.test alicepass"))["folders"]["INBOX"]}
+        assert "late@test" in inbox
+        assert "\\Seen" in inbox["m2@test"]["flags"]
+        print(mig("verify --report v2.json"))
+
+    with subtest("6. dns zone from SNM's DKIM keys"):
+        client.succeed("mkdir -p /root/mig/dkim")
+        for d in ["b4l.test", "poy.test"]:
+            key = new.succeed(f"cat /var/dkim/{d}.mail.txt")
+            client.succeed(f"cat > /root/mig/dkim/{d}.mail.txt <<'EOF'\n{key}\nEOF")
+        zone = mig("dns zone")
+        print(zone)
+        assert "@                        IN  MX 10 new.test." in zone
+        assert "mail._domainkey          IN  TXT" in zone and "v=DKIM1" in zone
+        assert "_imaps._tcp              IN  SRV 0 1 993 new.test." in zone
+  '';
+}

tests/expected-dovecot-master-user.nix

@@ -0,0 +1,20 @@
+# TEMPORARY: Dovecot master user for the mail migration (mailmig).
+# Lets "<account>*mailmig" log in to any mailbox with the master password.
+# Import this module on the SNM host only while migrating, then remove it.
+#
+# Create the passwd file on the host (root, 0640 root:dovecot2):
+#   printf 'mailmig:{CRYPT}%s\n' "$(mkpasswd -m sha-512)" > /var/lib/dovecot-master-users
+{ ... }:
+{
+  services.dovecot2.settings = {
+    auth_master_user_separator = "*";
+    "passdb mailmig-master" = {
+      driver = "passwd-file";
+      passwd_file_path = "/var/lib/dovecot-master-users";
+      master = true;
+      # After the master password checks out, still require the target
+      # account to exist in SNM's own passdb.
+      result_success = "continue";
+    };
+  };
+}

tests/expected-mail-accounts.nix

@@ -0,0 +1,35 @@
+# Golden output of `mailmig users` for the e2e fixture (comment lines are
+# ignored by the comparison). The `new` test node also imports this file, so
+# the test proves the generated Nix evaluates and works with SNM.
+{
+  domains = [
+    "b4l.test"
+    "poy.test"
+  ];
+
+  accounts = {
+    "alice@b4l.test" = {
+      aliases = [
+        "a.lias@poy.test"
+      ];
+      quota = "2G";
+    };
+    "bob@b4l.test" = {
+      aliases = [ ];
+    };
+    "carol@b4l.test" = {
+      aliases = [ ];
+    };
+    "dave@poy.test" = {
+      aliases = [ ];
+    };
+  };
+
+  forwards = {
+    "team@b4l.test" = [
+      "alice@b4l.test"
+      "bob@b4l.test"
+      "ext@example.org"
+    ];
+  };
+}

tests/imap_fixture.py

@@ -0,0 +1,132 @@
+"""IMAP helpers for the e2e test: seed the old server, inspect the new one."""
+import imaplib
+import json
+import re
+import ssl
+import sys
+
+CTX = ssl.create_default_context()  # trusts the test CA via security.pki
+
+
+def old(user, password):
+    c = imaplib.IMAP4("old.test", 143)
+    c.starttls(ssl_context=CTX)
+    c.login(user, password)
+    return c
+
+
+def new(user, password):
+    c = imaplib.IMAP4_SSL("new.test", 993, ssl_context=CTX)
+    c.login(user, password)
+    return c
+
+
+def msg(mid, subject, body="hello\r\n", extra=""):
+    hdr = f"From: Sender <sender@example.org>\r\nTo: x@b4l.test\r\nSubject: {subject}\r\nDate: Wed, 01 Jan 2020 10:00:00 +0000\r\n"
+    if mid:
+        hdr += f"Message-ID: <{mid}>\r\n"
+    return (hdr + extra + "\r\n" + body).encode()
+
+
+def ensure(c, box):
+    typ, _ = c.create(box)
+    print("create", box, typ)
+
+
+def append(c, box, data, flags="", date=None):
+    typ, resp = c.append(box, flags or None, date, data)
+    assert typ == "OK", (box, typ, resp)
+
+
+def seed():
+    a = old("alice", "alicepass")
+    append(a, "INBOX", msg("m1@test", "seen and flagged"), r"(\Seen \Flagged)", '"01-Jan-2020 10:00:00 +0000"')
+    append(a, "INBOX", msg("m2@test", "forwarded"), "($Forwarded)")
+    append(a, "INBOX", msg(None, "no message id"))
+    append(a, "INBOX", msg("dup@test", "duplicate"))
+    append(a, "INBOX", msg("dup@test", "duplicate"))
+    append(a, "INBOX", msg("del@test", "deleted"), r"(\Deleted)")
+    big = "\r\n".join(["x" * 76] * 40000) + "\r\n"  # ~3 MB
+    append(a, "INBOX", msg("big@test", "big", body=big))
+    ensure(a, '"Sent Items"')
+    append(a, '"Sent Items"', msg("s1@test", "sent 1"), r"(\Seen)")
+    append(a, '"Sent Items"', msg("s2@test", "sent 2"), r"(\Seen)")
+    ensure(a, "Projects/2024")
+    append(a, "Projects/2024", msg("p1@test", "project"))
+    ensure(a, "Projects/v1.2")
+    append(a, "Projects/v1.2", msg("p2@test", "dotted"))
+    ensure(a, "Caf&AOk-")
+    append(a, "Caf&AOk-", msg("c1@test", "unicode folder"))
+    print(a.list())
+    a.logout()
+
+    b = old("bob@b4l.test", "bobpass")
+    append(b, "INBOX", msg("b1@test", "bob 1"))
+    append(b, "INBOX", msg("b2@test", "bob 2"), r"(\Seen)")
+    b.logout()
+
+    c = old("carol", "carolpass")
+    append(c, "INBOX", msg("carol1@test", "carol"))
+    c.logout()
+
+    d = old("dave%migrator", "migratorpass")
+    append(d, "INBOX", msg("dave1@test", "dave"))
+    d.logout()
+
+
+def late():
+    """Delta pass: a new message arrives and an old one gets read."""
+    a = old("alice", "alicepass")
+    append(a, "INBOX", msg("late@test", "arrived after the bulk copy"))
+    a.select("INBOX")
+    typ, data = a.search(None, "HEADER", "Message-ID", "<m2@test>")
+    a.store(data[0].split()[0], "+FLAGS", r"(\Seen)")
+    a.logout()
+
+
+def inspect(user, password):
+    """Print a JSON summary of a mailbox on the new server."""
+    c = new(user, password)
+    typ, lst = c.list()
+    boxes = []
+    for line in lst:
+        m = re.match(rb'\((.*?)\) "(.*?)" "?(.*?)"?$', line)
+        boxes.append(m.group(3).decode())
+    out = {"boxes": boxes, "folders": {}}
+    for b in boxes:
+        typ, _ = c.select(f'"{b}"', readonly=True)
+        if typ != "OK":
+            continue
+        typ, data = c.fetch("1:*", "(FLAGS INTERNALDATE RFC822.SIZE BODY.PEEK[HEADER.FIELDS (MESSAGE-ID SUBJECT)])")
+        msgs = []
+        for item in data:
+            if not isinstance(item, tuple):
+                continue
+            meta, hdr = item
+            flags = re.search(rb"FLAGS \((.*?)\)", meta).group(1).decode().split()
+            date = re.search(rb'INTERNALDATE "(.*?)"', meta).group(1).decode()
+            size = int(re.search(rb"RFC822.SIZE (\d+)", meta).group(1))
+            mid = re.search(rb"Message-ID: <(.*?)>", hdr, re.I)
+            msgs.append({"mid": mid.group(1).decode() if mid else None, "flags": sorted(flags), "date": date, "size": size})
+        out["folders"][b] = msgs
+    c.logout()
+    print(json.dumps(out, indent=1))
+
+
+def login_new(user, password):
+    new(user, password).logout()
+    print("login ok", user)
+
+
+if __name__ == "__main__":
+    cmd = sys.argv[1]
+    if cmd == "seed":
+        seed()
+    elif cmd == "late":
+        late()
+    elif cmd == "inspect":
+        inspect(sys.argv[2], sys.argv[3])
+    elif cmd == "login-new":
+        login_new(sys.argv[2], sys.argv[3])
+    else:
+        raise SystemExit(f"unknown command {cmd}")

tests/provision.py

@@ -0,0 +1,48 @@
+"""Create the fixture directory on the old (Stalwart 0.15) server via its REST API."""
+import base64
+import json
+import sys
+import urllib.request
+
+API = "http://old.test:8080"
+AUTH = "Basic " + base64.b64encode(b"admin:adminpass").decode()
+
+alice_hash, carol_hash = sys.argv[1], sys.argv[2]
+
+
+def call(method, path, body=None):
+    req = urllib.request.Request(
+        API + path,
+        method=method,
+        data=json.dumps(body).encode() if body is not None else None,
+        headers={"Content-Type": "application/json", "Authorization": AUTH},
+    )
+    with urllib.request.urlopen(req) as r:
+        out = json.load(r)
+    if isinstance(out, dict) and "error" in out:
+        raise SystemExit(f"{method} {path}: {out}")
+    print(method, path, "->", out)
+    return out
+
+
+def principal(**kw):
+    p = {
+        "quota": 0, "description": "", "secrets": [], "emails": [], "urls": [], "memberOf": [],
+        "roles": [], "lists": [], "members": [], "enabledPermissions": [], "disabledPermissions": [],
+        "externalMembers": [],
+    }
+    p.update(kw)
+    return call("POST", "/api/principal", p)
+
+
+principal(type="domain", name="b4l.test")
+principal(type="domain", name="poy.test")
+principal(type="individual", name="alice", description="Alice Example", secrets=[alice_hash],
+          emails=["alice@b4l.test", "a.lias@poy.test"], quota=2 * 1024 ** 3, roles=["user"])
+# plaintext secret: Stalwart accepts unhashed secrets
+principal(type="individual", name="bob@b4l.test", secrets=["bobpass"], emails=["bob@b4l.test"], roles=["user"])
+principal(type="individual", name="carol", secrets=[carol_hash], emails=["carol@b4l.test"], roles=["user"])
+# no password at all: only reachable through the master user
+principal(type="individual", name="dave", emails=["dave@poy.test"], roles=["user"])
+principal(type="list", name="team", emails=["team@b4l.test"], members=["alice", "bob@b4l.test"],
+          externalMembers=["ext@example.org"])