# End-to-end test: Stalwart 0.15 (old) -> Simple NixOS Mailserver (new),
# driven by mailmig on a third machine. Exercises: REST inventory, password
# hash carry-over (sha512-crypt, bcrypt, plaintext, none), the generated Nix
# (imported by the SNM node), Stalwart master-user login over STARTTLS,
# Dovecot master-user login over implicit TLS, folder mapping, flags/dates,
# de-duplication, idempotence, delta and flag passes, and DKIM -> zone output.
{ mailmig, nixos-mailserver }:
{ pkgs, lib, ... }:
let
  # Python 3.13 verifies strictly (VERIFY_X509_STRICT): the CA needs keyUsage,
  # leaves need SKI/AKI.
  certs = pkgs.runCommand "mailmig-test-certs" { nativeBuildInputs = [ pkgs.openssl ]; } ''
    mkdir $out && cd $out
    cat > ca.cnf <<EOF
    [req]
    distinguished_name = dn
    x509_extensions = v3_ca
    prompt = no
    [dn]
    CN = mailmig test CA
    [v3_ca]
    basicConstraints = critical,CA:TRUE
    keyUsage = critical,keyCertSign,cRLSign
    subjectKeyIdentifier = hash
    EOF
    openssl req -x509 -config ca.cnf -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \
      -keyout ca.key -out ca.crt -days 3650
    for h in old new; do
      openssl req -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \
        -keyout $h.key -out $h.csr -subj "/CN=$h.test"
      cat > $h.ext <<EOF
    subjectAltName = DNS:$h.test
    extendedKeyUsage = serverAuth
    keyUsage = critical,digitalSignature
    basicConstraints = critical,CA:FALSE
    subjectKeyIdentifier = hash
    authorityKeyIdentifier = keyid
    EOF
      openssl x509 -req -in $h.csr -CA ca.crt -CAkey ca.key -CAcreateserial \
        -out $h.crt -days 3650 -extfile $h.ext
    done
    chmod 644 *
  '';

  masterUsers = pkgs.runCommand "dovecot-master-users" { nativeBuildInputs = [ pkgs.mkpasswd ]; } ''
    printf 'mailmig:{CRYPT}%s\n' "$(mkpasswd -m sha-512 masterpass)" > $out
  '';

  migrated = import ./expected-mail-accounts.nix;
  hashFile = login: "/var/lib/mailmig-hashes/${login}.hash";

  hosts =
    nodes:
    {
      ${nodes.old.networking.primaryIPAddress} = [ "old.test" ];
      ${nodes.new.networking.primaryIPAddress} = [ "new.test" ];
    };

  mailmigToml = pkgs.writeText "mailmig.toml" ''
    [source]
    host = "old.test"
    port = 143
    tls = "starttls"
    [source.auth]
    mode = "master"
    user = "migrator"
    password = "migratorpass"
    login_format = "{user}%{master}"

    [stalwart]
    url = "http://old.test:8080"
    user = "admin"
    password = "adminpass"

    [destination]
    host = "new.test"
    tls = "implicit"
    ca_file = "${certs}/ca.crt"
    [destination.auth]
    mode = "master"
    user = "mailmig"
    password_env = "DST_MASTER"
    login_format = "{user}*{master}"

    [folders]
    exclude = ["Shared Folders/*"]

    [sync]
    jobs = 2
    batch_messages = 3

    [dns]
    provider = "zonefile"
    mx = ["10 new.test"]
    dmarc = "v=DMARC1; p=quarantine"
    old_hosts = ["old.test"]
    client_host = "new.test"
  '';
in
{
  name = "mailmig-e2e";

  nodes.old =
    { nodes, ... }:
    {
      virtualisation.memorySize = 1024;
      networking.hosts = hosts nodes;
      services.stalwart = {
        enable = true;
        stateVersion = "26.05";
        openFirewall = true;
        settings = {
          server.hostname = "old.test";
          certificate.test = {
            cert = "%{file:${certs}/old.crt}%";
            private-key = "%{file:${certs}/old.key}%";
          };
          server.tls = {
            certificate = "test";
            enable = true;
            implicit = false;
          };
          server.listener = {
            imap = {
              bind = [ "[::]:143" ];
              protocol = "imap";
            };
            http = {
              bind = [ "[::]:8080" ];
              protocol = "http";
            };
          };
          authentication.fallback-admin = {
            user = "admin";
            secret = "adminpass";
          };
          # Stalwart 0.15 master user: "<account>%migrator" + this secret.
          authentication.master = {
            user = "migrator";
            secret = "migratorpass";
          };
        };
      };
    };

  nodes.new =
    { nodes, ... }:
    {
      imports = [
        nixos-mailserver.nixosModules.default
        ./expected-dovecot-master-user.nix
      ];
      virtualisation.memorySize = 1536;
      virtualisation.cores = 2;
      networking.hosts = hosts nodes;
      mailserver = {
        enable = true;
        stateVersion = 3;
        fqdn = "new.test";
        inherit (migrated) domains forwards;
        localDnsResolver = false;
        x509 = {
          certificateFile = "${certs}/new.crt";
          privateKeyFile = "${certs}/new.key";
        };
        # Same mapping the clan module uses, with plain files instead of clan vars.
        accounts = lib.mapAttrs (login: a: {
          inherit (a) aliases;
          quota = a.quota or null;
          hashedPasswordFile = hashFile login;
        }) migrated.accounts;
      };
      # Placeholder hashes ("!" never verifies) until mailmig's are copied in.
      systemd.tmpfiles.settings."10-mailmig" =
        lib.mapAttrs' (login: _: lib.nameValuePair (hashFile login) { f.argument = "!"; }) migrated.accounts
        // {
          "/var/lib/dovecot-master-users"."L+".argument = "${masterUsers}";
        };
      services.rspamd.locals."options.inc".text = ''
        dns {
          nameservers = ["127.0.0.1"];
          timeout = 0.0s;
          retransmits = 0;
        }
      '';
      services.rspamd.overrides."options.inc".enable = false;
      services.postfix-tlspol.configurePostfix = lib.mkForce false;
    };

  nodes.client =
    { nodes, ... }:
    {
      networking.hosts = hosts nodes;
      security.pki.certificateFiles = [ "${certs}/ca.crt" ];
      environment.systemPackages = [
        mailmig
        pkgs.python3
        pkgs.mkpasswd
        pkgs.jq
        pkgs.diffutils
      ];
    };

  testScript = ''
    import json

    def mig(args, ok=True):
        cmd = f"cd /root/mig && DST_MASTER=masterpass mailmig {args} 2>&1"
        return client.succeed(cmd) if ok else client.fail(cmd)

    def fixture(args):
        return client.succeed(f"python3 ${./imap_fixture.py} {args}")

    def report(path):
        return json.loads(client.succeed(f"cat /root/mig/{path}"))

    def total(rep, key):
        return sum(f[key] for a in rep for f in a["folders"])

    start_all()
    old.wait_for_unit("stalwart.service")
    old.wait_for_open_port(8080)
    old.wait_for_open_port(143)
    new.wait_for_unit("dovecot.service")
    new.wait_for_open_port(993)
    client.wait_for_unit("multi-user.target")

    with subtest("provision and seed Stalwart"):
        alice = client.succeed("mkpasswd -m sha-512 alicepass").strip()
        carol = client.succeed("mkpasswd -m bcrypt carolpass").strip()
        client.succeed(f"python3 ${./provision.py} '{alice}' '{carol}'")
        fixture("seed")

    with subtest("1. inventory"):
        client.succeed("mkdir -p /root/mig && cp ${mailmigToml} /root/mig/mailmig.toml")
        print(mig("inventory --raw raw.json"))
        plan = client.succeed("cat /root/mig/plan.toml")
        print(plan)
        secrets = json.loads(client.succeed("cat /root/mig/secrets.json"))["accounts"]
        assert secrets["alice@b4l.test"]["hash"] == alice, secrets["alice@b4l.test"]
        assert secrets["carol@b4l.test"]["source_scheme"] == "BLF-CRYPT"
        assert secrets["dave@poy.test"]["hash"] is None
        assert client.succeed("stat -c %a /root/mig/secrets.json").strip() == "600"
        mig("inventory", ok=False)  # refuses to clobber a reviewed plan

    with subtest("2. users"):
        print(mig("users --generate-missing"))
        client.succeed(
            "diff -u <(grep -v '^ *#' ${./expected-mail-accounts.nix}) <(grep -v '^ *#' /root/mig/out/mail-accounts.nix)"
        )
        client.succeed("diff -u ${./expected-dovecot-master-user.nix} /root/mig/out/dovecot-master-user.nix")
        for login in ["alice@b4l.test", "bob@b4l.test", "carol@b4l.test", "dave@poy.test"]:
            h = client.succeed(f"cat '/root/mig/out/hashes/{login}.hash'").strip()
            new.succeed(f"printf '%s\\n' '{h}' > '/var/lib/mailmig-hashes/{login}.hash'")
        new.succeed("systemctl restart dovecot.service")
        new.wait_for_open_port(993)
        # Users keep their Stalwart passwords; dave gets the generated one.
        fixture("login-new alice@b4l.test alicepass")
        fixture("login-new bob@b4l.test bobpass")
        fixture("login-new carol@b4l.test carolpass")
        dave_pw = client.succeed("grep ^dave@poy.test: /root/mig/out/generated-passwords.txt | cut -d: -f2-").strip()
        fixture(f"login-new dave@poy.test '{dave_pw}'")
        client.fail("python3 ${./imap_fixture.py} login-new alice@b4l.test wrongpass")
        print(client.succeed("cat /root/mig/out/clan-vars-set.sh"))

    with subtest("3. check"):
        print(mig("check"))

    with subtest("4. sync"):
        print(mig("sync --dry-run --report dry.json"))
        assert total(report("dry.json"), "copied") == 0
        mig("verify --report v0.json", ok=False)  # missing mail -> exit 1
        print(mig("sync --report s1.json"))
        s1 = report("s1.json")
        assert all(a["ok"] for a in s1), s1
        # 6 in alice's INBOX (\Deleted skipped) + 2 sent + 3 folders + bob 2 + carol 1 + dave 1
        assert total(s1, "copied") == 15, total(s1, "copied")
        assert total(s1, "skipped_deleted") == 1
        print(mig("verify --report v1.json"))

        box = json.loads(fixture("inspect alice@b4l.test alicepass"))
        print(json.dumps(box, indent=1))
        f = box["folders"]
        for name in ["INBOX", "Sent", "Junk", "Drafts", "Projects.2024", "Projects.v1_2", "Caf&AOk-"]:
            assert name in box["boxes"], (name, box["boxes"])
        # Stalwart's special-use folders land on SNM's names.
        mapped = {fr["src"]: fr["dst"] for a in s1 for fr in a["folders"] if a["login"] == "alice@b4l.test"}
        print("folder mapping:", mapped)
        assert mapped["Sent Items"] == "Sent"
        for src, dst in [("Deleted Items", "Trash"), ("Junk Mail", "Junk")]:
            if src in mapped:
                assert mapped[src] == dst, (src, mapped[src])
        inbox = {m["mid"]: m for m in f["INBOX"]}
        assert len(f["INBOX"]) == 6, f["INBOX"]
        assert "del@test" not in inbox
        assert [m["mid"] for m in f["INBOX"]].count("dup@test") == 2
        assert None in inbox  # message without Message-ID
        assert inbox["m1@test"]["flags"] == ["\\Flagged", "\\Seen"], inbox["m1@test"]
        assert inbox["m1@test"]["date"] == "01-Jan-2020 10:00:00 +0000", inbox["m1@test"]
        assert "$Forwarded" in inbox["m2@test"]["flags"]
        assert inbox["big@test"]["size"] > 3000000
        assert [m["mid"] for m in f["Sent"]] == ["s1@test", "s2@test"]
        assert [m["mid"] for m in f["Caf&AOk-"]] == ["c1@test"]
        dave = json.loads(fixture(f"inspect dave@poy.test '{dave_pw}'"))
        assert [m["mid"] for m in dave["folders"]["INBOX"]] == ["dave1@test"]

    with subtest("5. idempotent, delta and flag passes"):
        print(mig("sync --report s2.json"))
        assert total(report("s2.json"), "copied") == 0
        fixture("late")
        print(mig("sync --sync-flags --report s3.json"))
        s3 = report("s3.json")
        assert total(s3, "copied") == 1, s3
        assert total(s3, "flags_updated") == 1, s3
        inbox = {m["mid"]: m for m in json.loads(fixture("inspect alice@b4l.test alicepass"))["folders"]["INBOX"]}
        assert "late@test" in inbox
        assert "\\Seen" in inbox["m2@test"]["flags"]
        print(mig("verify --report v2.json"))

    with subtest("6. dns zone from SNM's DKIM keys"):
        client.succeed("mkdir -p /root/mig/dkim")
        for d in ["b4l.test", "poy.test"]:
            key = new.succeed(f"cat /var/dkim/{d}.mail.txt")
            client.succeed(f"cat > /root/mig/dkim/{d}.mail.txt <<'EOF'\n{key}\nEOF")
        zone = mig("dns zone")
        print(zone)
        assert "@                        IN  MX 10 new.test." in zone
        assert "mail._domainkey          IN  TXT" in zone and "v=DKIM1" in zone
        assert "_imaps._tcp              IN  SRV 0 1 993 new.test." in zone
  '';
}