diff --git a/machines/cursa/configuration.nix b/machines/cursa/configuration.nix index a492e5bc..b83eceb3 100644 --- a/machines/cursa/configuration.nix +++ b/machines/cursa/configuration.nix @@ -1,7 +1,12 @@ { + self, ... }: { + imports = [ + self.nixosModules.sd-card-image + ]; + clan.core.settings.machine.description = "VM machine for collecting prometheus metrics and fire alerts"; @@ -11,4 +16,10 @@ system.stateVersion = "25.11"; clan.core.sops.defaultGroups = [ "admins" ]; + + image.modules.sd-card = { + disabledModules = [ + ./disko.nix + ]; + }; } diff --git a/modules/nixos/flake-module.nix b/modules/nixos/flake-module.nix index f0a80174..2d4327bc 100644 --- a/modules/nixos/flake-module.nix +++ b/modules/nixos/flake-module.nix @@ -9,5 +9,8 @@ think-backend-gtcm = { imports = [ ./think-backend-gtcm.nix ]; }; + sd-card-image = { + imports = [ ./sd-card-image.nix ]; + }; }; } diff --git a/modules/nixos/sd-card-image.nix b/modules/nixos/sd-card-image.nix new file mode 100644 index 00000000..79156eee --- /dev/null +++ b/modules/nixos/sd-card-image.nix @@ -0,0 +1,180 @@ +# Local copy of the clan-core module in patches/upstream-sd-card-secret-injection.patch. +# Delete once clan-core ships clan.core.image.sd-card.addFilesScript +# (the option declaration would conflict). +{ + inputs, + lib, + pkgs, + ... +}: +let + parseAddFilesArgs = "${inputs.clan-core}/nixosModules/clanCore/image/parse-add-files-args.sh"; + + # Writes extra files straight into the ext4 root partition of an SD image + # with debugfs, so no boot-time hook is needed: the files exist on the root + # filesystem from the first boot. Runs unprivileged and without mounting. + sdCardAddFilesScript = pkgs.writeShellApplication { + name = "add-files"; + runtimeInputs = with pkgs; [ + coreutils + diffutils + e2fsprogs + findutils + util-linux + zstd + ]; + extraShellCheckFlags = [ "-x" ]; + text = '' + # shellcheck source=${parseAddFilesArgs} + source ${parseAddFilesArgs} + + # If no extra paths, just copy source to output + if [[ ''${#extra_paths[@]} -eq 0 ]]; then + cp -- "$source_image" "$output_image" + exit 0 + fi + + # The injection sandbox mounts /tmp as a RAM-backed tmpfs and a raw SD + # image is several GiB, so stage it next to the output, the only + # writable on-disk path. + workdir=$(dirname "$output_image") + img="$workdir/.$(basename "$output_image").raw" + cmds=$(mktemp) + trap 'rm -f -- "$img" "$cmds"' EXIT + + case "$source_image" in + *.zst) + zstd -d -q --no-progress -o "$img" -- "$source_image" + compress=1 + ;; + *) + cp --no-preserve=mode -- "$source_image" "$img" + compress=0 + ;; + esac + chmod u+w -- "$img" + + # sd-image.nix writes an MBR whose ext4 root is the (only) type-83 partition. + root_start="" + while read -r _ start type; do + if [[ "$type" == "0x83" ]]; then + root_start=$start + fi + done < <(partx --show -g -o NR,START,TYPE -- "$img") + if [[ -z "$root_start" ]]; then + echo "No Linux (type 83) root partition found in $source_image" >&2 + exit 1 + fi + # e2fsprogs opens a partition inside a disk image via "?offset=". + rootfs="$img?offset=$((root_start * 512))" + + # Build a debugfs command script. debugfs has neither mkdir -p nor a + # recursive write, so directories are created ancestor-first ("already + # exists" errors are harmless). write copies the source mode; ownership + # is forced to root since the script runs as an unprivileged user. + declare -A made_dirs=() + written=() + emit_mkdir_p() { + local dir="$1" path="" part + IFS=/ read -ra parts <<< "$dir" + for part in "''${parts[@]}"; do + [[ -z "$part" ]] && continue + path="$path/$part" + if [[ -z "''${made_dirs[$path]:-}" ]]; then + echo "mkdir \"$path\"" + made_dirs[$path]=1 + fi + done + } + emit_write() { + local src="$1" dest="$2" + echo "write \"$src\" \"$dest\"" + echo "sif \"$dest\" uid 0" + echo "sif \"$dest\" gid 0" + written+=("$src" "$dest") + } + + for ((i = 0; i < ''${#extra_paths[@]}; i += 2)); do + local_src="''${extra_paths[i]}" + # Normalize to an absolute path without trailing slash; "/" becomes "". + target="/''${extra_paths[i+1]#/}" + target="''${target%/}" + + if [[ -d "$local_src" ]]; then + emit_mkdir_p "$target" + while IFS= read -r -d "" entry; do + dest="$target/''${entry#"$local_src"/}" + if [[ -d "$entry" ]]; then + emit_mkdir_p "$dest" + elif [[ -f "$entry" ]]; then + emit_write "$entry" "$dest" + else + echo "Unsupported file type (only regular files and directories): $entry" >&2 + exit 1 + fi + done < <( + # Never treat our own staging/output directory as input, even if + # the caller placed it inside an extra path. + find "$local_src" -mindepth 1 -path "$workdir" -prune -o -print0 + ) + elif [[ -f "$local_src" ]]; then + emit_mkdir_p "$(dirname "$target")" + emit_write "$local_src" "$target" + else + echo "Extra path does not exist: $local_src" >&2 + exit 1 + fi + done > "$cmds" + + debugfs -w -f "$cmds" "$rootfs" + + # debugfs exits 0 even when commands fail: verify every file landed + # intact and the filesystem is still consistent. + for ((i = 0; i < ''${#written[@]}; i += 2)); do + if ! debugfs -R "cat \"''${written[i+1]}\"" "$rootfs" 2>/dev/null | cmp -s -- - "''${written[i]}"; then + echo "Failed to write ''${written[i+1]} into the root filesystem" >&2 + exit 1 + fi + done + if ! fsck_output=$(EXT2FS_NO_MTAB_OK=yes e2fsck -fn "$rootfs" 2>&1); then + echo "Root filesystem is inconsistent after injection:" >&2 + echo "$fsck_output" >&2 + exit 1 + fi + + if [[ "$compress" == 1 ]]; then + zstd -T0 -q --no-progress --rm -o "$output_image" -- "$img" + else + mv -- "$img" "$output_image" + fi + ''; + }; +in +{ + options.clan.core.image = { + sd-card.addFilesScript = lib.mkOption { + type = lib.types.package; + default = sdCardAddFilesScript; + defaultText = lib.literalExpression "pkgs.writeShellApplication { ... }"; + description = '' + Script to inject extra files into an SD card image + (`clan machines build --format sd-card`). + + Files are written directly into the ext4 root partition, so they are + present from the first boot without any initrd hook. Both compressed + (`.img.zst`, the `sdImage.compressImage` default) and raw `.img` + images are supported; the output keeps the source's format. + + Source `parse-add-files-args.sh` to get standardized argument parsing. + After sourcing, `$source_image`, `$output_image`, and `$extra_paths` + are available. + + Accepts the following arguments: + - --source ``: Path to the original built image (read-only) + - --output ``: Path for the modified output image + - --extra-path `` ``: Repeatable, maps a local + file or directory to a target filesystem path inside the image + ''; + }; + }; +} diff --git a/patches/clan-cli-image-filepath.patch b/patches/clan-cli-image-filepath.patch new file mode 100644 index 00000000..aa1d0e11 --- /dev/null +++ b/patches/clan-cli-image-filepath.patch @@ -0,0 +1,165 @@ +diff --git a/clan_lib/machines/build.py b/clan_lib/machines/build.py +index 00e2b16..d9ed5a1 100644 +--- a/clan_lib/machines/build.py ++++ b/clan_lib/machines/build.py +@@ -98,32 +98,42 @@ def _hostplatform_unset_hint(machine: Machine, build_format: str) -> str: + ) + + +-def _is_image_format(build_format: str) -> bool: +- """Check if the build format is an image format that supports secret injection.""" +- return build_format not in ("toplevel") ++def _image_add_files_script(machine: Machine, build_format: str) -> str | None: ++ """Return the format's addFilesScript store path. ++ ++ None when clan has no file-injection support for this format ++ (i.e. no `clan.core.image..addFilesScript` option), in which ++ case secret injection must be skipped rather than fail. ++ """ ++ result = machine.flake.select_machine( ++ machine.name, ++ f"config.clan.core.image.?{build_format}.addFilesScript", ++ ) ++ return result.get(build_format) + + +-def _find_image_file(build_dir: Path, build_format: str) -> Path: +- """Find the actual image file within a build output directory. ++def _find_image_file(machine: Machine, build_dir: Path, build_format: str) -> Path: ++ """Resolve the image file within an image build output directory. + +- Image builds output a directory (e.g., /nix/store/...-x86_64-linux.iso/) +- containing the image file in a subdirectory named after the format +- (e.g., iso/nixos-26.05-x86_64-linux.iso). ++ Image builds output a directory whose layout is format-specific ++ (iso/nixos-….iso, sd-image/nixos-….img.zst, …). nixpkgs exposes the ++ relative location as `config.image.filePath`, attached to the image ++ derivation as `passthru.filePath`. + """ +- format_dir = build_dir / build_format +- if not format_dir.is_dir(): +- msg = f"Expected directory {format_dir} not found in build output" +- raise ClanError(msg) +- files = [f for f in format_dir.iterdir() if f.is_file()] +- if len(files) != 1: +- msg = f"Expected exactly one file in {format_dir}, found {len(files)}" ++ file_path: str = machine.flake.select_machine( ++ machine.name, ++ f"config.system.build.images.{build_format}.passthru.filePath", ++ ) ++ image_file = build_dir / file_path ++ if not image_file.is_file(): ++ msg = f"Expected image file {image_file} not found in build output" + raise ClanError(msg) +- return files[0] ++ return image_file + + + def _inject_secrets_into_image( + machine: Machine, +- build_format: str, ++ add_files_script: str, + source_image_path: Path, + output_path: Path, + *, +@@ -137,7 +147,9 @@ def _inject_secrets_into_image( + secrets_target_dir = machine.secret_vars_store.get_upload_directory(machine.name) + + with TemporaryDirectory(prefix="clan-build-secrets-") as tmpdir: +- base_dir = Path(tmpdir).resolve() ++ # The tree mapped to "/" inside the image lives in its own subdir so ++ # the format script never sees its own output as input. ++ base_dir = Path(tmpdir).resolve() / "root" + upload_dir = base_dir / secrets_target_dir.lstrip("/") + upload_dir.mkdir(parents=True) + +@@ -153,12 +165,6 @@ def _inject_secrets_into_image( + if not any(upload_dir.iterdir()): + return False + +- # Get the addFilesScript path from the NixOS config +- script_path = machine.flake.select_machine( +- machine.name, +- f"config.clan.core.image.{build_format}.addFilesScript", +- ) +- + # Remove stale output from previous runs (xorriso refuses to + # write to an existing non-empty file when indev != outdev) + output = Path(output_path) +@@ -167,12 +173,12 @@ def _inject_secrets_into_image( + + # Write the image into the tmpdir so the sandbox only needs + # write access to the temp directory, not the final destination. +- sandbox_output = base_dir / "output" / output.name ++ sandbox_output = base_dir.parent / "output" / output.name + sandbox_output.parent.mkdir(parents=True) + + # Execute the addFilesScript; clean up partial output on failure + inject_cmd: list[str] = [ +- f"{script_path}/bin/add-files", ++ f"{add_files_script}/bin/add-files", + "--source", + str(source_image_path), + "--output", +@@ -265,33 +271,40 @@ def build_machine( + if build_path and (store := nix_test_store()): + build_path = store / str(build_path).lstrip("/") + +- # Post-build secret injection for image formats ++ # Post-build secret injection for image formats that support it + if ( + build_path +- and _is_image_format(options.format) ++ and options.format != "toplevel" + and not options.no_secrets + and not options.dry_run + ): +- # build_path is a directory; resolve the actual image file inside +- source_file = _find_image_file(build_path, options.format) +- # Output to a file next to the symlink, named after the source file +- output_file = Path.cwd() / source_file.name +- +- secrets_injected = _inject_secrets_into_image( +- machine=machine, +- build_format=options.format, +- source_image_path=source_file, +- output_path=output_file, +- use_sandbox=options.use_sandbox, +- ) +- if secrets_injected: +- log.warning( +- "This image contains secret key material " +- "(age private key). Treat this image as a sensitive " +- "artifact. Do not store it on shared or public media " +- "without encryption. Securely erase after use." ++ add_files_script = _image_add_files_script(machine, options.format) ++ if add_files_script is None: ++ log.info( ++ f"Image format '{options.format}' has no addFilesScript; " ++ "skipping secret injection" + ) +- build_path = output_file ++ else: ++ # build_path is a directory; resolve the actual image file inside ++ source_file = _find_image_file(machine, build_path, options.format) ++ # Output to a file next to the symlink, named after the source file ++ output_file = Path.cwd() / source_file.name ++ ++ secrets_injected = _inject_secrets_into_image( ++ machine=machine, ++ add_files_script=add_files_script, ++ source_image_path=source_file, ++ output_path=output_file, ++ use_sandbox=options.use_sandbox, ++ ) ++ if secrets_injected: ++ log.warning( ++ "This image contains secret key material " ++ "(age private key). Treat this image as a sensitive " ++ "artifact. Do not store it on shared or public media " ++ "without encryption. Securely erase after use." ++ ) ++ build_path = output_file + + return BuildResult( + machine_name=machine.name, diff --git a/patches/upstream-sd-card-secret-injection.patch b/patches/upstream-sd-card-secret-injection.patch new file mode 100644 index 00000000..bc0f0aa5 --- /dev/null +++ b/patches/upstream-sd-card-secret-injection.patch @@ -0,0 +1,814 @@ +From 78a3a5154f42f39bdeb913526439d51cd879f67c Mon Sep 17 00:00:00 2001 +From: kurogeek +Date: Fri, 25 Sep 2026 10:37:05 +0000 +Subject: [PATCH] clan machines build --format sd-card: fix image lookup, add + secret injection + +`clan machines build --format sd-card` failed after a successful nix +build: + + Expected directory /nix/store/...-nixos-image-sd-card-...img.zst/sd-card not found in build output + +Three bugs in the post-build secret injection step (clan_lib/machines/ +build.py): + +1. _find_image_file assumed the image lives at $out//. That only + holds for iso. nixpkgs' sd-card image writes $out/sd-image/.img.zst + (nixos/modules/installer/sd-card/sd-image.nix). nixpkgs exposes the + real location as config.image.filePath, attached to every + system.build.images. derivation as passthru.filePath. Use it. + +2. Injection ran for every non-toplevel format and then selected + config.clan.core.image..addFilesScript, which only iso + defines. Probe with the maybe-selector and skip injection (with a log + line) when the format has no addFilesScript, instead of failing after + an expensive image build. --no-secrets is no longer needed for such + formats. + +3. The tree passed as `--extra-path /` and the sandbox output + directory were the same directory, so every addFilesScript saw its + own output as input. The iso script tars the tree before xorriso + writes, so it shipped an empty /output directory in every ISO; a + script that stages a working copy of the image next to the output + would try to inject that copy into itself. Keep them as siblings: + tmpdir/root is mapped to /, tmpdir/output is the only writable path. + +Even with those fixed, sd-card images shipped without the age key, so a +freshly flashed machine could not decrypt its secrets on first boot. + +Add clan.core.image.sd-card.addFilesScript. Unlike the ISO (read-only +squashfs, needs an initrd hook to unpack /extra-files.tar), the SD +image's root is a plain ext4 partition, so the files are written +directly into it with debugfs: no boot-time component, no mount, no +privileges. The script locates the type-83 root partition with partx, +opens it via e2fsprogs' "?offset=" syntax, generates a +debugfs command script (mkdir ancestors, write, sif uid/gid 0), then +verifies every written file with cmp against the source and runs +e2fsck -fn, since debugfs exits 0 even when a command fails. Both +.img.zst (sdImage.compressImage default) and raw .img inputs are +handled; the output keeps the input's format. The raw image is staged +next to the output because the sandbox's /tmp is a RAM tmpfs. + +checks/clan-test-sd-card mirrors clan-test-iso: generate vars, run +`clan machines build peer1 --format sd-card` through the real CLI and +injection sandbox, boot the injected image in QEMU and verify over SSH +that a generated secret is decrypted (sops-nix found the injected age +key) and that /var/lib/sops-nix/key.txt is root:root 0600. The x86_64 +sd image boots via extlinux.conf, which QEMU cannot load by itself +(nixpkgs' boot-uboot-extlinux test is marked broken), so the test does +what a bootloader would: extract kernel, initrd and APPEND from /boot on +the image's root partition with debugfs and pass them to QEMU with +-kernel/-initrd/-append. + +Two unit tests in clan_cli/machines/build_test.py cover the CLI fixes; +both fail on main. +--- + checks/clan-test-sd-card/default.nix | 196 ++++++++++++++++++ + checks/clan-test-sd-card/flake.nix | 70 +++++++ + checks/flake-module.nix | 1 + + nixosModules/clanCore/image/default.nix | 1 + + .../clanCore/image/sd-card/default.nix | 179 ++++++++++++++++ + pkgs/clan-cli/clan_cli/machines/build_test.py | 67 +++++- + pkgs/clan-cli/clan_lib/machines/build.py | 107 +++++----- + 7 files changed, 573 insertions(+), 48 deletions(-) + create mode 100644 checks/clan-test-sd-card/default.nix + create mode 100644 checks/clan-test-sd-card/flake.nix + create mode 100644 nixosModules/clanCore/image/sd-card/default.nix + +diff --git a/checks/clan-test-sd-card/default.nix b/checks/clan-test-sd-card/default.nix +new file mode 100644 +index 0000000..e692242 +--- /dev/null ++++ b/checks/clan-test-sd-card/default.nix +@@ -0,0 +1,196 @@ ++{ ++ pkgs, ++ lib, ++ nixosLib, ++ clan-core, ++ ... ++}: ++let ++ system = pkgs.stdenv.hostPlatform.system; ++ qemu-common = import (pkgs.path + "/nixos/lib/qemu-common.nix") { ++ inherit lib; ++ inherit (pkgs) stdenv; ++ }; ++in ++nixosLib.runTest ( ++ { config, ... }: ++ let ++ closureInfo = pkgs.closureInfo { ++ rootPaths = [ ++ config.clan.test.machinesCross.${system}.peer1.config.system.build.images.sd-card.drvPath ++ config.clan.test.machinesCross.${system}.peer1.config.clan.core.image.sd-card.addFilesScript ++ config.clan.test.machinesCross.${system}.peer1.config.sops.package ++ config.clan.test.machinesCross.${system}.peer1.config.sops.gnupg.package ++ config.clan.test.systemsFile ++ config.clan.test.flakeForSandbox ++ ]; ++ }; ++ in ++ { ++ imports = [ ++ clan-core.modules.nixosTest.clanTest ++ ]; ++ ++ hostPkgs = pkgs; ++ ++ name = "clan-test-sd-card"; ++ ++ clan.test.fromFlake = ./.; ++ ++ # No framework-managed nodes; the SD image VM is created dynamically ++ containers = lib.mkForce { }; ++ ++ testScript = '' ++ import os ++ import shutil ++ import subprocess ++ import tempfile ++ import time ++ from nixos_test_lib.nix_setup import prepare_test_flake ++ from nixos_test_lib.port import find_free_port ++ ++ ++ # We don't have any machines to start but this is needed, as the container ++ # driver only sets up the environment correctly for nix build if this ++ # function es executed ++ start_all() ++ ++ with tempfile.TemporaryDirectory() as temp_dir: ++ flake, _ = prepare_test_flake(temp_dir, "${config.clan.test.flakeForSandbox}", "${closureInfo}") ++ ++ # generate sops keys ++ print("Generating vars...") ++ subprocess.run( ++ ["${ ++ clan-core.packages.${system}.clan-cli-full ++ }/bin/clan", "vars", "keygen", "--flake", str(flake), "--debug"], ++ check=True, ++ ) ++ ++ # Test 1: Build the SD image with clan CLI (includes secret injection) ++ print("Building SD image with clan CLI...") ++ result = subprocess.run( ++ ["${ ++ clan-core.packages.${system}.clan-cli-full ++ }/bin/clan", "machines", "build", "peer1", "--format", "sd-card", "--flake", str(flake), "--debug"], ++ check=True, ++ stdout=subprocess.PIPE, ++ text=True, ++ ) ++ # The last non-empty line of stdout is the injected image path ++ image_path = result.stdout.strip().splitlines()[-1] ++ print(f"Built SD image at: {image_path}") ++ assert os.path.isfile(image_path), f"SD image not found at {image_path}" ++ assert image_path.endswith(".img.zst"), f"expected a compressed image, got {image_path}" ++ ++ # Test 2: Boot the image and verify secrets via SSH ++ # QEMU needs a writable raw disk (the root partition is grown on first boot) ++ disk = os.path.join(temp_dir, "disk.img") ++ subprocess.run(["${pkgs.zstd}/bin/zstd", "-d", "-q", "-o", disk, "--", image_path], check=True) ++ ++ # The image boots via extlinux.conf, which QEMU cannot load by itself. ++ # Do what the bootloader would: read kernel, initrd and cmdline from ++ # /boot on the root partition and pass them to QEMU directly. ++ root_start = int(subprocess.run( ++ ["${pkgs.util-linux}/bin/partx", "--show", "-g", "-o", "START", "--nr", "2", disk], ++ check=True, capture_output=True, text=True, ++ ).stdout) ++ rootfs = f"{disk}?offset={root_start * 512}" ++ boot_dir = os.path.join(temp_dir, "boot") ++ os.mkdir(boot_dir) ++ subprocess.run( ++ ["${pkgs.e2fsprogs}/bin/debugfs", "-R", f"rdump /boot {boot_dir}", rootfs], ++ check=True, capture_output=True, ++ ) ++ extlinux_dir = os.path.join(boot_dir, "boot", "extlinux") ++ entry = {} ++ with open(os.path.join(extlinux_dir, "extlinux.conf")) as f: ++ for line in f: ++ key, _, value = line.strip().partition(" ") ++ if key in ("LINUX", "INITRD", "APPEND") and key not in entry: ++ entry[key] = value ++ kernel = os.path.normpath(os.path.join(extlinux_dir, entry["LINUX"])) ++ initrd = os.path.normpath(os.path.join(extlinux_dir, entry["INITRD"])) ++ print(f"Booting kernel={kernel} initrd={initrd} append={entry['APPEND']!r}") ++ ++ host_port = find_free_port() ++ ++ # Prepare SSH private key ++ ssh_key = os.path.join(temp_dir, "id_ed25519") ++ shutil.copy("${../assets/ssh/privkey}", ssh_key) ++ os.chmod(ssh_key, 0o600) ++ ++ qemu_cmd = [ ++ *"${qemu-common.qemuBinary pkgs.qemu_test}".split(), ++ "-m", "2048", ++ "-nographic", ++ "-kernel", kernel, ++ "-initrd", initrd, ++ "-append", entry["APPEND"], ++ "-drive", f"file={disk},format=raw,if=virtio", ++ "-netdev", f"user,id=net0,hostfwd=tcp::{host_port}-:22", ++ "-device", "virtio-net-pci,netdev=net0", ++ ] ++ ++ qemu_log = open(os.path.join(temp_dir, "qemu.log"), "wb") ++ qemu_proc = subprocess.Popen( ++ qemu_cmd, ++ stdin=subprocess.DEVNULL, ++ stdout=qemu_log, ++ stderr=subprocess.STDOUT, ++ ) ++ ++ ssh_cmd = [ ++ "${pkgs.openssh}/bin/ssh", ++ "-o", "UserKnownHostsFile=/dev/null", ++ "-o", "StrictHostKeyChecking=no", ++ "-o", "ConnectTimeout=3", ++ "-p", str(host_port), ++ "-i", ssh_key, ++ "root@localhost", ++ ] ++ ++ # Wait for SSH to become available by trying to connect ++ print(f"Waiting for SSH on host port {host_port}...") ++ for i in range(300): ++ if qemu_proc.poll() is not None: ++ qemu_log.close() ++ print(open(os.path.join(temp_dir, "qemu.log"), errors="replace").read()) ++ raise Exception(f"QEMU exited prematurely with code {qemu_proc.returncode}") ++ ret = subprocess.run(ssh_cmd + ["true"], capture_output=True) ++ if ret.returncode == 0: ++ print(f"SSH available after {i + 1} attempts") ++ break ++ time.sleep(1) ++ else: ++ qemu_log.close() ++ print(open(os.path.join(temp_dir, "qemu.log"), errors="replace").read()) ++ raise Exception(f"SSH never became available on port {host_port}") ++ ++ # Verify the test-generator secret is available with the expected value ++ result = subprocess.run( ++ ssh_cmd + ["cat /run/secrets/vars/per-machine/peer1/test-generator/test-secret"], ++ check=True, ++ capture_output=True, ++ text=True, ++ ) ++ actual_secret = result.stdout.strip() ++ assert actual_secret == "HongKong", f"Secret mismatch: got {actual_secret!r}, expected 'HongKong'" ++ ++ # The injected age key must be root-only on the root filesystem ++ result = subprocess.run( ++ ssh_cmd + ["stat -c '%U:%G %a' /var/lib/sops-nix/key.txt"], ++ check=True, ++ capture_output=True, ++ text=True, ++ ) ++ assert result.stdout.strip() == "root:root 600", f"unexpected key.txt ownership/mode: {result.stdout.strip()!r}" ++ ++ print("All SD image tests passed!") ++ ++ qemu_proc.terminate() ++ qemu_proc.wait() ++ qemu_log.close() ++ ''; ++ } ++) +diff --git a/checks/clan-test-sd-card/flake.nix b/checks/clan-test-sd-card/flake.nix +new file mode 100644 +index 0000000..94a0948 +--- /dev/null ++++ b/checks/clan-test-sd-card/flake.nix +@@ -0,0 +1,70 @@ ++{ ++ inputs.clan-core.url = "https://__replace__"; ++ inputs.nixpkgs.url = "https://__replace__"; ++ inputs.clan-core.inputs.nixpkgs.follows = "nixpkgs"; ++ inputs.systems.url = "https://__systems__"; ++ inputs.systems.flake = false; ++ ++ outputs = ++ { ++ self, ++ clan-core, ++ systems, ++ ... ++ }: ++ let ++ inherit (clan-core.inputs.nixpkgs) lib; ++ ++ hostSystem = lib.head (import systems); ++ ++ # Usage see: https://clan.lol/docs ++ clan = clan-core.lib.clan { ++ inherit self; ++ ++ machines.peer1 = ++ { modulesPath, ... }: ++ { ++ nixpkgs.hostPlatform = hostSystem; ++ ++ imports = [ ++ # Required for the NixOS test driver backdoor (hvc0 shell) ++ "${modulesPath}/testing/test-instrumentation.nix" ++ ]; ++ ++ # The image is booted from a virtio disk in QEMU ++ boot.initrd.availableKernelModules = [ ++ "virtio_pci" ++ "virtio_blk" ++ ]; ++ ++ # Disable vars state version generation ++ clan.core.settings.state-version.enable = true; ++ ++ # Enable one secret to be generated ++ clan.core.vars.generators.test-generator = { ++ files.test-secret.secret = true; ++ script = "echo HongKong > $out/test-secret"; ++ }; ++ ++ # Enable SSH for boot verification ++ services.openssh.enable = true; ++ services.openssh.settings.PermitRootLogin = "yes"; ++ users.users.root.openssh.authorizedKeys.keys = [ ++ (builtins.readFile (clan-core + "/checks/assets/ssh/pubkey")) ++ ]; ++ }; ++ ++ inventory = ++ { ... }: ++ { ++ meta.name = "foo"; ++ meta.domain = "foo"; ++ machines.peer1 = { }; ++ }; ++ }; ++ in ++ { ++ # all machines managed by Clan ++ inherit (clan.config) nixosConfigurations nixosModules clanInternals; ++ }; ++} +diff --git a/checks/flake-module.nix b/checks/flake-module.nix +index b6aa02a..5ce0733 100644 +--- a/checks/flake-module.nix ++++ b/checks/flake-module.nix +@@ -94,6 +94,7 @@ in + service-dummy-test = import ./service-dummy-test nixosTestArgs; + service-dummy-test-from-flake = import ./service-dummy-test-from-flake nixosTestArgs; + clan-test-iso = import ./clan-test-iso nixosTestArgs; ++ clan-test-sd-card = import ./clan-test-sd-card nixosTestArgs; + }; + + packagesToBuild = lib.removeAttrs self'.packages ( +diff --git a/nixosModules/clanCore/image/default.nix b/nixosModules/clanCore/image/default.nix +index 7dc3b5a..6191385 100644 +--- a/nixosModules/clanCore/image/default.nix ++++ b/nixosModules/clanCore/image/default.nix +@@ -2,5 +2,6 @@ + { + imports = [ + ./iso ++ ./sd-card + ]; + } +diff --git a/nixosModules/clanCore/image/sd-card/default.nix b/nixosModules/clanCore/image/sd-card/default.nix +new file mode 100644 +index 0000000..2ef9fee +--- /dev/null ++++ b/nixosModules/clanCore/image/sd-card/default.nix +@@ -0,0 +1,179 @@ ++{ ++ lib, ++ pkgs, ++ ... ++}: ++let ++ parseAddFilesArgs = builtins.path { ++ path = ../parse-add-files-args.sh; ++ name = "parse-add-files-args.sh"; ++ }; ++ ++ # Writes extra files straight into the ext4 root partition of an SD image ++ # with debugfs, so no boot-time hook is needed: the files exist on the root ++ # filesystem from the first boot. Runs unprivileged and without mounting. ++ sdCardAddFilesScript = pkgs.writeShellApplication { ++ name = "add-files"; ++ runtimeInputs = with pkgs; [ ++ coreutils ++ diffutils ++ e2fsprogs ++ findutils ++ util-linux ++ zstd ++ ]; ++ extraShellCheckFlags = [ "-x" ]; ++ text = '' ++ # shellcheck source=${parseAddFilesArgs} ++ source ${parseAddFilesArgs} ++ ++ # If no extra paths, just copy source to output ++ if [[ ''${#extra_paths[@]} -eq 0 ]]; then ++ cp -- "$source_image" "$output_image" ++ exit 0 ++ fi ++ ++ # The injection sandbox mounts /tmp as a RAM-backed tmpfs and a raw SD ++ # image is several GiB, so stage it next to the output, the only ++ # writable on-disk path. ++ workdir=$(dirname "$output_image") ++ img="$workdir/.$(basename "$output_image").raw" ++ cmds=$(mktemp) ++ trap 'rm -f -- "$img" "$cmds"' EXIT ++ ++ case "$source_image" in ++ *.zst) ++ zstd -d -q --no-progress -o "$img" -- "$source_image" ++ compress=1 ++ ;; ++ *) ++ cp --no-preserve=mode -- "$source_image" "$img" ++ compress=0 ++ ;; ++ esac ++ chmod u+w -- "$img" ++ ++ # sd-image.nix writes an MBR whose ext4 root is the (only) type-83 partition. ++ root_start="" ++ while read -r _ start type; do ++ if [[ "$type" == "0x83" ]]; then ++ root_start=$start ++ fi ++ done < <(partx --show -g -o NR,START,TYPE -- "$img") ++ if [[ -z "$root_start" ]]; then ++ echo "No Linux (type 83) root partition found in $source_image" >&2 ++ exit 1 ++ fi ++ # e2fsprogs opens a partition inside a disk image via "?offset=". ++ rootfs="$img?offset=$((root_start * 512))" ++ ++ # Build a debugfs command script. debugfs has neither mkdir -p nor a ++ # recursive write, so directories are created ancestor-first ("already ++ # exists" errors are harmless). write copies the source mode; ownership ++ # is forced to root since the script runs as an unprivileged user. ++ declare -A made_dirs=() ++ written=() ++ emit_mkdir_p() { ++ local dir="$1" path="" part ++ IFS=/ read -ra parts <<< "$dir" ++ for part in "''${parts[@]}"; do ++ [[ -z "$part" ]] && continue ++ path="$path/$part" ++ if [[ -z "''${made_dirs[$path]:-}" ]]; then ++ echo "mkdir \"$path\"" ++ made_dirs[$path]=1 ++ fi ++ done ++ } ++ emit_write() { ++ local src="$1" dest="$2" ++ echo "write \"$src\" \"$dest\"" ++ echo "sif \"$dest\" uid 0" ++ echo "sif \"$dest\" gid 0" ++ written+=("$src" "$dest") ++ } ++ ++ for ((i = 0; i < ''${#extra_paths[@]}; i += 2)); do ++ local_src="''${extra_paths[i]}" ++ # Normalize to an absolute path without trailing slash; "/" becomes "". ++ target="/''${extra_paths[i+1]#/}" ++ target="''${target%/}" ++ ++ if [[ -d "$local_src" ]]; then ++ emit_mkdir_p "$target" ++ while IFS= read -r -d "" entry; do ++ dest="$target/''${entry#"$local_src"/}" ++ if [[ -d "$entry" ]]; then ++ emit_mkdir_p "$dest" ++ elif [[ -f "$entry" ]]; then ++ emit_write "$entry" "$dest" ++ else ++ echo "Unsupported file type (only regular files and directories): $entry" >&2 ++ exit 1 ++ fi ++ done < <( ++ # Never treat our own staging/output directory as input, even if ++ # the caller placed it inside an extra path. ++ find "$local_src" -mindepth 1 -path "$workdir" -prune -o -print0 ++ ) ++ elif [[ -f "$local_src" ]]; then ++ emit_mkdir_p "$(dirname "$target")" ++ emit_write "$local_src" "$target" ++ else ++ echo "Extra path does not exist: $local_src" >&2 ++ exit 1 ++ fi ++ done > "$cmds" ++ ++ debugfs -w -f "$cmds" "$rootfs" ++ ++ # debugfs exits 0 even when commands fail: verify every file landed ++ # intact and the filesystem is still consistent. ++ for ((i = 0; i < ''${#written[@]}; i += 2)); do ++ if ! debugfs -R "cat \"''${written[i+1]}\"" "$rootfs" 2>/dev/null | cmp -s -- - "''${written[i]}"; then ++ echo "Failed to write ''${written[i+1]} into the root filesystem" >&2 ++ exit 1 ++ fi ++ done ++ if ! fsck_output=$(EXT2FS_NO_MTAB_OK=yes e2fsck -fn "$rootfs" 2>&1); then ++ echo "Root filesystem is inconsistent after injection:" >&2 ++ echo "$fsck_output" >&2 ++ exit 1 ++ fi ++ ++ if [[ "$compress" == 1 ]]; then ++ zstd -T0 -q --no-progress --rm -o "$output_image" -- "$img" ++ else ++ mv -- "$img" "$output_image" ++ fi ++ ''; ++ }; ++in ++{ ++ options.clan.core.image = { ++ sd-card.addFilesScript = lib.mkOption { ++ type = lib.types.package; ++ default = sdCardAddFilesScript; ++ defaultText = lib.literalExpression "pkgs.writeShellApplication { ... }"; ++ description = '' ++ Script to inject extra files into an SD card image ++ (`clan machines build --format sd-card`). ++ ++ Files are written directly into the ext4 root partition, so they are ++ present from the first boot without any initrd hook. Both compressed ++ (`.img.zst`, the `sdImage.compressImage` default) and raw `.img` ++ images are supported; the output keeps the source's format. ++ ++ Source `parse-add-files-args.sh` to get standardized argument parsing. ++ After sourcing, `$source_image`, `$output_image`, and `$extra_paths` ++ are available. ++ ++ Accepts the following arguments: ++ - --source ``: Path to the original built image (read-only) ++ - --output ``: Path for the modified output image ++ - --extra-path `` ``: Repeatable, maps a local ++ file or directory to a target filesystem path inside the image ++ ''; ++ }; ++ }; ++} +diff --git a/pkgs/clan-cli/clan_cli/machines/build_test.py b/pkgs/clan-cli/clan_cli/machines/build_test.py +index ad0caf0..b5642a0 100644 +--- a/pkgs/clan-cli/clan_cli/machines/build_test.py ++++ b/pkgs/clan-cli/clan_cli/machines/build_test.py +@@ -1,8 +1,9 @@ + from pathlib import Path + + import pytest +-from clan_lib.errors import ClanError ++from clan_lib.errors import ClanError, CmdOut + from clan_lib.flake import Flake ++from clan_lib.machines import build as build_lib + from clan_lib.machines.build import BuildOptions, build_machine + from clan_lib.machines.machines import Machine + +@@ -131,3 +132,67 @@ def test_build_machine_without_hostplatform_raises_friendly_error( + assert "nixpkgs.hostPlatform" in msg + assert "--system" in msg + assert "jon" in msg ++ ++ ++def test_build_machine_skips_injection_for_format_without_add_files_script( ++ tmp_path: Path, ++ monkeypatch: pytest.MonkeyPatch, ++) -> None: ++ """Formats without clan.core.image..addFilesScript (e.g. sd-card) ++ must build successfully and return the raw nix output, not fail trying ++ to locate an image file for injection. ++ """ ++ build_dir = tmp_path / "out" ++ (build_dir / "sd-image").mkdir(parents=True) ++ (build_dir / "sd-image" / "nixos.img.zst").touch() ++ ++ def fake_run(cmd: list[str], *_args: object, **_kwargs: object) -> CmdOut: ++ return CmdOut( ++ stdout=f"{build_dir}\n", ++ stderr="", ++ env=None, ++ cwd=tmp_path, ++ command_list=cmd, ++ returncode=0, ++ msg=None, ++ ) ++ ++ def fake_select_machine(_self: Flake, _machine: str, selector: str) -> object: ++ assert selector == "config.clan.core.image.?sd-card.addFilesScript" ++ return {} ++ ++ def no_secrets(*_args: object, **_kwargs: object) -> object: ++ msg = "secrets must not be generated for a format without addFilesScript" ++ raise AssertionError(msg) ++ ++ monkeypatch.setattr(build_lib, "get_build_target", lambda *_a, **_k: "flake#x") ++ monkeypatch.setattr(build_lib, "run", fake_run) ++ monkeypatch.setattr(build_lib, "get_machine_generators", no_secrets) ++ monkeypatch.setattr(Flake, "select_machine", fake_select_machine) ++ ++ machine = Machine(name="jon", flake=Flake(str(tmp_path))) ++ result = build_machine(machine, BuildOptions(format="sd-card", no_link=True)) ++ ++ assert result.build_path == build_dir ++ ++ ++def test_find_image_file_uses_image_filepath( ++ tmp_path: Path, ++ monkeypatch: pytest.MonkeyPatch, ++) -> None: ++ """The image lives at config.image.filePath inside the build output, ++ which is not necessarily a directory named after the format ++ (sd-card -> sd-image/...). ++ """ ++ image = tmp_path / "sd-image" / "nixos-image-sd-card-x86_64-linux.img.zst" ++ image.parent.mkdir() ++ image.touch() ++ ++ def fake_select_machine(_self: Flake, _machine: str, selector: str) -> str: ++ assert selector == "config.system.build.images.sd-card.passthru.filePath" ++ return "sd-image/nixos-image-sd-card-x86_64-linux.img.zst" ++ ++ monkeypatch.setattr(Flake, "select_machine", fake_select_machine) ++ machine = Machine(name="jon", flake=Flake(str(tmp_path))) ++ ++ assert build_lib._find_image_file(machine, tmp_path, "sd-card") == image +diff --git a/pkgs/clan-cli/clan_lib/machines/build.py b/pkgs/clan-cli/clan_lib/machines/build.py +index 00e2b16..d9ed5a1 100644 +--- a/pkgs/clan-cli/clan_lib/machines/build.py ++++ b/pkgs/clan-cli/clan_lib/machines/build.py +@@ -98,32 +98,42 @@ def _hostplatform_unset_hint(machine: Machine, build_format: str) -> str: + ) + + +-def _is_image_format(build_format: str) -> bool: +- """Check if the build format is an image format that supports secret injection.""" +- return build_format not in ("toplevel") ++def _image_add_files_script(machine: Machine, build_format: str) -> str | None: ++ """Return the format's addFilesScript store path. ++ ++ None when clan has no file-injection support for this format ++ (i.e. no `clan.core.image..addFilesScript` option), in which ++ case secret injection must be skipped rather than fail. ++ """ ++ result = machine.flake.select_machine( ++ machine.name, ++ f"config.clan.core.image.?{build_format}.addFilesScript", ++ ) ++ return result.get(build_format) + + +-def _find_image_file(build_dir: Path, build_format: str) -> Path: +- """Find the actual image file within a build output directory. ++def _find_image_file(machine: Machine, build_dir: Path, build_format: str) -> Path: ++ """Resolve the image file within an image build output directory. + +- Image builds output a directory (e.g., /nix/store/...-x86_64-linux.iso/) +- containing the image file in a subdirectory named after the format +- (e.g., iso/nixos-26.05-x86_64-linux.iso). ++ Image builds output a directory whose layout is format-specific ++ (iso/nixos-….iso, sd-image/nixos-….img.zst, …). nixpkgs exposes the ++ relative location as `config.image.filePath`, attached to the image ++ derivation as `passthru.filePath`. + """ +- format_dir = build_dir / build_format +- if not format_dir.is_dir(): +- msg = f"Expected directory {format_dir} not found in build output" +- raise ClanError(msg) +- files = [f for f in format_dir.iterdir() if f.is_file()] +- if len(files) != 1: +- msg = f"Expected exactly one file in {format_dir}, found {len(files)}" ++ file_path: str = machine.flake.select_machine( ++ machine.name, ++ f"config.system.build.images.{build_format}.passthru.filePath", ++ ) ++ image_file = build_dir / file_path ++ if not image_file.is_file(): ++ msg = f"Expected image file {image_file} not found in build output" + raise ClanError(msg) +- return files[0] ++ return image_file + + + def _inject_secrets_into_image( + machine: Machine, +- build_format: str, ++ add_files_script: str, + source_image_path: Path, + output_path: Path, + *, +@@ -137,7 +147,9 @@ def _inject_secrets_into_image( + secrets_target_dir = machine.secret_vars_store.get_upload_directory(machine.name) + + with TemporaryDirectory(prefix="clan-build-secrets-") as tmpdir: +- base_dir = Path(tmpdir).resolve() ++ # The tree mapped to "/" inside the image lives in its own subdir so ++ # the format script never sees its own output as input. ++ base_dir = Path(tmpdir).resolve() / "root" + upload_dir = base_dir / secrets_target_dir.lstrip("/") + upload_dir.mkdir(parents=True) + +@@ -153,12 +165,6 @@ def _inject_secrets_into_image( + if not any(upload_dir.iterdir()): + return False + +- # Get the addFilesScript path from the NixOS config +- script_path = machine.flake.select_machine( +- machine.name, +- f"config.clan.core.image.{build_format}.addFilesScript", +- ) +- + # Remove stale output from previous runs (xorriso refuses to + # write to an existing non-empty file when indev != outdev) + output = Path(output_path) +@@ -167,12 +173,12 @@ def _inject_secrets_into_image( + + # Write the image into the tmpdir so the sandbox only needs + # write access to the temp directory, not the final destination. +- sandbox_output = base_dir / "output" / output.name ++ sandbox_output = base_dir.parent / "output" / output.name + sandbox_output.parent.mkdir(parents=True) + + # Execute the addFilesScript; clean up partial output on failure + inject_cmd: list[str] = [ +- f"{script_path}/bin/add-files", ++ f"{add_files_script}/bin/add-files", + "--source", + str(source_image_path), + "--output", +@@ -265,33 +271,40 @@ def build_machine( + if build_path and (store := nix_test_store()): + build_path = store / str(build_path).lstrip("/") + +- # Post-build secret injection for image formats ++ # Post-build secret injection for image formats that support it + if ( + build_path +- and _is_image_format(options.format) ++ and options.format != "toplevel" + and not options.no_secrets + and not options.dry_run + ): +- # build_path is a directory; resolve the actual image file inside +- source_file = _find_image_file(build_path, options.format) +- # Output to a file next to the symlink, named after the source file +- output_file = Path.cwd() / source_file.name +- +- secrets_injected = _inject_secrets_into_image( +- machine=machine, +- build_format=options.format, +- source_image_path=source_file, +- output_path=output_file, +- use_sandbox=options.use_sandbox, +- ) +- if secrets_injected: +- log.warning( +- "This image contains secret key material " +- "(age private key). Treat this image as a sensitive " +- "artifact. Do not store it on shared or public media " +- "without encryption. Securely erase after use." ++ add_files_script = _image_add_files_script(machine, options.format) ++ if add_files_script is None: ++ log.info( ++ f"Image format '{options.format}' has no addFilesScript; " ++ "skipping secret injection" + ) +- build_path = output_file ++ else: ++ # build_path is a directory; resolve the actual image file inside ++ source_file = _find_image_file(machine, build_path, options.format) ++ # Output to a file next to the symlink, named after the source file ++ output_file = Path.cwd() / source_file.name ++ ++ secrets_injected = _inject_secrets_into_image( ++ machine=machine, ++ add_files_script=add_files_script, ++ source_image_path=source_file, ++ output_path=output_file, ++ use_sandbox=options.use_sandbox, ++ ) ++ if secrets_injected: ++ log.warning( ++ "This image contains secret key material " ++ "(age private key). Treat this image as a sensitive " ++ "artifact. Do not store it on shared or public media " ++ "without encryption. Securely erase after use." ++ ) ++ build_path = output_file + + return BuildResult( + machine_name=machine.name, +-- +2.54.0 + diff --git a/shell.nix b/shell.nix index 001796aa..83d37dab 100644 --- a/shell.nix +++ b/shell.nix @@ -9,7 +9,15 @@ devshells.default = { devshell = { packages = [ - inputs.clan-core.packages.${system}.clan-cli + # Local fix until upstream lands (patches/upstream-sd-card-secret-injection.patch): + # `clan machines build --format sd-card` resolves the image via + # image.filePath, skips secret injection for formats without + # clan.core.image..addFilesScript, and stages the + # injected tree apart from the sandbox output. The sd-card + # addFilesScript itself is modules/nixos/sd-card-image.nix. + (inputs.clan-core.packages.${system}.clan-cli.overrideAttrs (old: { + patches = (old.patches or [ ]) ++ [ ./patches/clan-cli-image-filepath.patch ]; + })) ]; }; };