#!/usr/bin/env python3 """The release files of a prototype stage: every board's manufacturing files and the documentation. python3 .gitea/scripts/release-files.py v0.2 [--out release] Writes into --out (default release/; existing content is replaced): README.md version, commit, checks, file list, SHA-256, changelog hardware//--gerbers.zip Gerbers and Excellon drill files for the PCB fab --bom.csv bill of materials, grouped, DNP left out --cpl.csv pick and place, both sides, mm --schematic.pdf the schematic --pcb.pdf one page per copper, silkscreen and fab layer --top.png 3D render, top --bottom.png 3D render, bottom docs/-.pdf docs/*.md (pandoc, pdflatex) and docs/tex/*.tex (pdflatex) Boards are the [[boards]] of proplan.toml, or hardware/*/*.kicad_pro when it names none. DRC and ERC run and their results go into README.md; they do not stop the release, because the tag records what was ordered. A document that does not build is listed as not built. Exit 1 when the files of a board could not be made (the workflow then publishes nothing). Used by .gitea/workflows/release.yml. Locally, from the repository root, in the CI image: docker run --rm -v "$PWD:/w" -w /w git.b4l.co.th/b4l/kicad10_auto_node:latest \\ python3 .gitea/scripts/release-files.py v0.1 --out build/release """ import argparse import csv import datetime import hashlib import io import json import os import re import shutil import subprocess import sys import tempfile import tomllib import zipfile from pathlib import Path ROOT = Path(__file__).resolve().parents[2] KICAD_CLI = os.environ.get("KICAD_CLI") or shutil.which("kicad-cli") or "kicad-cli" RENDER_SIZE = (os.environ.get("RENDER_WIDTH", "1400"), os.environ.get("RENDER_HEIGHT", "1540")) ZIP_TIME = (2020, 1, 1, 0, 0, 0) # fixed timestamps: the same files give the same zip BOM_FIELDS = ["Reference", "Value", "Footprint", "Manufacturer", "MPN", "LCSC", "QUANTITY"] BOM_LABELS = ["Reference", "Value", "Footprint", "Manufacturer", "MPN", "LCSC", "Qty"] # Characters pdflatex does not know, as LaTeX. Anything else it refuses is printed as "?" and # reported (pdflatex because the image's lualatex cannot load its fonts). UNICODE = { 0x2192: r"\ensuremath{\rightarrow}", 0x2190: r"\ensuremath{\leftarrow}", 0x2194: r"\ensuremath{\leftrightarrow}", 0x2191: r"\ensuremath{\uparrow}", 0x2193: r"\ensuremath{\downarrow}", 0x21D2: r"\ensuremath{\Rightarrow}", 0x21D0: r"\ensuremath{\Leftarrow}", 0x21D4: r"\ensuremath{\Leftrightarrow}", 0x21A6: r"\ensuremath{\mapsto}", 0x2264: r"\ensuremath{\leq}", 0x2265: r"\ensuremath{\geq}", 0x2248: r"\ensuremath{\approx}", 0x2260: r"\ensuremath{\neq}", 0x2261: r"\ensuremath{\equiv}", 0x2212: r"\ensuremath{-}", 0x221E: r"\ensuremath{\infty}", 0x221A: r"\ensuremath{\surd}", 0x2211: r"\ensuremath{\sum}", 0x220F: r"\ensuremath{\prod}", 0x2202: r"\ensuremath{\partial}", 0x2208: r"\ensuremath{\in}", 0x2205: r"\ensuremath{\emptyset}", 0x2227: r"\ensuremath{\wedge}", 0x2228: r"\ensuremath{\vee}", 0x2295: r"\ensuremath{\oplus}", 0x22C5: r"\ensuremath{\cdot}", 0x2026: r"\ldots{}", 0x2713: r"\ensuremath{\checkmark}", 0x2714: r"\ensuremath{\checkmark}", 0x2705: r"\ensuremath{\checkmark}", 0x2717: r"\ensuremath{\times}", 0x2718: r"\ensuremath{\times}", 0x274C: r"\ensuremath{\times}", 0x2126: r"\ensuremath{\Omega}", 0x2103: r"\textdegree{}C", 0x2032: r"\ensuremath{'}", 0x2033: r"\ensuremath{''}", 0x2009: r"\,", 0x202F: r"\,", 0x200B: "", 0xFE0F: "", 0x25B6: r"\ensuremath{\blacktriangleright}", 0x25C0: r"\ensuremath{\blacktriangleleft}", 0x25B2: r"\ensuremath{\blacktriangle}", 0x25BC: r"\ensuremath{\blacktriangledown}", 0x25CF: r"\ensuremath{\bullet}", 0x25CB: r"\ensuremath{\circ}", 0x26A0: r"\textbf{!}", # box drawing (trees in code blocks); \mbox keeps two dashes from becoming an en dash 0x2500: r"\mbox{-}", 0x2501: r"\mbox{-}", 0x2502: "|", 0x2503: "|", 0x2550: r"\mbox{=}", 0x2551: "|", **{c: "+" for c in (0x250C, 0x2510, 0x2514, 0x2518, 0x251C, 0x2524, 0x252C, 0x2534, 0x253C, 0x2554, 0x2557, 0x255A, 0x255D, 0x2560, 0x2563, 0x2566, 0x2569, 0x256C)}, } GREEK = "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho - sigma tau upsilon phi chi psi omega" for i, n in enumerate(GREEK.split()): if n not in ("-", "omicron"): UNICODE[0x3B1 + i] = rf"\ensuremath{{\{n}}}" for c, n in {0x393: "Gamma", 0x394: "Delta", 0x398: "Theta", 0x39B: "Lambda", 0x39E: "Xi", 0x3A0: "Pi", 0x3A3: "Sigma", 0x3A5: "Upsilon", 0x3A6: "Phi", 0x3A8: "Psi", 0x3A9: "Omega"}.items(): UNICODE[c] = rf"\ensuremath{{\{n}}}" UNICODE[0x3C2] = r"\ensuremath{\varsigma}" class Failed(Exception): pass def run(cmd, cwd=None, check=True): r = subprocess.run([str(c) for c in cmd], cwd=cwd, capture_output=True, text=True, errors="replace") if check and r.returncode != 0: raise Failed(f"{Path(str(cmd[0])).name} {' '.join(map(str, cmd[1:4]))}: exit {r.returncode}\n" f"{(r.stdout + r.stderr)[-1500:]}") return r def cli(*args): return run([KICAD_CLI, *args]).stdout def git(*args): r = run(["git", "-C", ROOT, *args], check=False) return r.stdout.strip() if r.returncode == 0 else "" def tex_escape(s): return re.sub(r"([\\{}$&#^_%~])", lambda m: {"\\": r"\textbackslash{}", "^": r"\^{}", "~": r"\~{}"} .get(m.group(1), "\\" + m.group(1)), s) def clear(folder): """Empty a folder but keep it (a bind-mounted folder may refuse to be removed).""" folder.mkdir(parents=True, exist_ok=True) for p in folder.iterdir(): shutil.rmtree(p) if p.is_dir() else p.unlink() def manifest(): path = ROOT / "proplan.toml" if not path.exists(): return {} try: return tomllib.loads(path.read_text(encoding="utf-8")) except tomllib.TOMLDecodeError as e: print(f"::warning::proplan.toml cannot be read ({e}); taking hardware/*/*.kicad_pro") return {} def projects(m): """(board folder, .kicad_pro or None) for every board.""" dirs = [ROOT / b["path"] for b in m.get("boards", []) if isinstance(b, dict) and isinstance(b.get("path"), str)] if not dirs: dirs = sorted({p.parent for p in ROOT.glob("hardware/*/*.kicad_pro")}) out = [] for d in dirs: pros = sorted(d.glob("*.kicad_pro")) out.append((d, next((p for p in pros if p.stem == d.name), pros[0] if pros else None))) return out def erc(sch, tmp): rpt = tmp / "erc.json" cli("sch", "erc", "--format", "json", "--severity-error", "-o", rpt, sch) d = json.loads(rpt.read_text(encoding="utf-8")) return sum(len(s.get("violations", [])) for s in d.get("sheets", [])) def drc(pcb, tmp): rpt = tmp / "drc.json" cli("pcb", "drc", "--format", "json", "--severity-error", "--schematic-parity", "-o", rpt, pcb) d = json.loads(rpt.read_text(encoding="utf-8")) return {k: len(d.get(k, [])) for k in ("violations", "unconnected_items", "schematic_parity")} def copper_layers(pcb): return re.findall(r'\(\d+ "([A-Za-z0-9]+\.Cu)" (?:signal|power|mixed|jumper)\)', pcb.read_text(encoding="utf-8")) def gerbers_zip(pcb, dest, tmp): out = tmp / "gerbers" out.mkdir() copper = copper_layers(pcb) layers = copper + ["F.Paste", "B.Paste", "F.Silkscreen", "B.Silkscreen", "F.Mask", "B.Mask", "Edge.Cuts"] cli("pcb", "export", "gerbers", "-o", f"{out}/", "--layers", ",".join(layers), "--subtract-soldermask", "--no-x2", "--no-netlist", "--check-zones", pcb) cli("pcb", "export", "drill", "-o", f"{out}/", "--format", "excellon", "--drill-origin", "absolute", "--excellon-units", "mm", "--excellon-zeros-format", "decimal", "--excellon-oval-format", "alternate", "--excellon-separate-th", pcb) files = sorted(p for p in out.iterdir() if p.suffix.lower() != ".gbrjob") drills = [p for p in files if p.suffix.lower() == ".drl"] if len(files) - len(drills) != len(layers) or not drills: raise Failed(f"expected {len(layers)} Gerbers and drill files, got {[p.name for p in files]}") with zipfile.ZipFile(dest, "w", zipfile.ZIP_DEFLATED) as z: for p in files: info = zipfile.ZipInfo(p.name, date_time=ZIP_TIME) info.compress_type = zipfile.ZIP_DEFLATED z.writestr(info, p.read_bytes()) return f"Gerbers ({len(copper)} copper layers) and drill files for the PCB fab" def bom(sch, dest): cli("sch", "export", "bom", "-o", dest, "--fields", ",".join(BOM_FIELDS), "--labels", ",".join(BOM_LABELS), "--group-by", "Value,Footprint,Manufacturer,MPN,LCSC", "--exclude-dnp", sch) rows = list(csv.reader(io.StringIO(dest.read_text(encoding="utf-8")))) return f"bill of materials: {max(len(rows) - 1, 0)} lines, DNP parts left out" def cpl(pcb, dest): cli("pcb", "export", "pos", "-o", dest, "--format", "csv", "--units", "mm", "--side", "both", "--exclude-dnp", pcb) rows = list(csv.reader(io.StringIO(dest.read_text(encoding="utf-8")))) return f"pick and place: {max(len(rows) - 1, 0)} parts, both sides, mm, DNP left out" def board(board_dir, pro, version, out, tmp): """Write one board's files; returns (name, [(file, what)], checks).""" name = pro.stem sch, pcb = board_dir / f"{name}.kicad_sch", board_dir / f"{name}.kicad_pcb" dest = out / "hardware" / name dest.mkdir(parents=True) base = f"{name}-{version}" files, checks = [], [] if sch.exists(): n = erc(sch, tmp) checks.append(f"ERC: {n} errors") files.append((f"{base}-schematic.pdf", "schematic")) cli("sch", "export", "pdf", "-o", dest / files[-1][0], sch) files.append((f"{base}-bom.csv", bom(sch, dest / f"{base}-bom.csv"))) if pcb.exists(): d = drc(pcb, tmp) checks.append(f"DRC: {d['violations']} errors, {d['unconnected_items']} unconnected, " f"{d['schematic_parity']} schematic parity issues") files.insert(0, (f"{base}-gerbers.zip", gerbers_zip(pcb, dest / f"{base}-gerbers.zip", tmp))) files.append((f"{base}-cpl.csv", cpl(pcb, dest / f"{base}-cpl.csv"))) copper = ",".join(copper_layers(pcb)) cli("pcb", "export", "pdf", "--mode-multipage", "--include-border-title", "--layers", f"{copper},F.Silkscreen,B.Silkscreen,F.Fab,B.Fab", "--common-layers", "Edge.Cuts", "-o", dest / f"{base}-pcb.pdf", pcb) files.append((f"{base}-pcb.pdf", "PCB: one page per copper, silkscreen and fab layer")) for side in ("top", "bottom"): cli("pcb", "render", "--side", side, "--width", RENDER_SIZE[0], "--height", RENDER_SIZE[1], "--quality", "high", "-o", dest / f"{base}-{side}.png", pcb) files.append((f"{base}-{side}.png", f"3D render, {side}")) if not files: raise Failed(f"{name}: neither {sch.name} nor {pcb.name}") return name, files, checks def unicode_header(extra): lines = [rf"\DeclareUnicodeCharacter{{{c:04X}}}{{{v}}}" for c, v in sorted({**UNICODE, **extra}.items())] return "\n".join(lines) + "\n" def footer(text): return ("\\usepackage{fancyhdr}\n\\pagestyle{fancy}\n\\fancyhf{}\n\\renewcommand{\\headrulewidth}{0pt}\n" f"\\fancyfoot[L]{{\\footnotesize {text}}}\n\\fancyfoot[R]{{\\footnotesize\\thepage}}\n" "\\fancypagestyle{plain}{\\pagestyle{fancy}}\n") def markdown_pdf(md, dest, foot, tmp): """pandoc + pdflatex; characters pdflatex refuses are retried as '?'. Returns the replaced ones.""" extra = {} header = tmp / "header.tex" for _ in range(40): header.write_text(footer(foot) + unicode_header(extra), encoding="utf-8") r = run(["pandoc", md.name, "-o", dest, "--pdf-engine=pdflatex", "-H", header, "-V", "geometry:margin=2cm", "-V", "colorlinks=true", "--resource-path", "."], cwd=md.parent, check=False) if r.returncode == 0: return sorted(extra) m = re.search(r"Unicode character .*?\(U\+([0-9A-Fa-f]+)\)", r.stdout + r.stderr, re.S) if not m: raise Failed((r.stdout + r.stderr)[-1200:]) extra[int(m.group(1), 16)] = "?" raise Failed("too many characters pdflatex cannot print") def latex_pdf(tex, dest, tmp): work = tmp / "tex" shutil.copytree(tex.parent, work) for i in range(3): r = run(["pdflatex", "-interaction=nonstopmode", "-halt-on-error", tex.name], cwd=work, check=False) if r.returncode != 0: raise Failed(r.stdout[-1200:]) aux = work / f"{tex.stem}.aux" if i == 0 and aux.exists() and "\\bibdata" in aux.read_text(encoding="utf-8", errors="replace"): run(["bibtex", tex.stem], cwd=work, check=False) shutil.copyfile(work / f"{tex.stem}.pdf", dest) def documents(version, foot, out): """docs/*.md (not README.md, input.md) and docs/tex/*.tex with a \\documentclass, as PDF.""" made, failed = [], [] docs = ROOT / "docs" sources = [p for p in sorted(docs.glob("*.md")) if p.name.lower() not in ("readme.md", "input.md")] sources += [p for p in sorted((docs / "tex").glob("*.tex")) if "\\documentclass" in p.read_text(encoding="utf-8", errors="replace")] names = {} for src in sources: stem = src.stem if src.stem not in names else f"{src.stem}-{src.suffix[1:]}" names[stem] = src dest = out / "docs" / f"{stem}-{version}.pdf" dest.parent.mkdir(exist_ok=True) rel = src.relative_to(ROOT).as_posix() with tempfile.TemporaryDirectory() as t: try: if src.suffix == ".md": replaced = markdown_pdf(src, dest, foot, Path(t)) note = f"from `{rel}`" + (f"; printed as ?: {', '.join(f'U+{c:04X}' for c in replaced)}" if replaced else "") else: latex_pdf(src, dest, Path(t)) note = f"from `{rel}`" made.append((dest.name, note)) print(f" {dest.relative_to(out).as_posix()}") except Failed as e: dest.unlink(missing_ok=True) lines = str(e).strip().splitlines() or ["failed"] why = next((l for l in lines if l.startswith("! ")), lines[-1])[:200] failed.append((rel, why)) print(f"::warning::{rel} did not build: {why}") print(" " + "\n ".join(lines[-15:])) return made, failed def changelog(version): path = ROOT / "CHANGELOG.md" if not path.exists(): return "" m = re.search(rf"^## {re.escape(version)}\b.*?$(.*?)(?=^## |\Z)", path.read_text(encoding="utf-8"), re.M | re.S) return m.group(1).strip() if m else "" def size(p): n = p.stat().st_size return f"{n / 1e6:.1f} MB" if n >= 1e6 else f"{max(n / 1e3, 0.1):.0f} kB" def readme(product, version, commit, when, kicad, boards, docs_made, docs_failed, skipped, out): L = [f"# {product} {version}: release files", "", f"The latest prototype stage of {product}: manufacturing files and documentation built by CI from tag " f"`{version}` (commit `{commit}`, {when}, KiCad {kicad}). Every stage tag replaces this folder; " "older stages are on the repository's Releases page with the same files. Do not edit it by hand.", ""] for name, files, checks in boards: L += [f"## Board {name}", "", "; ".join(checks) + "." if checks else "", "", "| File | What | Size |", "|---|---|---|"] L += [f"| [{f}](hardware/{name}/{f}) | {what} | {size(out / 'hardware' / name / f)} |" for f, what in files] L.append("") for d, why in skipped: L += [f"Not in this release: `{d}` ({why}).", ""] if not boards and not skipped: L += ["No board yet.", ""] if docs_made or docs_failed: L += ["## Documentation", "", "| File | What | Size |", "|---|---|---|"] L += [f"| [{f}](docs/{f}) | {what} | {size(out / 'docs' / f)} |" for f, what in docs_made] L += [f"| | `{src}` did not build: {why} | |" for src, why in docs_failed] L.append("") notes = changelog(version) if notes: L += [f"## Changes in {version} (CHANGELOG.md)", "", notes, ""] sums = [f"{hashlib.sha256(p.read_bytes()).hexdigest()} {p.relative_to(out).as_posix()}" for p in sorted(out.rglob("*")) if p.is_file() and p.name != "README.md"] if sums: L += ["## SHA-256", "", "```", *sums, "```", ""] (out / "README.md").write_text("\n".join(L).replace("\n\n\n", "\n\n"), encoding="utf-8", newline="\n") def main(): ap = argparse.ArgumentParser(description="Release files of a prototype stage") ap.add_argument("version", help="the stage tag, e.g. v0.2") ap.add_argument("--out", default="release", help="output folder (default: release)") a = ap.parse_args() if not re.fullmatch(r"[0-9A-Za-z._+-]+", a.version): sys.exit(f"not a version: {a.version!r}") out = Path(a.out) if Path(a.out).is_absolute() else ROOT / a.out m = manifest() product = (m.get("product") or {}).get("name") or os.environ.get("GITHUB_REPOSITORY", "").split("/")[-1] or ROOT.name commit = git("rev-parse", "--short", "HEAD") or "?" when = f"{datetime.datetime.now(datetime.timezone.utc):%Y-%m-%d %H:%M} UTC" kicad = cli("version").strip() clear(out) boards, skipped, ok = [], [], True for board_dir, pro in projects(m): rel = board_dir.relative_to(ROOT).as_posix() if pro is None: print(f"== {rel}: no KiCad project yet") skipped.append((rel, "no KiCad project yet")) continue print(f"== {pro.stem}") with tempfile.TemporaryDirectory() as t: try: name, files, checks = board(board_dir, pro, a.version, out, Path(t)) except Failed as e: print(f"::error::{pro.stem}: {e}") ok = False continue boards.append((name, files, checks)) for c in checks: print(f" {c}") if re.search(r"[1-9]\d* (errors|unconnected|schematic)", c): print(f"::warning::{name} {c}") for f, _ in files: print(f" hardware/{name}/{f}") print("== documentation") foot = tex_escape(f"{product} {a.version}, {commit}, {when[:10]}") docs_made, docs_failed = documents(a.version, foot, out) readme(product, a.version, commit, when, kicad, boards, docs_made, docs_failed, skipped, out) print(f"== {out}: {sum(len(f) for _, f, _ in boards)} board files, {len(docs_made)} documents") sys.exit(0 if ok else 1) if __name__ == "__main__": main()