diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..4b72413 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,12 @@ +# release/: CI's files, kept byte for byte on every checkout (release/README.md +# lists their SHA-256) +release/** -text +# CI scripts run in the Linux CI image +.gitea/scripts/* text eol=lf +*.sh text eol=lf +# binary files +*.png binary +*.jpg binary +*.pdf binary +*.zip binary +*.step binary diff --git a/.gitea/scripts/release-attach.py b/.gitea/scripts/release-attach.py new file mode 100644 index 0000000..54c4457 --- /dev/null +++ b/.gitea/scripts/release-attach.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Attach the files of a release folder (release-files.py) to the Gitea release of the tag. + + TOKEN= python3 .gitea/scripts/release-attach.py /tmp/release + +Reads GITHUB_SERVER_URL, GITHUB_REPOSITORY and GITHUB_REF_NAME (the tag), as Gitea Actions sets +them. Creates the release when the tag has none, with the tag's CHANGELOG.md section as its text. +Uploads every file of the folder except README.md, and -.zip with the whole folder; +an asset of the same name from an earlier run of the tag is replaced. Used by +.gitea/workflows/release.yml. +""" +import io +import json +import os +import re +import sys +import urllib.error +import urllib.parse +import urllib.request +import uuid +import zipfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +ZIP_TIME = (2020, 1, 1, 0, 0, 0) + + +def changelog(tag): + path = ROOT / "CHANGELOG.md" + if not path.exists(): + return "" + m = re.search(rf"^## {re.escape(tag)}\b.*?$(.*?)(?=^## |\Z)", path.read_text(encoding="utf-8"), re.M | re.S) + return m.group(1).strip() if m else "" + + +def bundle(folder, top): + """The whole folder as a zip under /, with fixed timestamps.""" + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as z: + for p in sorted(folder.rglob("*")): + if p.is_file(): + info = zipfile.ZipInfo(f"{top}/{p.relative_to(folder).as_posix()}", date_time=ZIP_TIME) + info.compress_type = zipfile.ZIP_DEFLATED + z.writestr(info, p.read_bytes()) + return buf.getvalue() + + +class Gitea: + def __init__(self, server, repo, token): + self.api = f"{server.rstrip('/')}/api/v1/repos/{repo}" + self.auth = {"Authorization": f"token {token}"} + + def call(self, method, path, body=None, ctype="application/json"): + req = urllib.request.Request(self.api + path, data=body, method=method, + headers={**self.auth, "Content-Type": ctype, "Accept": "application/json"}) + with urllib.request.urlopen(req, timeout=120) as r: + data = r.read() + return json.loads(data) if data.strip() else None + + def release(self, tag): + try: + return self.call("GET", f"/releases/tags/{urllib.parse.quote(tag, safe='')}"), False + except urllib.error.HTTPError as e: + if e.code != 404: + raise + notes = changelog(tag) + body = (f"{notes}\n\n" if notes else "") + "Gerbers, BOM, pick and place, PDFs and documentation: the files below." + return self.call("POST", "/releases", json.dumps({"tag_name": tag, "name": tag, "body": body}).encode()), True + + def upload(self, rel, name, data): + b = uuid.uuid4().hex + head = (f"--{b}\r\nContent-Disposition: form-data; name=\"attachment\"; filename=\"{name}\"\r\n" + "Content-Type: application/octet-stream\r\n\r\n").encode() + body = head + data + f"\r\n--{b}--\r\n".encode() + self.call("POST", f"/releases/{rel['id']}/assets?name={urllib.parse.quote(name)}", body, + f"multipart/form-data; boundary={b}") + + +def main(): + if len(sys.argv) != 2: + sys.exit(__doc__) + folder = Path(sys.argv[1]) + env = {k: os.environ.get(k, "") for k in ("GITHUB_SERVER_URL", "GITHUB_REPOSITORY", "GITHUB_REF_NAME", "TOKEN")} + missing = [k for k, v in env.items() if not v] + if missing: + sys.exit(f"missing environment: {', '.join(missing)}") + tag, repo = env["GITHUB_REF_NAME"], env["GITHUB_REPOSITORY"] + files = [(p.name, p.read_bytes()) for p in sorted(folder.rglob("*")) if p.is_file() and p.name != "README.md"] + top = f"{repo.split('/')[-1]}-{tag}" + files.append((f"{top}.zip", bundle(folder, top))) + names = [n for n, _ in files] + if len(set(names)) != len(names): + sys.exit(f"two files of the same name: {sorted(n for n in names if names.count(n) > 1)}") + + g = Gitea(env["GITHUB_SERVER_URL"], repo, env["TOKEN"]) + rel, created = g.release(tag) + print(("created" if created else "found") + f" release {tag} (id {rel['id']})") + old = {a["name"]: a["id"] for a in rel.get("assets") or []} + for name, data in files: + if name in old: + g.call("DELETE", f"/releases/{rel['id']}/assets/{old[name]}") + g.upload(rel, name, data) + print(f" {'replaced' if name in old else 'attached'} {name} ({len(data):,} bytes)") + + +if __name__ == "__main__": + main() diff --git a/.gitea/scripts/release-files.py b/.gitea/scripts/release-files.py new file mode 100644 index 0000000..615bf1f --- /dev/null +++ b/.gitea/scripts/release-files.py @@ -0,0 +1,391 @@ +#!/usr/bin/env python3 +"""The release files of a prototype stage: every board's manufacturing files and the documentation. + + python3 .gitea/scripts/release-files.py v0.2 [--out release] + +Writes into --out (default release/; existing content is replaced): + + README.md version, commit, checks, file list, SHA-256, changelog + hardware//--gerbers.zip Gerbers and Excellon drill files for the PCB fab + --bom.csv bill of materials, grouped, DNP left out + --cpl.csv pick and place, both sides, mm + --schematic.pdf the schematic + --pcb.pdf one page per copper, silkscreen and fab layer + --top.png 3D render, top + --bottom.png 3D render, bottom + docs/-.pdf docs/*.md (pandoc, pdflatex) and docs/tex/*.tex (pdflatex) + +Boards are the [[boards]] of proplan.toml, or hardware/*/*.kicad_pro when it names none. DRC and ERC +run and their results go into README.md; they do not stop the release, because the tag records what +was ordered. A document that does not build is listed as not built. Exit 1 when the files of a board +could not be made (the workflow then publishes nothing). + +Used by .gitea/workflows/release.yml. Locally, from the repository root, in the CI image: + docker run --rm -v "$PWD:/w" -w /w git.b4l.co.th/b4l/kicad10_auto_node:latest \\ + python3 .gitea/scripts/release-files.py v0.1 --out build/release +""" +import argparse +import csv +import datetime +import hashlib +import io +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import tomllib +import zipfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +KICAD_CLI = os.environ.get("KICAD_CLI") or shutil.which("kicad-cli") or "kicad-cli" +RENDER_SIZE = (os.environ.get("RENDER_WIDTH", "1400"), os.environ.get("RENDER_HEIGHT", "1540")) +ZIP_TIME = (2020, 1, 1, 0, 0, 0) # fixed timestamps: the same files give the same zip +BOM_FIELDS = ["Reference", "Value", "Footprint", "Manufacturer", "MPN", "LCSC", "QUANTITY"] +BOM_LABELS = ["Reference", "Value", "Footprint", "Manufacturer", "MPN", "LCSC", "Qty"] + +# Characters pdflatex does not know, as LaTeX. Anything else it refuses is printed as "?" and +# reported (pdflatex because the image's lualatex cannot load its fonts). +UNICODE = { + 0x2192: r"\ensuremath{\rightarrow}", 0x2190: r"\ensuremath{\leftarrow}", 0x2194: r"\ensuremath{\leftrightarrow}", + 0x2191: r"\ensuremath{\uparrow}", 0x2193: r"\ensuremath{\downarrow}", 0x21D2: r"\ensuremath{\Rightarrow}", + 0x21D0: r"\ensuremath{\Leftarrow}", 0x21D4: r"\ensuremath{\Leftrightarrow}", 0x21A6: r"\ensuremath{\mapsto}", + 0x2264: r"\ensuremath{\leq}", 0x2265: r"\ensuremath{\geq}", 0x2248: r"\ensuremath{\approx}", + 0x2260: r"\ensuremath{\neq}", 0x2261: r"\ensuremath{\equiv}", 0x2212: r"\ensuremath{-}", + 0x221E: r"\ensuremath{\infty}", 0x221A: r"\ensuremath{\surd}", 0x2211: r"\ensuremath{\sum}", + 0x220F: r"\ensuremath{\prod}", 0x2202: r"\ensuremath{\partial}", 0x2208: r"\ensuremath{\in}", + 0x2205: r"\ensuremath{\emptyset}", 0x2227: r"\ensuremath{\wedge}", 0x2228: r"\ensuremath{\vee}", + 0x2295: r"\ensuremath{\oplus}", 0x22C5: r"\ensuremath{\cdot}", 0x2026: r"\ldots{}", + 0x2713: r"\ensuremath{\checkmark}", 0x2714: r"\ensuremath{\checkmark}", 0x2705: r"\ensuremath{\checkmark}", + 0x2717: r"\ensuremath{\times}", 0x2718: r"\ensuremath{\times}", 0x274C: r"\ensuremath{\times}", + 0x2126: r"\ensuremath{\Omega}", 0x2103: r"\textdegree{}C", 0x2032: r"\ensuremath{'}", + 0x2033: r"\ensuremath{''}", 0x2009: r"\,", 0x202F: r"\,", 0x200B: "", 0xFE0F: "", + 0x25B6: r"\ensuremath{\blacktriangleright}", 0x25C0: r"\ensuremath{\blacktriangleleft}", + 0x25B2: r"\ensuremath{\blacktriangle}", 0x25BC: r"\ensuremath{\blacktriangledown}", + 0x25CF: r"\ensuremath{\bullet}", 0x25CB: r"\ensuremath{\circ}", 0x26A0: r"\textbf{!}", + # box drawing (trees in code blocks); \mbox keeps two dashes from becoming an en dash + 0x2500: r"\mbox{-}", 0x2501: r"\mbox{-}", 0x2502: "|", 0x2503: "|", 0x2550: r"\mbox{=}", 0x2551: "|", + **{c: "+" for c in (0x250C, 0x2510, 0x2514, 0x2518, 0x251C, 0x2524, 0x252C, 0x2534, 0x253C, + 0x2554, 0x2557, 0x255A, 0x255D, 0x2560, 0x2563, 0x2566, 0x2569, 0x256C)}, +} +GREEK = "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho - sigma tau upsilon phi chi psi omega" +for i, n in enumerate(GREEK.split()): + if n not in ("-", "omicron"): + UNICODE[0x3B1 + i] = rf"\ensuremath{{\{n}}}" +for c, n in {0x393: "Gamma", 0x394: "Delta", 0x398: "Theta", 0x39B: "Lambda", 0x39E: "Xi", 0x3A0: "Pi", + 0x3A3: "Sigma", 0x3A5: "Upsilon", 0x3A6: "Phi", 0x3A8: "Psi", 0x3A9: "Omega"}.items(): + UNICODE[c] = rf"\ensuremath{{\{n}}}" +UNICODE[0x3C2] = r"\ensuremath{\varsigma}" + + +class Failed(Exception): + pass + + +def run(cmd, cwd=None, check=True): + r = subprocess.run([str(c) for c in cmd], cwd=cwd, capture_output=True, text=True, errors="replace") + if check and r.returncode != 0: + raise Failed(f"{Path(str(cmd[0])).name} {' '.join(map(str, cmd[1:4]))}: exit {r.returncode}\n" + f"{(r.stdout + r.stderr)[-1500:]}") + return r + + +def cli(*args): + return run([KICAD_CLI, *args]).stdout + + +def git(*args): + r = run(["git", "-C", ROOT, *args], check=False) + return r.stdout.strip() if r.returncode == 0 else "" + + +def tex_escape(s): + return re.sub(r"([\\{}$&#^_%~])", lambda m: {"\\": r"\textbackslash{}", "^": r"\^{}", "~": r"\~{}"} + .get(m.group(1), "\\" + m.group(1)), s) + + +def clear(folder): + """Empty a folder but keep it (a bind-mounted folder may refuse to be removed).""" + folder.mkdir(parents=True, exist_ok=True) + for p in folder.iterdir(): + shutil.rmtree(p) if p.is_dir() else p.unlink() + + +def manifest(): + path = ROOT / "proplan.toml" + if not path.exists(): + return {} + try: + return tomllib.loads(path.read_text(encoding="utf-8")) + except tomllib.TOMLDecodeError as e: + print(f"::warning::proplan.toml cannot be read ({e}); taking hardware/*/*.kicad_pro") + return {} + + +def projects(m): + """(board folder, .kicad_pro or None) for every board.""" + dirs = [ROOT / b["path"] for b in m.get("boards", []) if isinstance(b, dict) and isinstance(b.get("path"), str)] + if not dirs: + dirs = sorted({p.parent for p in ROOT.glob("hardware/*/*.kicad_pro")}) + out = [] + for d in dirs: + pros = sorted(d.glob("*.kicad_pro")) + out.append((d, next((p for p in pros if p.stem == d.name), pros[0] if pros else None))) + return out + + +def erc(sch, tmp): + rpt = tmp / "erc.json" + cli("sch", "erc", "--format", "json", "--severity-error", "-o", rpt, sch) + d = json.loads(rpt.read_text(encoding="utf-8")) + return sum(len(s.get("violations", [])) for s in d.get("sheets", [])) + + +def drc(pcb, tmp): + rpt = tmp / "drc.json" + cli("pcb", "drc", "--format", "json", "--severity-error", "--schematic-parity", "-o", rpt, pcb) + d = json.loads(rpt.read_text(encoding="utf-8")) + return {k: len(d.get(k, [])) for k in ("violations", "unconnected_items", "schematic_parity")} + + +def copper_layers(pcb): + return re.findall(r'\(\d+ "([A-Za-z0-9]+\.Cu)" (?:signal|power|mixed|jumper)\)', pcb.read_text(encoding="utf-8")) + + +def gerbers_zip(pcb, dest, tmp): + out = tmp / "gerbers" + out.mkdir() + copper = copper_layers(pcb) + layers = copper + ["F.Paste", "B.Paste", "F.Silkscreen", "B.Silkscreen", "F.Mask", "B.Mask", "Edge.Cuts"] + cli("pcb", "export", "gerbers", "-o", f"{out}/", "--layers", ",".join(layers), "--subtract-soldermask", + "--no-x2", "--no-netlist", "--check-zones", pcb) + cli("pcb", "export", "drill", "-o", f"{out}/", "--format", "excellon", "--drill-origin", "absolute", + "--excellon-units", "mm", "--excellon-zeros-format", "decimal", "--excellon-oval-format", "alternate", + "--excellon-separate-th", pcb) + files = sorted(p for p in out.iterdir() if p.suffix.lower() != ".gbrjob") + drills = [p for p in files if p.suffix.lower() == ".drl"] + if len(files) - len(drills) != len(layers) or not drills: + raise Failed(f"expected {len(layers)} Gerbers and drill files, got {[p.name for p in files]}") + with zipfile.ZipFile(dest, "w", zipfile.ZIP_DEFLATED) as z: + for p in files: + info = zipfile.ZipInfo(p.name, date_time=ZIP_TIME) + info.compress_type = zipfile.ZIP_DEFLATED + z.writestr(info, p.read_bytes()) + return f"Gerbers ({len(copper)} copper layers) and drill files for the PCB fab" + + +def bom(sch, dest): + cli("sch", "export", "bom", "-o", dest, "--fields", ",".join(BOM_FIELDS), "--labels", ",".join(BOM_LABELS), + "--group-by", "Value,Footprint,Manufacturer,MPN,LCSC", "--exclude-dnp", sch) + rows = list(csv.reader(io.StringIO(dest.read_text(encoding="utf-8")))) + return f"bill of materials: {max(len(rows) - 1, 0)} lines, DNP parts left out" + + +def cpl(pcb, dest): + cli("pcb", "export", "pos", "-o", dest, "--format", "csv", "--units", "mm", "--side", "both", "--exclude-dnp", pcb) + rows = list(csv.reader(io.StringIO(dest.read_text(encoding="utf-8")))) + return f"pick and place: {max(len(rows) - 1, 0)} parts, both sides, mm, DNP left out" + + +def board(board_dir, pro, version, out, tmp): + """Write one board's files; returns (name, [(file, what)], checks).""" + name = pro.stem + sch, pcb = board_dir / f"{name}.kicad_sch", board_dir / f"{name}.kicad_pcb" + dest = out / "hardware" / name + dest.mkdir(parents=True) + base = f"{name}-{version}" + files, checks = [], [] + if sch.exists(): + n = erc(sch, tmp) + checks.append(f"ERC: {n} errors") + files.append((f"{base}-schematic.pdf", "schematic")) + cli("sch", "export", "pdf", "-o", dest / files[-1][0], sch) + files.append((f"{base}-bom.csv", bom(sch, dest / f"{base}-bom.csv"))) + if pcb.exists(): + d = drc(pcb, tmp) + checks.append(f"DRC: {d['violations']} errors, {d['unconnected_items']} unconnected, " + f"{d['schematic_parity']} schematic parity issues") + files.insert(0, (f"{base}-gerbers.zip", gerbers_zip(pcb, dest / f"{base}-gerbers.zip", tmp))) + files.append((f"{base}-cpl.csv", cpl(pcb, dest / f"{base}-cpl.csv"))) + copper = ",".join(copper_layers(pcb)) + cli("pcb", "export", "pdf", "--mode-multipage", "--include-border-title", + "--layers", f"{copper},F.Silkscreen,B.Silkscreen,F.Fab,B.Fab", "--common-layers", "Edge.Cuts", + "-o", dest / f"{base}-pcb.pdf", pcb) + files.append((f"{base}-pcb.pdf", "PCB: one page per copper, silkscreen and fab layer")) + for side in ("top", "bottom"): + cli("pcb", "render", "--side", side, "--width", RENDER_SIZE[0], "--height", RENDER_SIZE[1], + "--quality", "high", "-o", dest / f"{base}-{side}.png", pcb) + files.append((f"{base}-{side}.png", f"3D render, {side}")) + if not files: + raise Failed(f"{name}: neither {sch.name} nor {pcb.name}") + return name, files, checks + + +def unicode_header(extra): + lines = [rf"\DeclareUnicodeCharacter{{{c:04X}}}{{{v}}}" for c, v in sorted({**UNICODE, **extra}.items())] + return "\n".join(lines) + "\n" + + +def footer(text): + return ("\\usepackage{fancyhdr}\n\\pagestyle{fancy}\n\\fancyhf{}\n\\renewcommand{\\headrulewidth}{0pt}\n" + f"\\fancyfoot[L]{{\\footnotesize {text}}}\n\\fancyfoot[R]{{\\footnotesize\\thepage}}\n" + "\\fancypagestyle{plain}{\\pagestyle{fancy}}\n") + + +def markdown_pdf(md, dest, foot, tmp): + """pandoc + pdflatex; characters pdflatex refuses are retried as '?'. Returns the replaced ones.""" + extra = {} + header = tmp / "header.tex" + for _ in range(40): + header.write_text(footer(foot) + unicode_header(extra), encoding="utf-8") + r = run(["pandoc", md.name, "-o", dest, "--pdf-engine=pdflatex", "-H", header, + "-V", "geometry:margin=2cm", "-V", "colorlinks=true", "--resource-path", "."], + cwd=md.parent, check=False) + if r.returncode == 0: + return sorted(extra) + m = re.search(r"Unicode character .*?\(U\+([0-9A-Fa-f]+)\)", r.stdout + r.stderr, re.S) + if not m: + raise Failed((r.stdout + r.stderr)[-1200:]) + extra[int(m.group(1), 16)] = "?" + raise Failed("too many characters pdflatex cannot print") + + +def latex_pdf(tex, dest, tmp): + work = tmp / "tex" + shutil.copytree(tex.parent, work) + for i in range(3): + r = run(["pdflatex", "-interaction=nonstopmode", "-halt-on-error", tex.name], cwd=work, check=False) + if r.returncode != 0: + raise Failed(r.stdout[-1200:]) + aux = work / f"{tex.stem}.aux" + if i == 0 and aux.exists() and "\\bibdata" in aux.read_text(encoding="utf-8", errors="replace"): + run(["bibtex", tex.stem], cwd=work, check=False) + shutil.copyfile(work / f"{tex.stem}.pdf", dest) + + +def documents(version, foot, out): + """docs/*.md (not README.md, input.md) and docs/tex/*.tex with a \\documentclass, as PDF.""" + made, failed = [], [] + docs = ROOT / "docs" + sources = [p for p in sorted(docs.glob("*.md")) if p.name.lower() not in ("readme.md", "input.md")] + sources += [p for p in sorted((docs / "tex").glob("*.tex")) + if "\\documentclass" in p.read_text(encoding="utf-8", errors="replace")] + names = {} + for src in sources: + stem = src.stem if src.stem not in names else f"{src.stem}-{src.suffix[1:]}" + names[stem] = src + dest = out / "docs" / f"{stem}-{version}.pdf" + dest.parent.mkdir(exist_ok=True) + rel = src.relative_to(ROOT).as_posix() + with tempfile.TemporaryDirectory() as t: + try: + if src.suffix == ".md": + replaced = markdown_pdf(src, dest, foot, Path(t)) + note = f"from `{rel}`" + (f"; printed as ?: {', '.join(f'U+{c:04X}' for c in replaced)}" + if replaced else "") + else: + latex_pdf(src, dest, Path(t)) + note = f"from `{rel}`" + made.append((dest.name, note)) + print(f" {dest.relative_to(out).as_posix()}") + except Failed as e: + dest.unlink(missing_ok=True) + lines = str(e).strip().splitlines() or ["failed"] + why = next((l for l in lines if l.startswith("! ")), lines[-1])[:200] + failed.append((rel, why)) + print(f"::warning::{rel} did not build: {why}") + print(" " + "\n ".join(lines[-15:])) + return made, failed + + +def changelog(version): + path = ROOT / "CHANGELOG.md" + if not path.exists(): + return "" + m = re.search(rf"^## {re.escape(version)}\b.*?$(.*?)(?=^## |\Z)", path.read_text(encoding="utf-8"), re.M | re.S) + return m.group(1).strip() if m else "" + + +def size(p): + n = p.stat().st_size + return f"{n / 1e6:.1f} MB" if n >= 1e6 else f"{max(n / 1e3, 0.1):.0f} kB" + + +def readme(product, version, commit, when, kicad, boards, docs_made, docs_failed, skipped, out): + L = [f"# {product} {version}: release files", "", + f"The latest prototype stage of {product}: manufacturing files and documentation built by CI from tag " + f"`{version}` (commit `{commit}`, {when}, KiCad {kicad}). Every stage tag replaces this folder; " + "older stages are on the repository's Releases page with the same files. Do not edit it by hand.", ""] + for name, files, checks in boards: + L += [f"## Board {name}", "", "; ".join(checks) + "." if checks else "", "", + "| File | What | Size |", "|---|---|---|"] + L += [f"| [{f}](hardware/{name}/{f}) | {what} | {size(out / 'hardware' / name / f)} |" for f, what in files] + L.append("") + for d, why in skipped: + L += [f"Not in this release: `{d}` ({why}).", ""] + if not boards and not skipped: + L += ["No board yet.", ""] + if docs_made or docs_failed: + L += ["## Documentation", "", "| File | What | Size |", "|---|---|---|"] + L += [f"| [{f}](docs/{f}) | {what} | {size(out / 'docs' / f)} |" for f, what in docs_made] + L += [f"| | `{src}` did not build: {why} | |" for src, why in docs_failed] + L.append("") + notes = changelog(version) + if notes: + L += [f"## Changes in {version} (CHANGELOG.md)", "", notes, ""] + sums = [f"{hashlib.sha256(p.read_bytes()).hexdigest()} {p.relative_to(out).as_posix()}" + for p in sorted(out.rglob("*")) if p.is_file() and p.name != "README.md"] + if sums: + L += ["## SHA-256", "", "```", *sums, "```", ""] + (out / "README.md").write_text("\n".join(L).replace("\n\n\n", "\n\n"), encoding="utf-8", newline="\n") + + +def main(): + ap = argparse.ArgumentParser(description="Release files of a prototype stage") + ap.add_argument("version", help="the stage tag, e.g. v0.2") + ap.add_argument("--out", default="release", help="output folder (default: release)") + a = ap.parse_args() + if not re.fullmatch(r"[0-9A-Za-z._+-]+", a.version): + sys.exit(f"not a version: {a.version!r}") + out = Path(a.out) if Path(a.out).is_absolute() else ROOT / a.out + m = manifest() + product = (m.get("product") or {}).get("name") or os.environ.get("GITHUB_REPOSITORY", "").split("/")[-1] or ROOT.name + commit = git("rev-parse", "--short", "HEAD") or "?" + when = f"{datetime.datetime.now(datetime.timezone.utc):%Y-%m-%d %H:%M} UTC" + kicad = cli("version").strip() + clear(out) + boards, skipped, ok = [], [], True + for board_dir, pro in projects(m): + rel = board_dir.relative_to(ROOT).as_posix() + if pro is None: + print(f"== {rel}: no KiCad project yet") + skipped.append((rel, "no KiCad project yet")) + continue + print(f"== {pro.stem}") + with tempfile.TemporaryDirectory() as t: + try: + name, files, checks = board(board_dir, pro, a.version, out, Path(t)) + except Failed as e: + print(f"::error::{pro.stem}: {e}") + ok = False + continue + boards.append((name, files, checks)) + for c in checks: + print(f" {c}") + if re.search(r"[1-9]\d* (errors|unconnected|schematic)", c): + print(f"::warning::{name} {c}") + for f, _ in files: + print(f" hardware/{name}/{f}") + print("== documentation") + foot = tex_escape(f"{product} {a.version}, {commit}, {when[:10]}") + docs_made, docs_failed = documents(a.version, foot, out) + readme(product, a.version, commit, when, kicad, boards, docs_made, docs_failed, skipped, out) + print(f"== {out}: {sum(len(f) for _, f, _ in boards)} board files, {len(docs_made)} documents") + sys.exit(0 if ok else 1) + + +if __name__ == "__main__": + main() diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index b4b4df3..d445f06 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -1,9 +1,14 @@ -name: Release outputs +name: Release -# On a tag v* (a prototype stage): the KiBot outputs of every board, zipped per -# board, attached to the tag's release (created when missing). The job token -# must be allowed to write releases. Adapted from LVSolarBuck; runs for the -# first time at the first tag. +# On a tag v* (a prototype stage), .gitea/scripts/release-files.py builds the +# stage's release files: for every board Gerbers and drill (zip), BOM, pick +# and place, schematic and PCB PDFs and 3D renders; the documentation +# (docs/*.md, docs/tex/*.tex) as PDF. They are attached to the tag's release +# (created when missing) and, when the tag is the newest stage, committed to +# release/ on main, so release/ always holds the latest stage. The commit +# carries [skip ci] and touches no path another workflow watches. The job +# token must be allowed to write releases and to push to main. Firmware and +# software tags (firmware/..., software/...) do not start it. on: push: tags: ['v*'] @@ -11,47 +16,54 @@ on: jobs: release: runs-on: kicad10_auto + permissions: + contents: write + concurrency: + group: release-${{ github.repository }} + cancel-in-progress: false steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - - name: KiBot outputs per board - run: | - mkdir -p dist - for pro in $(find hardware -mindepth 2 -maxdepth 2 -name '*.kicad_pro' | sort); do - dir=$(dirname "$pro"); board=$(basename "$dir") - (cd "$dir" && kibot -c "$GITHUB_WORKSPACE/kibot.yaml" -d "$GITHUB_WORKSPACE/out/$board") \ - || echo "::warning::KiBot reported problems for $board" - python3 -c 'import shutil, sys; shutil.make_archive(sys.argv[1], "zip", sys.argv[2])' \ - "dist/$board-${GITHUB_REF_NAME}" "out/$board" - done - ls -la dist + - name: Release files + run: python3 .gitea/scripts/release-files.py "$GITHUB_REF_NAME" --out /tmp/release - name: Attach to the release env: TOKEN: ${{ secrets.GITEA_TOKEN }} + run: python3 .gitea/scripts/release-attach.py /tmp/release + + - name: Commit to release/ on main + env: + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} run: | - python3 - <<'EOF' - import json, os, pathlib, urllib.error, urllib.request, uuid - api = f"{os.environ['GITHUB_SERVER_URL']}/api/v1/repos/{os.environ['GITHUB_REPOSITORY']}" - tag = os.environ["GITHUB_REF_NAME"] - auth = {"Authorization": f"token {os.environ['TOKEN']}"} - - def call(method, url, body=None, ctype="application/json"): - req = urllib.request.Request(url, data=body, method=method, headers={**auth, "Content-Type": ctype}) - with urllib.request.urlopen(req) as r: - return json.load(r) - - try: - rel = call("GET", f"{api}/releases/tags/{tag}") - except urllib.error.HTTPError as e: - if e.code != 404: - raise - rel = call("POST", f"{api}/releases", json.dumps({"tag_name": tag, "name": tag}).encode()) - for f in sorted(pathlib.Path("dist").glob("*.zip")): - b = uuid.uuid4().hex - head = (f"--{b}\r\nContent-Disposition: form-data; name=\"attachment\"; filename=\"{f.name}\"\r\n" - "Content-Type: application/zip\r\n\r\n").encode() - body = head + f.read_bytes() + f"\r\n--{b}--\r\n".encode() - call("POST", f"{api}/releases/{rel['id']}/assets?name={f.name}", body, f"multipart/form-data; boundary={b}") - print("attached", f.name) - EOF + tag="$GITHUB_REF_NAME" + # stage tags only (v0.1, v0.2, v1.0); a pre-release such as v0.3-rc.1 leaves release/ alone + newest=$(git tag -l 'v*' --sort=-v:refname | grep -E '^v[0-9]+(\.[0-9]+)+$' | head -1 || true) + if [ "$tag" != "$newest" ]; then + echo "$tag is not the newest stage (${newest:-none}): release/ is left as it is." + exit 0 + fi + git config user.name "Release CI" + git config user.email "ci@noreply.git.b4l.co.th" + REPO_URL=$(git config --get remote.origin.url) + AUTHED_URL=${REPO_URL/https:\/\//https:\/\/${GITEA_TOKEN}@} + # main may move while this runs: start again from the newest main + for i in 1 2 3; do + git fetch -q "$AUTHED_URL" main + git checkout -q -f -B main FETCH_HEAD + rm -rf release + mkdir release + cp -r /tmp/release/. release/ + git add -A -- release + if git diff --cached --quiet; then echo "release/ already holds $tag."; exit 0; fi + git commit -q -m "Release $tag: manufacturing files and documentation [skip ci]" + if git push -q "$AUTHED_URL" HEAD:main; then + echo "Pushed $(git rev-parse --short HEAD): release/ holds $tag" + exit 0 + fi + sleep 5 + done + echo "::error::Could not push release/ to main" + exit 1 diff --git a/.gitignore b/.gitignore index dfae03e..3541631 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,9 @@ node_modules/ Generated/ Fabrication/ kibot_output_*/ +# ...except release/, the newest stage's files, which CI commits (release.yml). +# Without this line Release/ above would hide it on Windows (case-insensitive). +!/release/ # OS .DS_Store diff --git a/README.md b/README.md index 5d26bfa..29396d1 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ TEMPLATE.md explains the layout, the rules and `proplan.toml`. | Folder | What goes there | |---|---| +| `release/` | **the latest stage's files**: Gerbers, BOM, pick and place, PDFs, renders and documentation; written by CI at every stage tag, never by hand | | `assets/` | the product image (`assets/product.png`, shown by proplan), renders | | `hardware/$REPO_NAME/` | the KiCad project of the board | | `hardware/lib/` | symbols, footprints and 3D models shared by the boards | @@ -20,9 +21,11 @@ TEMPLATE.md explains the layout, the rules and `proplan.toml`. ## Prototype stages -Each prototype is a git tag (`v0.1`, `v0.2`, ...). CI builds gerbers, BOM and -schematic PDFs for the tag and attaches them to its release; CHANGELOG.md says -what each stage changes. In proplan the prototype stage links the tag. +Each prototype is a git tag (`v0.1`, `v0.2`, ...). For the tag, CI builds the +Gerbers and drill files, BOM, pick and place, schematic and PCB PDFs, 3D renders +and the documentation as PDF, attaches them to the tag's release and puts them +into `release/`, which always holds the newest stage. CHANGELOG.md says what +each stage changes. In proplan the prototype stage links the tag. ## Versioning diff --git a/TEMPLATE.md b/TEMPLATE.md index 6901eb9..6b96352 100644 --- a/TEMPLATE.md +++ b/TEMPLATE.md @@ -20,7 +20,8 @@ description. Gitea puts the name into `README.md`, `CHANGELOG.md`, project file is `hardware//.kicad_pro`). 3. Commit and push. The KiBot workflow checks the board on every push. 4. Tag the first prototype `v0.1` when its boards are ordered, and link the - tag to the stage on the project's Prototypes tab in proplan. + tag to the stage on the project's Prototypes tab in proplan. CI then puts + the stage's manufacturing files and documentation into `release/`. ## Layout @@ -28,6 +29,7 @@ description. Gitea puts the name into `README.md`, `CHANGELOG.md`, |---|---| | `proplan.toml` | the manifest proplan reads (below) | | `README.md`, `CHANGELOG.md` | what the product is; one changelog section per stage (`## v0.2`) | +| `release/` | the newest stage's manufacturing files and documentation, written by CI (below) | | `assets/product.png` | the product image | | `hardware//` | one KiCad project per board | | `hardware/lib/` | symbols, footprints and 3D models shared by the boards | @@ -47,7 +49,9 @@ description. Gitea puts the name into `README.md`, `CHANGELOG.md`, (48 V and 24 V, 2-board and 3-board) is its own folder under `hardware/` and its own `[[boards]]` entry. - Generated output (gerbers, BOMs, production zips, PDFs, plots) is built by CI - and attached to the release; it is not committed. + and attached to the release; it is not committed. The one exception is + `release/`, which CI fills at a stage tag: nothing goes there by hand, and + generated files go nowhere else. - Vendor datasheets go to `docs/datasheets/`, not the repository root. - Third-party code is a git submodule in `external/` and is not edited here. - IDE build output and tool caches (`Debug/`, `__pycache__/`, `.claude/`, @@ -98,9 +102,53 @@ For all three: folder. The root `CHANGELOG.md` has one section per prototype stage and lists the firmware and software versions that stage was built with. - Firmware and software releases are made by hand in Gitea for now (push the - tag, then New Release); `release.yml` builds board outputs for `v*` tags + tag, then New Release); `release.yml` builds the release files for `v*` tags only. +## Release files (`release/`) + +`release/` always holds the newest prototype stage, ready to send to a +manufacturer or a customer: whoever needs the current boards or documents +takes them from there, without KiCad and without knowing the tags. CI writes +it (`.gitea/workflows/release.yml`, `.gitea/scripts/release-files.py`) when a +stage tag is pushed; nobody edits it. + +| Path | What | +|---|---| +| `release/README.md` | tag, commit, date, KiCad version, ERC and DRC result per board, the file list with sizes and SHA-256, the stage's CHANGELOG section | +| `release/hardware//--gerbers.zip` | Gerbers (every copper layer, paste, silkscreen, mask, outline; Protel names, no X2) and Excellon drill files (mm, PTH and NPTH apart): what a PCB fab such as JLCPCB takes | +| `release/hardware//--bom.csv` | bill of materials grouped by value, footprint, manufacturer, MPN and LCSC; DNP parts left out | +| `release/hardware//--cpl.csv` | pick and place (KiCad position file, mm, both sides, DNP left out) | +| `release/hardware//--schematic.pdf`, `-pcb.pdf` | the schematic; the PCB with one page per copper, silkscreen and fab layer | +| `release/hardware//--top.png`, `-bottom.png` | 3D renders | +| `release/docs/-.pdf` | every `docs/*.md` except `README.md` and `input.md` (pandoc), and every `docs/tex/*.tex` with a `\documentclass` (pdflatex); the footer carries product, tag, commit and date | + +- Boards are the `[[boards]]` of `proplan.toml` (or `hardware/*/*.kicad_pro` + when it names none), so a repository that adopted the layout without moving + folders gets its release files too. +- The tag is in every file name: a file sent to a fab or forwarded by mail + still says which stage it is. +- The same files, plus one zip of the whole folder, are attached to the tag's + release, so older stages stay one click away. `release/` follows the newest + stage tag only: a tag of an older stage or a pre-release (`v0.3-rc.1`) gets + its release attachments but leaves `release/` as it is. +- ERC and DRC errors do not stop the release (the tag records what was + ordered), but they are printed in `release/README.md` and as warnings on the + workflow run. Check the KiBot run before tagging. +- A document that does not build is listed in `release/README.md` as not + built; characters pdflatex cannot print (emoji) come out as `?` and are + named there. +- The commit says `[skip ci]` and `release/` is outside every workflow's + paths, so it starts no other workflow. `.gitattributes` keeps the files + byte for byte (`release/** -text`), so the SHA-256 match on Windows too. +- Locally, from the repository root, in the CI image (Docker): + `docker run --rm -v "$PWD:/w" -w /w git.b4l.co.th/b4l/kicad10_auto_node:latest python3 .gitea/scripts/release-files.py v0.1 --out build/release` + writes the same files to `build/` (ignored by git). + +Not in it (yet): firmware binaries (firmware has its own tags), STEP models +(`mech/`), JLCPCB-specific BOM/CPL with rotation corrections (harNES's +`jlcpcb-files.py` does that per push). + ## proplan.toml proplan reads this file and the image it names, nothing else of the @@ -204,9 +252,12 @@ external = "external" ERC and DRC and builds schematic PDF, gerbers, drill files, pick and place and BOM for every board (`hardware/*/*.kicad_pro`); the outputs are kept as a workflow artifact. -- `.gitea/workflows/release.yml`: on a tag `v*`, the same outputs are zipped - per board and attached to the tag's release (created when missing). The - Actions token must be allowed to write releases. Firmware and software tags +- `.gitea/workflows/release.yml`: on a tag `v*`, builds the release files + (above) with kicad-cli, pandoc and pdflatex, attaches them to the tag's + release (created when missing, with the stage's CHANGELOG section) and, for + the newest stage, commits them to `release/` on main. The Actions token must + be allowed to write releases and to push to main (if main is protected, its + push rules must let the Actions bot through). Firmware and software tags (`firmware/...`, `software/...`) do not start it. - `.gitea/workflows/docgen.yml`: with a `docs/input.md` (typecad-docgen front matter, as in PowerModule), builds the fabrication document of the first diff --git a/docs/README.md b/docs/README.md index 951fb52..e942ea1 100644 --- a/docs/README.md +++ b/docs/README.md @@ -3,3 +3,7 @@ Datasheets of the parts in `docs/datasheets/`, LaTeX sources in `docs/tex/`. A `docs/input.md` (typecad-docgen front matter) turns on the fabrication document workflow. + +At every stage tag CI prints each `docs/*.md` (except this README and +`input.md`) and each LaTeX document in `docs/tex/` as PDF into `release/docs/`, +with the tag in the file name and the footer. diff --git a/docs/tex/README.md b/docs/tex/README.md index b0dfbad..b5ab8dc 100644 --- a/docs/tex/README.md +++ b/docs/tex/README.md @@ -1,3 +1,6 @@ # LaTeX sources -Sources of the developer documentation. +Sources of the developer documentation. Every `.tex` file here with a +`\documentclass` is built with pdflatex (and bibtex when it cites) into +`release/docs/` at a stage tag; included parts without one are not built on +their own. diff --git a/hardware/$REPO_NAME/README.md b/hardware/$REPO_NAME/README.md index 9ddbd4f..8325377 100644 --- a/hardware/$REPO_NAME/README.md +++ b/hardware/$REPO_NAME/README.md @@ -7,5 +7,6 @@ Project) as `hardware/$REPO_NAME/$REPO_NAME.kicad_pro`. - A real variant built next to it (48 V, 2-board) gets its own folder under `hardware/` and its own `[[boards]]` entry in `proplan.toml`. - Shared symbols, footprints and 3D models go to `hardware/lib/`. -- Gerbers, BOMs and PDFs are built by CI (`.gitea/workflows/kibot.yml`); do not - commit them. +- Gerbers, BOMs and PDFs are built by CI, never committed by hand: KiBot checks + every push (`.gitea/workflows/kibot.yml`), and a stage tag puts the board's + files into `release/hardware/$REPO_NAME/` (`.gitea/workflows/release.yml`). diff --git a/release/README.md b/release/README.md new file mode 100644 index 0000000..bfb585e --- /dev/null +++ b/release/README.md @@ -0,0 +1,14 @@ +# Release files + +The latest prototype stage of this product: its manufacturing files and its +documentation. CI (`.gitea/workflows/release.yml`) replaces this folder when a +stage tag (`v0.1`, `v0.2`, ...) is pushed: + +- `hardware//`: Gerbers and drill files (zip), BOM, pick and place, + schematic and PCB PDFs, 3D renders of every board; +- `docs/`: `docs/*.md` and `docs/tex/*.tex` as PDF; +- this README: the tag, the commit, the ERC/DRC results, the file list with + SHA-256 and the stage's CHANGELOG section. + +Empty until the first stage is tagged. Do not edit it by hand: the next tag +replaces it. Older stages are on the Releases page, with the same files.