Files
product-template/.gitea/scripts/release-files.py
T
grabowski 52a40c1c3e release/: the newest stage's manufacturing files and documentation
A stage tag (v*) now builds, with kicad-cli, pandoc and pdflatex in the CI
image (.gitea/scripts/release-files.py):
- per board: Gerbers + Excellon drill zip (JLCPCB-compatible settings), BOM
  (grouped, DNP left out), pick and place, schematic and PCB PDFs, 3D renders;
- docs/*.md and docs/tex/*.tex as PDF, tag and commit in the footer;
- release/README.md with tag, commit, ERC/DRC result, files, SHA-256 and the
  stage's CHANGELOG section.
Every file name carries the tag. release-attach.py attaches the files plus a
zip of the whole folder to the tag's release (replacing assets of an earlier
run); for the newest stage tag (not older stages, not pre-releases) the job
commits them to release/ on main with [skip ci].

.gitignore: !/release/ after Release/ (Windows matches it case-insensitively);
.gitattributes: release/** -text so the SHA-256 match on every checkout.
TEMPLATE.md: "Release files" section, rules and CI updated.

Tested in git.b4l.co.th/b4l/kicad10_auto_node with harNES's board and docs:
v0.1 committed, v0.0 and v0.2-rc.1 left release/ alone, v0.2 tagged on an
older commit landed on top of a moved main; uploads byte-exact against a stub
of the Gitea release API.
2026-10-16 18:50:01 +07:00

392 lines
19 KiB
Python

#!/usr/bin/env python3
"""The release files of a prototype stage: every board's manufacturing files and the documentation.
python3 .gitea/scripts/release-files.py v0.2 [--out release]
Writes into --out (default release/; existing content is replaced):
README.md version, commit, checks, file list, SHA-256, changelog
hardware/<board>/<board>-<version>-gerbers.zip Gerbers and Excellon drill files for the PCB fab
<board>-<version>-bom.csv bill of materials, grouped, DNP left out
<board>-<version>-cpl.csv pick and place, both sides, mm
<board>-<version>-schematic.pdf the schematic
<board>-<version>-pcb.pdf one page per copper, silkscreen and fab layer
<board>-<version>-top.png 3D render, top
<board>-<version>-bottom.png 3D render, bottom
docs/<name>-<version>.pdf docs/*.md (pandoc, pdflatex) and docs/tex/*.tex (pdflatex)
Boards are the [[boards]] of proplan.toml, or hardware/*/*.kicad_pro when it names none. DRC and ERC
run and their results go into README.md; they do not stop the release, because the tag records what
was ordered. A document that does not build is listed as not built. Exit 1 when the files of a board
could not be made (the workflow then publishes nothing).
Used by .gitea/workflows/release.yml. Locally, from the repository root, in the CI image:
docker run --rm -v "$PWD:/w" -w /w git.b4l.co.th/b4l/kicad10_auto_node:latest \\
python3 .gitea/scripts/release-files.py v0.1 --out build/release
"""
import argparse
import csv
import datetime
import hashlib
import io
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
import tomllib
import zipfile
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
KICAD_CLI = os.environ.get("KICAD_CLI") or shutil.which("kicad-cli") or "kicad-cli"
RENDER_SIZE = (os.environ.get("RENDER_WIDTH", "1400"), os.environ.get("RENDER_HEIGHT", "1540"))
ZIP_TIME = (2020, 1, 1, 0, 0, 0) # fixed timestamps: the same files give the same zip
BOM_FIELDS = ["Reference", "Value", "Footprint", "Manufacturer", "MPN", "LCSC", "QUANTITY"]
BOM_LABELS = ["Reference", "Value", "Footprint", "Manufacturer", "MPN", "LCSC", "Qty"]
# Characters pdflatex does not know, as LaTeX. Anything else it refuses is printed as "?" and
# reported (pdflatex because the image's lualatex cannot load its fonts).
UNICODE = {
0x2192: r"\ensuremath{\rightarrow}", 0x2190: r"\ensuremath{\leftarrow}", 0x2194: r"\ensuremath{\leftrightarrow}",
0x2191: r"\ensuremath{\uparrow}", 0x2193: r"\ensuremath{\downarrow}", 0x21D2: r"\ensuremath{\Rightarrow}",
0x21D0: r"\ensuremath{\Leftarrow}", 0x21D4: r"\ensuremath{\Leftrightarrow}", 0x21A6: r"\ensuremath{\mapsto}",
0x2264: r"\ensuremath{\leq}", 0x2265: r"\ensuremath{\geq}", 0x2248: r"\ensuremath{\approx}",
0x2260: r"\ensuremath{\neq}", 0x2261: r"\ensuremath{\equiv}", 0x2212: r"\ensuremath{-}",
0x221E: r"\ensuremath{\infty}", 0x221A: r"\ensuremath{\surd}", 0x2211: r"\ensuremath{\sum}",
0x220F: r"\ensuremath{\prod}", 0x2202: r"\ensuremath{\partial}", 0x2208: r"\ensuremath{\in}",
0x2205: r"\ensuremath{\emptyset}", 0x2227: r"\ensuremath{\wedge}", 0x2228: r"\ensuremath{\vee}",
0x2295: r"\ensuremath{\oplus}", 0x22C5: r"\ensuremath{\cdot}", 0x2026: r"\ldots{}",
0x2713: r"\ensuremath{\checkmark}", 0x2714: r"\ensuremath{\checkmark}", 0x2705: r"\ensuremath{\checkmark}",
0x2717: r"\ensuremath{\times}", 0x2718: r"\ensuremath{\times}", 0x274C: r"\ensuremath{\times}",
0x2126: r"\ensuremath{\Omega}", 0x2103: r"\textdegree{}C", 0x2032: r"\ensuremath{'}",
0x2033: r"\ensuremath{''}", 0x2009: r"\,", 0x202F: r"\,", 0x200B: "", 0xFE0F: "",
0x25B6: r"\ensuremath{\blacktriangleright}", 0x25C0: r"\ensuremath{\blacktriangleleft}",
0x25B2: r"\ensuremath{\blacktriangle}", 0x25BC: r"\ensuremath{\blacktriangledown}",
0x25CF: r"\ensuremath{\bullet}", 0x25CB: r"\ensuremath{\circ}", 0x26A0: r"\textbf{!}",
# box drawing (trees in code blocks); \mbox keeps two dashes from becoming an en dash
0x2500: r"\mbox{-}", 0x2501: r"\mbox{-}", 0x2502: "|", 0x2503: "|", 0x2550: r"\mbox{=}", 0x2551: "|",
**{c: "+" for c in (0x250C, 0x2510, 0x2514, 0x2518, 0x251C, 0x2524, 0x252C, 0x2534, 0x253C,
0x2554, 0x2557, 0x255A, 0x255D, 0x2560, 0x2563, 0x2566, 0x2569, 0x256C)},
}
GREEK = "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho - sigma tau upsilon phi chi psi omega"
for i, n in enumerate(GREEK.split()):
if n not in ("-", "omicron"):
UNICODE[0x3B1 + i] = rf"\ensuremath{{\{n}}}"
for c, n in {0x393: "Gamma", 0x394: "Delta", 0x398: "Theta", 0x39B: "Lambda", 0x39E: "Xi", 0x3A0: "Pi",
0x3A3: "Sigma", 0x3A5: "Upsilon", 0x3A6: "Phi", 0x3A8: "Psi", 0x3A9: "Omega"}.items():
UNICODE[c] = rf"\ensuremath{{\{n}}}"
UNICODE[0x3C2] = r"\ensuremath{\varsigma}"
class Failed(Exception):
pass
def run(cmd, cwd=None, check=True):
r = subprocess.run([str(c) for c in cmd], cwd=cwd, capture_output=True, text=True, errors="replace")
if check and r.returncode != 0:
raise Failed(f"{Path(str(cmd[0])).name} {' '.join(map(str, cmd[1:4]))}: exit {r.returncode}\n"
f"{(r.stdout + r.stderr)[-1500:]}")
return r
def cli(*args):
return run([KICAD_CLI, *args]).stdout
def git(*args):
r = run(["git", "-C", ROOT, *args], check=False)
return r.stdout.strip() if r.returncode == 0 else ""
def tex_escape(s):
return re.sub(r"([\\{}$&#^_%~])", lambda m: {"\\": r"\textbackslash{}", "^": r"\^{}", "~": r"\~{}"}
.get(m.group(1), "\\" + m.group(1)), s)
def clear(folder):
"""Empty a folder but keep it (a bind-mounted folder may refuse to be removed)."""
folder.mkdir(parents=True, exist_ok=True)
for p in folder.iterdir():
shutil.rmtree(p) if p.is_dir() else p.unlink()
def manifest():
path = ROOT / "proplan.toml"
if not path.exists():
return {}
try:
return tomllib.loads(path.read_text(encoding="utf-8"))
except tomllib.TOMLDecodeError as e:
print(f"::warning::proplan.toml cannot be read ({e}); taking hardware/*/*.kicad_pro")
return {}
def projects(m):
"""(board folder, .kicad_pro or None) for every board."""
dirs = [ROOT / b["path"] for b in m.get("boards", []) if isinstance(b, dict) and isinstance(b.get("path"), str)]
if not dirs:
dirs = sorted({p.parent for p in ROOT.glob("hardware/*/*.kicad_pro")})
out = []
for d in dirs:
pros = sorted(d.glob("*.kicad_pro"))
out.append((d, next((p for p in pros if p.stem == d.name), pros[0] if pros else None)))
return out
def erc(sch, tmp):
rpt = tmp / "erc.json"
cli("sch", "erc", "--format", "json", "--severity-error", "-o", rpt, sch)
d = json.loads(rpt.read_text(encoding="utf-8"))
return sum(len(s.get("violations", [])) for s in d.get("sheets", []))
def drc(pcb, tmp):
rpt = tmp / "drc.json"
cli("pcb", "drc", "--format", "json", "--severity-error", "--schematic-parity", "-o", rpt, pcb)
d = json.loads(rpt.read_text(encoding="utf-8"))
return {k: len(d.get(k, [])) for k in ("violations", "unconnected_items", "schematic_parity")}
def copper_layers(pcb):
return re.findall(r'\(\d+ "([A-Za-z0-9]+\.Cu)" (?:signal|power|mixed|jumper)\)', pcb.read_text(encoding="utf-8"))
def gerbers_zip(pcb, dest, tmp):
out = tmp / "gerbers"
out.mkdir()
copper = copper_layers(pcb)
layers = copper + ["F.Paste", "B.Paste", "F.Silkscreen", "B.Silkscreen", "F.Mask", "B.Mask", "Edge.Cuts"]
cli("pcb", "export", "gerbers", "-o", f"{out}/", "--layers", ",".join(layers), "--subtract-soldermask",
"--no-x2", "--no-netlist", "--check-zones", pcb)
cli("pcb", "export", "drill", "-o", f"{out}/", "--format", "excellon", "--drill-origin", "absolute",
"--excellon-units", "mm", "--excellon-zeros-format", "decimal", "--excellon-oval-format", "alternate",
"--excellon-separate-th", pcb)
files = sorted(p for p in out.iterdir() if p.suffix.lower() != ".gbrjob")
drills = [p for p in files if p.suffix.lower() == ".drl"]
if len(files) - len(drills) != len(layers) or not drills:
raise Failed(f"expected {len(layers)} Gerbers and drill files, got {[p.name for p in files]}")
with zipfile.ZipFile(dest, "w", zipfile.ZIP_DEFLATED) as z:
for p in files:
info = zipfile.ZipInfo(p.name, date_time=ZIP_TIME)
info.compress_type = zipfile.ZIP_DEFLATED
z.writestr(info, p.read_bytes())
return f"Gerbers ({len(copper)} copper layers) and drill files for the PCB fab"
def bom(sch, dest):
cli("sch", "export", "bom", "-o", dest, "--fields", ",".join(BOM_FIELDS), "--labels", ",".join(BOM_LABELS),
"--group-by", "Value,Footprint,Manufacturer,MPN,LCSC", "--exclude-dnp", sch)
rows = list(csv.reader(io.StringIO(dest.read_text(encoding="utf-8"))))
return f"bill of materials: {max(len(rows) - 1, 0)} lines, DNP parts left out"
def cpl(pcb, dest):
cli("pcb", "export", "pos", "-o", dest, "--format", "csv", "--units", "mm", "--side", "both", "--exclude-dnp", pcb)
rows = list(csv.reader(io.StringIO(dest.read_text(encoding="utf-8"))))
return f"pick and place: {max(len(rows) - 1, 0)} parts, both sides, mm, DNP left out"
def board(board_dir, pro, version, out, tmp):
"""Write one board's files; returns (name, [(file, what)], checks)."""
name = pro.stem
sch, pcb = board_dir / f"{name}.kicad_sch", board_dir / f"{name}.kicad_pcb"
dest = out / "hardware" / name
dest.mkdir(parents=True)
base = f"{name}-{version}"
files, checks = [], []
if sch.exists():
n = erc(sch, tmp)
checks.append(f"ERC: {n} errors")
files.append((f"{base}-schematic.pdf", "schematic"))
cli("sch", "export", "pdf", "-o", dest / files[-1][0], sch)
files.append((f"{base}-bom.csv", bom(sch, dest / f"{base}-bom.csv")))
if pcb.exists():
d = drc(pcb, tmp)
checks.append(f"DRC: {d['violations']} errors, {d['unconnected_items']} unconnected, "
f"{d['schematic_parity']} schematic parity issues")
files.insert(0, (f"{base}-gerbers.zip", gerbers_zip(pcb, dest / f"{base}-gerbers.zip", tmp)))
files.append((f"{base}-cpl.csv", cpl(pcb, dest / f"{base}-cpl.csv")))
copper = ",".join(copper_layers(pcb))
cli("pcb", "export", "pdf", "--mode-multipage", "--include-border-title",
"--layers", f"{copper},F.Silkscreen,B.Silkscreen,F.Fab,B.Fab", "--common-layers", "Edge.Cuts",
"-o", dest / f"{base}-pcb.pdf", pcb)
files.append((f"{base}-pcb.pdf", "PCB: one page per copper, silkscreen and fab layer"))
for side in ("top", "bottom"):
cli("pcb", "render", "--side", side, "--width", RENDER_SIZE[0], "--height", RENDER_SIZE[1],
"--quality", "high", "-o", dest / f"{base}-{side}.png", pcb)
files.append((f"{base}-{side}.png", f"3D render, {side}"))
if not files:
raise Failed(f"{name}: neither {sch.name} nor {pcb.name}")
return name, files, checks
def unicode_header(extra):
lines = [rf"\DeclareUnicodeCharacter{{{c:04X}}}{{{v}}}" for c, v in sorted({**UNICODE, **extra}.items())]
return "\n".join(lines) + "\n"
def footer(text):
return ("\\usepackage{fancyhdr}\n\\pagestyle{fancy}\n\\fancyhf{}\n\\renewcommand{\\headrulewidth}{0pt}\n"
f"\\fancyfoot[L]{{\\footnotesize {text}}}\n\\fancyfoot[R]{{\\footnotesize\\thepage}}\n"
"\\fancypagestyle{plain}{\\pagestyle{fancy}}\n")
def markdown_pdf(md, dest, foot, tmp):
"""pandoc + pdflatex; characters pdflatex refuses are retried as '?'. Returns the replaced ones."""
extra = {}
header = tmp / "header.tex"
for _ in range(40):
header.write_text(footer(foot) + unicode_header(extra), encoding="utf-8")
r = run(["pandoc", md.name, "-o", dest, "--pdf-engine=pdflatex", "-H", header,
"-V", "geometry:margin=2cm", "-V", "colorlinks=true", "--resource-path", "."],
cwd=md.parent, check=False)
if r.returncode == 0:
return sorted(extra)
m = re.search(r"Unicode character .*?\(U\+([0-9A-Fa-f]+)\)", r.stdout + r.stderr, re.S)
if not m:
raise Failed((r.stdout + r.stderr)[-1200:])
extra[int(m.group(1), 16)] = "?"
raise Failed("too many characters pdflatex cannot print")
def latex_pdf(tex, dest, tmp):
work = tmp / "tex"
shutil.copytree(tex.parent, work)
for i in range(3):
r = run(["pdflatex", "-interaction=nonstopmode", "-halt-on-error", tex.name], cwd=work, check=False)
if r.returncode != 0:
raise Failed(r.stdout[-1200:])
aux = work / f"{tex.stem}.aux"
if i == 0 and aux.exists() and "\\bibdata" in aux.read_text(encoding="utf-8", errors="replace"):
run(["bibtex", tex.stem], cwd=work, check=False)
shutil.copyfile(work / f"{tex.stem}.pdf", dest)
def documents(version, foot, out):
"""docs/*.md (not README.md, input.md) and docs/tex/*.tex with a \\documentclass, as PDF."""
made, failed = [], []
docs = ROOT / "docs"
sources = [p for p in sorted(docs.glob("*.md")) if p.name.lower() not in ("readme.md", "input.md")]
sources += [p for p in sorted((docs / "tex").glob("*.tex"))
if "\\documentclass" in p.read_text(encoding="utf-8", errors="replace")]
names = {}
for src in sources:
stem = src.stem if src.stem not in names else f"{src.stem}-{src.suffix[1:]}"
names[stem] = src
dest = out / "docs" / f"{stem}-{version}.pdf"
dest.parent.mkdir(exist_ok=True)
rel = src.relative_to(ROOT).as_posix()
with tempfile.TemporaryDirectory() as t:
try:
if src.suffix == ".md":
replaced = markdown_pdf(src, dest, foot, Path(t))
note = f"from `{rel}`" + (f"; printed as ?: {', '.join(f'U+{c:04X}' for c in replaced)}"
if replaced else "")
else:
latex_pdf(src, dest, Path(t))
note = f"from `{rel}`"
made.append((dest.name, note))
print(f" {dest.relative_to(out).as_posix()}")
except Failed as e:
dest.unlink(missing_ok=True)
lines = str(e).strip().splitlines() or ["failed"]
why = next((l for l in lines if l.startswith("! ")), lines[-1])[:200]
failed.append((rel, why))
print(f"::warning::{rel} did not build: {why}")
print(" " + "\n ".join(lines[-15:]))
return made, failed
def changelog(version):
path = ROOT / "CHANGELOG.md"
if not path.exists():
return ""
m = re.search(rf"^## {re.escape(version)}\b.*?$(.*?)(?=^## |\Z)", path.read_text(encoding="utf-8"), re.M | re.S)
return m.group(1).strip() if m else ""
def size(p):
n = p.stat().st_size
return f"{n / 1e6:.1f} MB" if n >= 1e6 else f"{max(n / 1e3, 0.1):.0f} kB"
def readme(product, version, commit, when, kicad, boards, docs_made, docs_failed, skipped, out):
L = [f"# {product} {version}: release files", "",
f"The latest prototype stage of {product}: manufacturing files and documentation built by CI from tag "
f"`{version}` (commit `{commit}`, {when}, KiCad {kicad}). Every stage tag replaces this folder; "
"older stages are on the repository's Releases page with the same files. Do not edit it by hand.", ""]
for name, files, checks in boards:
L += [f"## Board {name}", "", "; ".join(checks) + "." if checks else "", "",
"| File | What | Size |", "|---|---|---|"]
L += [f"| [{f}](hardware/{name}/{f}) | {what} | {size(out / 'hardware' / name / f)} |" for f, what in files]
L.append("")
for d, why in skipped:
L += [f"Not in this release: `{d}` ({why}).", ""]
if not boards and not skipped:
L += ["No board yet.", ""]
if docs_made or docs_failed:
L += ["## Documentation", "", "| File | What | Size |", "|---|---|---|"]
L += [f"| [{f}](docs/{f}) | {what} | {size(out / 'docs' / f)} |" for f, what in docs_made]
L += [f"| | `{src}` did not build: {why} | |" for src, why in docs_failed]
L.append("")
notes = changelog(version)
if notes:
L += [f"## Changes in {version} (CHANGELOG.md)", "", notes, ""]
sums = [f"{hashlib.sha256(p.read_bytes()).hexdigest()} {p.relative_to(out).as_posix()}"
for p in sorted(out.rglob("*")) if p.is_file() and p.name != "README.md"]
if sums:
L += ["## SHA-256", "", "```", *sums, "```", ""]
(out / "README.md").write_text("\n".join(L).replace("\n\n\n", "\n\n"), encoding="utf-8", newline="\n")
def main():
ap = argparse.ArgumentParser(description="Release files of a prototype stage")
ap.add_argument("version", help="the stage tag, e.g. v0.2")
ap.add_argument("--out", default="release", help="output folder (default: release)")
a = ap.parse_args()
if not re.fullmatch(r"[0-9A-Za-z._+-]+", a.version):
sys.exit(f"not a version: {a.version!r}")
out = Path(a.out) if Path(a.out).is_absolute() else ROOT / a.out
m = manifest()
product = (m.get("product") or {}).get("name") or os.environ.get("GITHUB_REPOSITORY", "").split("/")[-1] or ROOT.name
commit = git("rev-parse", "--short", "HEAD") or "?"
when = f"{datetime.datetime.now(datetime.timezone.utc):%Y-%m-%d %H:%M} UTC"
kicad = cli("version").strip()
clear(out)
boards, skipped, ok = [], [], True
for board_dir, pro in projects(m):
rel = board_dir.relative_to(ROOT).as_posix()
if pro is None:
print(f"== {rel}: no KiCad project yet")
skipped.append((rel, "no KiCad project yet"))
continue
print(f"== {pro.stem}")
with tempfile.TemporaryDirectory() as t:
try:
name, files, checks = board(board_dir, pro, a.version, out, Path(t))
except Failed as e:
print(f"::error::{pro.stem}: {e}")
ok = False
continue
boards.append((name, files, checks))
for c in checks:
print(f" {c}")
if re.search(r"[1-9]\d* (errors|unconnected|schematic)", c):
print(f"::warning::{name} {c}")
for f, _ in files:
print(f" hardware/{name}/{f}")
print("== documentation")
foot = tex_escape(f"{product} {a.version}, {commit}, {when[:10]}")
docs_made, docs_failed = documents(a.version, foot, out)
readme(product, a.version, commit, when, kicad, boards, docs_made, docs_failed, skipped, out)
print(f"== {out}: {sum(len(f) for _, f, _ in boards)} board files, {len(docs_made)} documents")
sys.exit(0 if ok else 1)
if __name__ == "__main__":
main()