Template
release/: the newest stage's manufacturing files and documentation
A stage tag (v*) now builds, with kicad-cli, pandoc and pdflatex in the CI image (.gitea/scripts/release-files.py): - per board: Gerbers + Excellon drill zip (JLCPCB-compatible settings), BOM (grouped, DNP left out), pick and place, schematic and PCB PDFs, 3D renders; - docs/*.md and docs/tex/*.tex as PDF, tag and commit in the footer; - release/README.md with tag, commit, ERC/DRC result, files, SHA-256 and the stage's CHANGELOG section. Every file name carries the tag. release-attach.py attaches the files plus a zip of the whole folder to the tag's release (replacing assets of an earlier run); for the newest stage tag (not older stages, not pre-releases) the job commits them to release/ on main with [skip ci]. .gitignore: !/release/ after Release/ (Windows matches it case-insensitively); .gitattributes: release/** -text so the SHA-256 match on every checkout. TEMPLATE.md: "Release files" section, rules and CI updated. Tested in git.b4l.co.th/b4l/kicad10_auto_node with harNES's board and docs: v0.1 committed, v0.0 and v0.2-rc.1 left release/ alone, v0.2 tagged on an older commit landed on top of a moved main; uploads byte-exact against a stub of the Gitea release API.
This commit is contained in:
grabowski committed
1 parent
0e002be80c
commit
52a40c1c3e
11 files changed
+654
-53
No files matched your search
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Attach the files of a release folder (release-files.py) to the Gitea release of the tag.
|
||||
|
||||
TOKEN=<job token> python3 .gitea/scripts/release-attach.py /tmp/release
|
||||
|
||||
Reads GITHUB_SERVER_URL, GITHUB_REPOSITORY and GITHUB_REF_NAME (the tag), as Gitea Actions sets
|
||||
them. Creates the release when the tag has none, with the tag's CHANGELOG.md section as its text.
|
||||
Uploads every file of the folder except README.md, and <repository>-<tag>.zip with the whole folder;
|
||||
an asset of the same name from an earlier run of the tag is replaced. Used by
|
||||
.gitea/workflows/release.yml.
|
||||
"""
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
ZIP_TIME = (2020, 1, 1, 0, 0, 0)
|
||||
|
||||
|
||||
def changelog(tag):
|
||||
path = ROOT / "CHANGELOG.md"
|
||||
if not path.exists():
|
||||
return ""
|
||||
m = re.search(rf"^## {re.escape(tag)}\b.*?$(.*?)(?=^## |\Z)", path.read_text(encoding="utf-8"), re.M | re.S)
|
||||
return m.group(1).strip() if m else ""
|
||||
|
||||
|
||||
def bundle(folder, top):
|
||||
"""The whole folder as a zip under <top>/, with fixed timestamps."""
|
||||
buf = io.BytesIO()
|
||||
with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as z:
|
||||
for p in sorted(folder.rglob("*")):
|
||||
if p.is_file():
|
||||
info = zipfile.ZipInfo(f"{top}/{p.relative_to(folder).as_posix()}", date_time=ZIP_TIME)
|
||||
info.compress_type = zipfile.ZIP_DEFLATED
|
||||
z.writestr(info, p.read_bytes())
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
class Gitea:
|
||||
def __init__(self, server, repo, token):
|
||||
self.api = f"{server.rstrip('/')}/api/v1/repos/{repo}"
|
||||
self.auth = {"Authorization": f"token {token}"}
|
||||
|
||||
def call(self, method, path, body=None, ctype="application/json"):
|
||||
req = urllib.request.Request(self.api + path, data=body, method=method,
|
||||
headers={**self.auth, "Content-Type": ctype, "Accept": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=120) as r:
|
||||
data = r.read()
|
||||
return json.loads(data) if data.strip() else None
|
||||
|
||||
def release(self, tag):
|
||||
try:
|
||||
return self.call("GET", f"/releases/tags/{urllib.parse.quote(tag, safe='')}"), False
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code != 404:
|
||||
raise
|
||||
notes = changelog(tag)
|
||||
body = (f"{notes}\n\n" if notes else "") + "Gerbers, BOM, pick and place, PDFs and documentation: the files below."
|
||||
return self.call("POST", "/releases", json.dumps({"tag_name": tag, "name": tag, "body": body}).encode()), True
|
||||
|
||||
def upload(self, rel, name, data):
|
||||
b = uuid.uuid4().hex
|
||||
head = (f"--{b}\r\nContent-Disposition: form-data; name=\"attachment\"; filename=\"{name}\"\r\n"
|
||||
"Content-Type: application/octet-stream\r\n\r\n").encode()
|
||||
body = head + data + f"\r\n--{b}--\r\n".encode()
|
||||
self.call("POST", f"/releases/{rel['id']}/assets?name={urllib.parse.quote(name)}", body,
|
||||
f"multipart/form-data; boundary={b}")
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
sys.exit(__doc__)
|
||||
folder = Path(sys.argv[1])
|
||||
env = {k: os.environ.get(k, "") for k in ("GITHUB_SERVER_URL", "GITHUB_REPOSITORY", "GITHUB_REF_NAME", "TOKEN")}
|
||||
missing = [k for k, v in env.items() if not v]
|
||||
if missing:
|
||||
sys.exit(f"missing environment: {', '.join(missing)}")
|
||||
tag, repo = env["GITHUB_REF_NAME"], env["GITHUB_REPOSITORY"]
|
||||
files = [(p.name, p.read_bytes()) for p in sorted(folder.rglob("*")) if p.is_file() and p.name != "README.md"]
|
||||
top = f"{repo.split('/')[-1]}-{tag}"
|
||||
files.append((f"{top}.zip", bundle(folder, top)))
|
||||
names = [n for n, _ in files]
|
||||
if len(set(names)) != len(names):
|
||||
sys.exit(f"two files of the same name: {sorted(n for n in names if names.count(n) > 1)}")
|
||||
|
||||
g = Gitea(env["GITHUB_SERVER_URL"], repo, env["TOKEN"])
|
||||
rel, created = g.release(tag)
|
||||
print(("created" if created else "found") + f" release {tag} (id {rel['id']})")
|
||||
old = {a["name"]: a["id"] for a in rel.get("assets") or []}
|
||||
for name, data in files:
|
||||
if name in old:
|
||||
g.call("DELETE", f"/releases/{rel['id']}/assets/{old[name]}")
|
||||
g.upload(rel, name, data)
|
||||
print(f" {'replaced' if name in old else 'attached'} {name} ({len(data):,} bytes)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,391 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The release files of a prototype stage: every board's manufacturing files and the documentation.
|
||||
|
||||
python3 .gitea/scripts/release-files.py v0.2 [--out release]
|
||||
|
||||
Writes into --out (default release/; existing content is replaced):
|
||||
|
||||
README.md version, commit, checks, file list, SHA-256, changelog
|
||||
hardware/<board>/<board>-<version>-gerbers.zip Gerbers and Excellon drill files for the PCB fab
|
||||
<board>-<version>-bom.csv bill of materials, grouped, DNP left out
|
||||
<board>-<version>-cpl.csv pick and place, both sides, mm
|
||||
<board>-<version>-schematic.pdf the schematic
|
||||
<board>-<version>-pcb.pdf one page per copper, silkscreen and fab layer
|
||||
<board>-<version>-top.png 3D render, top
|
||||
<board>-<version>-bottom.png 3D render, bottom
|
||||
docs/<name>-<version>.pdf docs/*.md (pandoc, pdflatex) and docs/tex/*.tex (pdflatex)
|
||||
|
||||
Boards are the [[boards]] of proplan.toml, or hardware/*/*.kicad_pro when it names none. DRC and ERC
|
||||
run and their results go into README.md; they do not stop the release, because the tag records what
|
||||
was ordered. A document that does not build is listed as not built. Exit 1 when the files of a board
|
||||
could not be made (the workflow then publishes nothing).
|
||||
|
||||
Used by .gitea/workflows/release.yml. Locally, from the repository root, in the CI image:
|
||||
docker run --rm -v "$PWD:/w" -w /w git.b4l.co.th/b4l/kicad10_auto_node:latest \\
|
||||
python3 .gitea/scripts/release-files.py v0.1 --out build/release
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import datetime
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import tomllib
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
KICAD_CLI = os.environ.get("KICAD_CLI") or shutil.which("kicad-cli") or "kicad-cli"
|
||||
RENDER_SIZE = (os.environ.get("RENDER_WIDTH", "1400"), os.environ.get("RENDER_HEIGHT", "1540"))
|
||||
ZIP_TIME = (2020, 1, 1, 0, 0, 0) # fixed timestamps: the same files give the same zip
|
||||
BOM_FIELDS = ["Reference", "Value", "Footprint", "Manufacturer", "MPN", "LCSC", "QUANTITY"]
|
||||
BOM_LABELS = ["Reference", "Value", "Footprint", "Manufacturer", "MPN", "LCSC", "Qty"]
|
||||
|
||||
# Characters pdflatex does not know, as LaTeX. Anything else it refuses is printed as "?" and
|
||||
# reported (pdflatex because the image's lualatex cannot load its fonts).
|
||||
UNICODE = {
|
||||
0x2192: r"\ensuremath{\rightarrow}", 0x2190: r"\ensuremath{\leftarrow}", 0x2194: r"\ensuremath{\leftrightarrow}",
|
||||
0x2191: r"\ensuremath{\uparrow}", 0x2193: r"\ensuremath{\downarrow}", 0x21D2: r"\ensuremath{\Rightarrow}",
|
||||
0x21D0: r"\ensuremath{\Leftarrow}", 0x21D4: r"\ensuremath{\Leftrightarrow}", 0x21A6: r"\ensuremath{\mapsto}",
|
||||
0x2264: r"\ensuremath{\leq}", 0x2265: r"\ensuremath{\geq}", 0x2248: r"\ensuremath{\approx}",
|
||||
0x2260: r"\ensuremath{\neq}", 0x2261: r"\ensuremath{\equiv}", 0x2212: r"\ensuremath{-}",
|
||||
0x221E: r"\ensuremath{\infty}", 0x221A: r"\ensuremath{\surd}", 0x2211: r"\ensuremath{\sum}",
|
||||
0x220F: r"\ensuremath{\prod}", 0x2202: r"\ensuremath{\partial}", 0x2208: r"\ensuremath{\in}",
|
||||
0x2205: r"\ensuremath{\emptyset}", 0x2227: r"\ensuremath{\wedge}", 0x2228: r"\ensuremath{\vee}",
|
||||
0x2295: r"\ensuremath{\oplus}", 0x22C5: r"\ensuremath{\cdot}", 0x2026: r"\ldots{}",
|
||||
0x2713: r"\ensuremath{\checkmark}", 0x2714: r"\ensuremath{\checkmark}", 0x2705: r"\ensuremath{\checkmark}",
|
||||
0x2717: r"\ensuremath{\times}", 0x2718: r"\ensuremath{\times}", 0x274C: r"\ensuremath{\times}",
|
||||
0x2126: r"\ensuremath{\Omega}", 0x2103: r"\textdegree{}C", 0x2032: r"\ensuremath{'}",
|
||||
0x2033: r"\ensuremath{''}", 0x2009: r"\,", 0x202F: r"\,", 0x200B: "", 0xFE0F: "",
|
||||
0x25B6: r"\ensuremath{\blacktriangleright}", 0x25C0: r"\ensuremath{\blacktriangleleft}",
|
||||
0x25B2: r"\ensuremath{\blacktriangle}", 0x25BC: r"\ensuremath{\blacktriangledown}",
|
||||
0x25CF: r"\ensuremath{\bullet}", 0x25CB: r"\ensuremath{\circ}", 0x26A0: r"\textbf{!}",
|
||||
# box drawing (trees in code blocks); \mbox keeps two dashes from becoming an en dash
|
||||
0x2500: r"\mbox{-}", 0x2501: r"\mbox{-}", 0x2502: "|", 0x2503: "|", 0x2550: r"\mbox{=}", 0x2551: "|",
|
||||
**{c: "+" for c in (0x250C, 0x2510, 0x2514, 0x2518, 0x251C, 0x2524, 0x252C, 0x2534, 0x253C,
|
||||
0x2554, 0x2557, 0x255A, 0x255D, 0x2560, 0x2563, 0x2566, 0x2569, 0x256C)},
|
||||
}
|
||||
GREEK = "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho - sigma tau upsilon phi chi psi omega"
|
||||
for i, n in enumerate(GREEK.split()):
|
||||
if n not in ("-", "omicron"):
|
||||
UNICODE[0x3B1 + i] = rf"\ensuremath{{\{n}}}"
|
||||
for c, n in {0x393: "Gamma", 0x394: "Delta", 0x398: "Theta", 0x39B: "Lambda", 0x39E: "Xi", 0x3A0: "Pi",
|
||||
0x3A3: "Sigma", 0x3A5: "Upsilon", 0x3A6: "Phi", 0x3A8: "Psi", 0x3A9: "Omega"}.items():
|
||||
UNICODE[c] = rf"\ensuremath{{\{n}}}"
|
||||
UNICODE[0x3C2] = r"\ensuremath{\varsigma}"
|
||||
|
||||
|
||||
class Failed(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def run(cmd, cwd=None, check=True):
|
||||
r = subprocess.run([str(c) for c in cmd], cwd=cwd, capture_output=True, text=True, errors="replace")
|
||||
if check and r.returncode != 0:
|
||||
raise Failed(f"{Path(str(cmd[0])).name} {' '.join(map(str, cmd[1:4]))}: exit {r.returncode}\n"
|
||||
f"{(r.stdout + r.stderr)[-1500:]}")
|
||||
return r
|
||||
|
||||
|
||||
def cli(*args):
|
||||
return run([KICAD_CLI, *args]).stdout
|
||||
|
||||
|
||||
def git(*args):
|
||||
r = run(["git", "-C", ROOT, *args], check=False)
|
||||
return r.stdout.strip() if r.returncode == 0 else ""
|
||||
|
||||
|
||||
def tex_escape(s):
|
||||
return re.sub(r"([\\{}$&#^_%~])", lambda m: {"\\": r"\textbackslash{}", "^": r"\^{}", "~": r"\~{}"}
|
||||
.get(m.group(1), "\\" + m.group(1)), s)
|
||||
|
||||
|
||||
def clear(folder):
|
||||
"""Empty a folder but keep it (a bind-mounted folder may refuse to be removed)."""
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
for p in folder.iterdir():
|
||||
shutil.rmtree(p) if p.is_dir() else p.unlink()
|
||||
|
||||
|
||||
def manifest():
|
||||
path = ROOT / "proplan.toml"
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
return tomllib.loads(path.read_text(encoding="utf-8"))
|
||||
except tomllib.TOMLDecodeError as e:
|
||||
print(f"::warning::proplan.toml cannot be read ({e}); taking hardware/*/*.kicad_pro")
|
||||
return {}
|
||||
|
||||
|
||||
def projects(m):
|
||||
"""(board folder, .kicad_pro or None) for every board."""
|
||||
dirs = [ROOT / b["path"] for b in m.get("boards", []) if isinstance(b, dict) and isinstance(b.get("path"), str)]
|
||||
if not dirs:
|
||||
dirs = sorted({p.parent for p in ROOT.glob("hardware/*/*.kicad_pro")})
|
||||
out = []
|
||||
for d in dirs:
|
||||
pros = sorted(d.glob("*.kicad_pro"))
|
||||
out.append((d, next((p for p in pros if p.stem == d.name), pros[0] if pros else None)))
|
||||
return out
|
||||
|
||||
|
||||
def erc(sch, tmp):
|
||||
rpt = tmp / "erc.json"
|
||||
cli("sch", "erc", "--format", "json", "--severity-error", "-o", rpt, sch)
|
||||
d = json.loads(rpt.read_text(encoding="utf-8"))
|
||||
return sum(len(s.get("violations", [])) for s in d.get("sheets", []))
|
||||
|
||||
|
||||
def drc(pcb, tmp):
|
||||
rpt = tmp / "drc.json"
|
||||
cli("pcb", "drc", "--format", "json", "--severity-error", "--schematic-parity", "-o", rpt, pcb)
|
||||
d = json.loads(rpt.read_text(encoding="utf-8"))
|
||||
return {k: len(d.get(k, [])) for k in ("violations", "unconnected_items", "schematic_parity")}
|
||||
|
||||
|
||||
def copper_layers(pcb):
|
||||
return re.findall(r'\(\d+ "([A-Za-z0-9]+\.Cu)" (?:signal|power|mixed|jumper)\)', pcb.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def gerbers_zip(pcb, dest, tmp):
|
||||
out = tmp / "gerbers"
|
||||
out.mkdir()
|
||||
copper = copper_layers(pcb)
|
||||
layers = copper + ["F.Paste", "B.Paste", "F.Silkscreen", "B.Silkscreen", "F.Mask", "B.Mask", "Edge.Cuts"]
|
||||
cli("pcb", "export", "gerbers", "-o", f"{out}/", "--layers", ",".join(layers), "--subtract-soldermask",
|
||||
"--no-x2", "--no-netlist", "--check-zones", pcb)
|
||||
cli("pcb", "export", "drill", "-o", f"{out}/", "--format", "excellon", "--drill-origin", "absolute",
|
||||
"--excellon-units", "mm", "--excellon-zeros-format", "decimal", "--excellon-oval-format", "alternate",
|
||||
"--excellon-separate-th", pcb)
|
||||
files = sorted(p for p in out.iterdir() if p.suffix.lower() != ".gbrjob")
|
||||
drills = [p for p in files if p.suffix.lower() == ".drl"]
|
||||
if len(files) - len(drills) != len(layers) or not drills:
|
||||
raise Failed(f"expected {len(layers)} Gerbers and drill files, got {[p.name for p in files]}")
|
||||
with zipfile.ZipFile(dest, "w", zipfile.ZIP_DEFLATED) as z:
|
||||
for p in files:
|
||||
info = zipfile.ZipInfo(p.name, date_time=ZIP_TIME)
|
||||
info.compress_type = zipfile.ZIP_DEFLATED
|
||||
z.writestr(info, p.read_bytes())
|
||||
return f"Gerbers ({len(copper)} copper layers) and drill files for the PCB fab"
|
||||
|
||||
|
||||
def bom(sch, dest):
|
||||
cli("sch", "export", "bom", "-o", dest, "--fields", ",".join(BOM_FIELDS), "--labels", ",".join(BOM_LABELS),
|
||||
"--group-by", "Value,Footprint,Manufacturer,MPN,LCSC", "--exclude-dnp", sch)
|
||||
rows = list(csv.reader(io.StringIO(dest.read_text(encoding="utf-8"))))
|
||||
return f"bill of materials: {max(len(rows) - 1, 0)} lines, DNP parts left out"
|
||||
|
||||
|
||||
def cpl(pcb, dest):
|
||||
cli("pcb", "export", "pos", "-o", dest, "--format", "csv", "--units", "mm", "--side", "both", "--exclude-dnp", pcb)
|
||||
rows = list(csv.reader(io.StringIO(dest.read_text(encoding="utf-8"))))
|
||||
return f"pick and place: {max(len(rows) - 1, 0)} parts, both sides, mm, DNP left out"
|
||||
|
||||
|
||||
def board(board_dir, pro, version, out, tmp):
|
||||
"""Write one board's files; returns (name, [(file, what)], checks)."""
|
||||
name = pro.stem
|
||||
sch, pcb = board_dir / f"{name}.kicad_sch", board_dir / f"{name}.kicad_pcb"
|
||||
dest = out / "hardware" / name
|
||||
dest.mkdir(parents=True)
|
||||
base = f"{name}-{version}"
|
||||
files, checks = [], []
|
||||
if sch.exists():
|
||||
n = erc(sch, tmp)
|
||||
checks.append(f"ERC: {n} errors")
|
||||
files.append((f"{base}-schematic.pdf", "schematic"))
|
||||
cli("sch", "export", "pdf", "-o", dest / files[-1][0], sch)
|
||||
files.append((f"{base}-bom.csv", bom(sch, dest / f"{base}-bom.csv")))
|
||||
if pcb.exists():
|
||||
d = drc(pcb, tmp)
|
||||
checks.append(f"DRC: {d['violations']} errors, {d['unconnected_items']} unconnected, "
|
||||
f"{d['schematic_parity']} schematic parity issues")
|
||||
files.insert(0, (f"{base}-gerbers.zip", gerbers_zip(pcb, dest / f"{base}-gerbers.zip", tmp)))
|
||||
files.append((f"{base}-cpl.csv", cpl(pcb, dest / f"{base}-cpl.csv")))
|
||||
copper = ",".join(copper_layers(pcb))
|
||||
cli("pcb", "export", "pdf", "--mode-multipage", "--include-border-title",
|
||||
"--layers", f"{copper},F.Silkscreen,B.Silkscreen,F.Fab,B.Fab", "--common-layers", "Edge.Cuts",
|
||||
"-o", dest / f"{base}-pcb.pdf", pcb)
|
||||
files.append((f"{base}-pcb.pdf", "PCB: one page per copper, silkscreen and fab layer"))
|
||||
for side in ("top", "bottom"):
|
||||
cli("pcb", "render", "--side", side, "--width", RENDER_SIZE[0], "--height", RENDER_SIZE[1],
|
||||
"--quality", "high", "-o", dest / f"{base}-{side}.png", pcb)
|
||||
files.append((f"{base}-{side}.png", f"3D render, {side}"))
|
||||
if not files:
|
||||
raise Failed(f"{name}: neither {sch.name} nor {pcb.name}")
|
||||
return name, files, checks
|
||||
|
||||
|
||||
def unicode_header(extra):
|
||||
lines = [rf"\DeclareUnicodeCharacter{{{c:04X}}}{{{v}}}" for c, v in sorted({**UNICODE, **extra}.items())]
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def footer(text):
|
||||
return ("\\usepackage{fancyhdr}\n\\pagestyle{fancy}\n\\fancyhf{}\n\\renewcommand{\\headrulewidth}{0pt}\n"
|
||||
f"\\fancyfoot[L]{{\\footnotesize {text}}}\n\\fancyfoot[R]{{\\footnotesize\\thepage}}\n"
|
||||
"\\fancypagestyle{plain}{\\pagestyle{fancy}}\n")
|
||||
|
||||
|
||||
def markdown_pdf(md, dest, foot, tmp):
|
||||
"""pandoc + pdflatex; characters pdflatex refuses are retried as '?'. Returns the replaced ones."""
|
||||
extra = {}
|
||||
header = tmp / "header.tex"
|
||||
for _ in range(40):
|
||||
header.write_text(footer(foot) + unicode_header(extra), encoding="utf-8")
|
||||
r = run(["pandoc", md.name, "-o", dest, "--pdf-engine=pdflatex", "-H", header,
|
||||
"-V", "geometry:margin=2cm", "-V", "colorlinks=true", "--resource-path", "."],
|
||||
cwd=md.parent, check=False)
|
||||
if r.returncode == 0:
|
||||
return sorted(extra)
|
||||
m = re.search(r"Unicode character .*?\(U\+([0-9A-Fa-f]+)\)", r.stdout + r.stderr, re.S)
|
||||
if not m:
|
||||
raise Failed((r.stdout + r.stderr)[-1200:])
|
||||
extra[int(m.group(1), 16)] = "?"
|
||||
raise Failed("too many characters pdflatex cannot print")
|
||||
|
||||
|
||||
def latex_pdf(tex, dest, tmp):
|
||||
work = tmp / "tex"
|
||||
shutil.copytree(tex.parent, work)
|
||||
for i in range(3):
|
||||
r = run(["pdflatex", "-interaction=nonstopmode", "-halt-on-error", tex.name], cwd=work, check=False)
|
||||
if r.returncode != 0:
|
||||
raise Failed(r.stdout[-1200:])
|
||||
aux = work / f"{tex.stem}.aux"
|
||||
if i == 0 and aux.exists() and "\\bibdata" in aux.read_text(encoding="utf-8", errors="replace"):
|
||||
run(["bibtex", tex.stem], cwd=work, check=False)
|
||||
shutil.copyfile(work / f"{tex.stem}.pdf", dest)
|
||||
|
||||
|
||||
def documents(version, foot, out):
|
||||
"""docs/*.md (not README.md, input.md) and docs/tex/*.tex with a \\documentclass, as PDF."""
|
||||
made, failed = [], []
|
||||
docs = ROOT / "docs"
|
||||
sources = [p for p in sorted(docs.glob("*.md")) if p.name.lower() not in ("readme.md", "input.md")]
|
||||
sources += [p for p in sorted((docs / "tex").glob("*.tex"))
|
||||
if "\\documentclass" in p.read_text(encoding="utf-8", errors="replace")]
|
||||
names = {}
|
||||
for src in sources:
|
||||
stem = src.stem if src.stem not in names else f"{src.stem}-{src.suffix[1:]}"
|
||||
names[stem] = src
|
||||
dest = out / "docs" / f"{stem}-{version}.pdf"
|
||||
dest.parent.mkdir(exist_ok=True)
|
||||
rel = src.relative_to(ROOT).as_posix()
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
try:
|
||||
if src.suffix == ".md":
|
||||
replaced = markdown_pdf(src, dest, foot, Path(t))
|
||||
note = f"from `{rel}`" + (f"; printed as ?: {', '.join(f'U+{c:04X}' for c in replaced)}"
|
||||
if replaced else "")
|
||||
else:
|
||||
latex_pdf(src, dest, Path(t))
|
||||
note = f"from `{rel}`"
|
||||
made.append((dest.name, note))
|
||||
print(f" {dest.relative_to(out).as_posix()}")
|
||||
except Failed as e:
|
||||
dest.unlink(missing_ok=True)
|
||||
lines = str(e).strip().splitlines() or ["failed"]
|
||||
why = next((l for l in lines if l.startswith("! ")), lines[-1])[:200]
|
||||
failed.append((rel, why))
|
||||
print(f"::warning::{rel} did not build: {why}")
|
||||
print(" " + "\n ".join(lines[-15:]))
|
||||
return made, failed
|
||||
|
||||
|
||||
def changelog(version):
|
||||
path = ROOT / "CHANGELOG.md"
|
||||
if not path.exists():
|
||||
return ""
|
||||
m = re.search(rf"^## {re.escape(version)}\b.*?$(.*?)(?=^## |\Z)", path.read_text(encoding="utf-8"), re.M | re.S)
|
||||
return m.group(1).strip() if m else ""
|
||||
|
||||
|
||||
def size(p):
|
||||
n = p.stat().st_size
|
||||
return f"{n / 1e6:.1f} MB" if n >= 1e6 else f"{max(n / 1e3, 0.1):.0f} kB"
|
||||
|
||||
|
||||
def readme(product, version, commit, when, kicad, boards, docs_made, docs_failed, skipped, out):
|
||||
L = [f"# {product} {version}: release files", "",
|
||||
f"The latest prototype stage of {product}: manufacturing files and documentation built by CI from tag "
|
||||
f"`{version}` (commit `{commit}`, {when}, KiCad {kicad}). Every stage tag replaces this folder; "
|
||||
"older stages are on the repository's Releases page with the same files. Do not edit it by hand.", ""]
|
||||
for name, files, checks in boards:
|
||||
L += [f"## Board {name}", "", "; ".join(checks) + "." if checks else "", "",
|
||||
"| File | What | Size |", "|---|---|---|"]
|
||||
L += [f"| [{f}](hardware/{name}/{f}) | {what} | {size(out / 'hardware' / name / f)} |" for f, what in files]
|
||||
L.append("")
|
||||
for d, why in skipped:
|
||||
L += [f"Not in this release: `{d}` ({why}).", ""]
|
||||
if not boards and not skipped:
|
||||
L += ["No board yet.", ""]
|
||||
if docs_made or docs_failed:
|
||||
L += ["## Documentation", "", "| File | What | Size |", "|---|---|---|"]
|
||||
L += [f"| [{f}](docs/{f}) | {what} | {size(out / 'docs' / f)} |" for f, what in docs_made]
|
||||
L += [f"| | `{src}` did not build: {why} | |" for src, why in docs_failed]
|
||||
L.append("")
|
||||
notes = changelog(version)
|
||||
if notes:
|
||||
L += [f"## Changes in {version} (CHANGELOG.md)", "", notes, ""]
|
||||
sums = [f"{hashlib.sha256(p.read_bytes()).hexdigest()} {p.relative_to(out).as_posix()}"
|
||||
for p in sorted(out.rglob("*")) if p.is_file() and p.name != "README.md"]
|
||||
if sums:
|
||||
L += ["## SHA-256", "", "```", *sums, "```", ""]
|
||||
(out / "README.md").write_text("\n".join(L).replace("\n\n\n", "\n\n"), encoding="utf-8", newline="\n")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="Release files of a prototype stage")
|
||||
ap.add_argument("version", help="the stage tag, e.g. v0.2")
|
||||
ap.add_argument("--out", default="release", help="output folder (default: release)")
|
||||
a = ap.parse_args()
|
||||
if not re.fullmatch(r"[0-9A-Za-z._+-]+", a.version):
|
||||
sys.exit(f"not a version: {a.version!r}")
|
||||
out = Path(a.out) if Path(a.out).is_absolute() else ROOT / a.out
|
||||
m = manifest()
|
||||
product = (m.get("product") or {}).get("name") or os.environ.get("GITHUB_REPOSITORY", "").split("/")[-1] or ROOT.name
|
||||
commit = git("rev-parse", "--short", "HEAD") or "?"
|
||||
when = f"{datetime.datetime.now(datetime.timezone.utc):%Y-%m-%d %H:%M} UTC"
|
||||
kicad = cli("version").strip()
|
||||
clear(out)
|
||||
boards, skipped, ok = [], [], True
|
||||
for board_dir, pro in projects(m):
|
||||
rel = board_dir.relative_to(ROOT).as_posix()
|
||||
if pro is None:
|
||||
print(f"== {rel}: no KiCad project yet")
|
||||
skipped.append((rel, "no KiCad project yet"))
|
||||
continue
|
||||
print(f"== {pro.stem}")
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
try:
|
||||
name, files, checks = board(board_dir, pro, a.version, out, Path(t))
|
||||
except Failed as e:
|
||||
print(f"::error::{pro.stem}: {e}")
|
||||
ok = False
|
||||
continue
|
||||
boards.append((name, files, checks))
|
||||
for c in checks:
|
||||
print(f" {c}")
|
||||
if re.search(r"[1-9]\d* (errors|unconnected|schematic)", c):
|
||||
print(f"::warning::{name} {c}")
|
||||
for f, _ in files:
|
||||
print(f" hardware/{name}/{f}")
|
||||
print("== documentation")
|
||||
foot = tex_escape(f"{product} {a.version}, {commit}, {when[:10]}")
|
||||
docs_made, docs_failed = documents(a.version, foot, out)
|
||||
readme(product, a.version, commit, when, kicad, boards, docs_made, docs_failed, skipped, out)
|
||||
print(f"== {out}: {sum(len(f) for _, f, _ in boards)} board files, {len(docs_made)} documents")
|
||||
sys.exit(0 if ok else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,9 +1,14 @@
|
||||
name: Release outputs
|
||||
name: Release
|
||||
|
||||
# On a tag v* (a prototype stage): the KiBot outputs of every board, zipped per
|
||||
# board, attached to the tag's release (created when missing). The job token
|
||||
# must be allowed to write releases. Adapted from LVSolarBuck; runs for the
|
||||
# first time at the first tag.
|
||||
# On a tag v* (a prototype stage), .gitea/scripts/release-files.py builds the
|
||||
# stage's release files: for every board Gerbers and drill (zip), BOM, pick
|
||||
# and place, schematic and PCB PDFs and 3D renders; the documentation
|
||||
# (docs/*.md, docs/tex/*.tex) as PDF. They are attached to the tag's release
|
||||
# (created when missing) and, when the tag is the newest stage, committed to
|
||||
# release/ on main, so release/ always holds the latest stage. The commit
|
||||
# carries [skip ci] and touches no path another workflow watches. The job
|
||||
# token must be allowed to write releases and to push to main. Firmware and
|
||||
# software tags (firmware/..., software/...) do not start it.
|
||||
on:
|
||||
push:
|
||||
tags: ['v*']
|
||||
@@ -11,47 +16,54 @@ on:
|
||||
jobs:
|
||||
release:
|
||||
runs-on: kicad10_auto
|
||||
permissions:
|
||||
contents: write
|
||||
concurrency:
|
||||
group: release-${{ github.repository }}
|
||||
cancel-in-progress: false
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: KiBot outputs per board
|
||||
run: |
|
||||
mkdir -p dist
|
||||
for pro in $(find hardware -mindepth 2 -maxdepth 2 -name '*.kicad_pro' | sort); do
|
||||
dir=$(dirname "$pro"); board=$(basename "$dir")
|
||||
(cd "$dir" && kibot -c "$GITHUB_WORKSPACE/kibot.yaml" -d "$GITHUB_WORKSPACE/out/$board") \
|
||||
|| echo "::warning::KiBot reported problems for $board"
|
||||
python3 -c 'import shutil, sys; shutil.make_archive(sys.argv[1], "zip", sys.argv[2])' \
|
||||
"dist/$board-${GITHUB_REF_NAME}" "out/$board"
|
||||
done
|
||||
ls -la dist
|
||||
- name: Release files
|
||||
run: python3 .gitea/scripts/release-files.py "$GITHUB_REF_NAME" --out /tmp/release
|
||||
|
||||
- name: Attach to the release
|
||||
env:
|
||||
TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
run: python3 .gitea/scripts/release-attach.py /tmp/release
|
||||
|
||||
- name: Commit to release/ on main
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
run: |
|
||||
python3 - <<'EOF'
|
||||
import json, os, pathlib, urllib.error, urllib.request, uuid
|
||||
api = f"{os.environ['GITHUB_SERVER_URL']}/api/v1/repos/{os.environ['GITHUB_REPOSITORY']}"
|
||||
tag = os.environ["GITHUB_REF_NAME"]
|
||||
auth = {"Authorization": f"token {os.environ['TOKEN']}"}
|
||||
|
||||
def call(method, url, body=None, ctype="application/json"):
|
||||
req = urllib.request.Request(url, data=body, method=method, headers={**auth, "Content-Type": ctype})
|
||||
with urllib.request.urlopen(req) as r:
|
||||
return json.load(r)
|
||||
|
||||
try:
|
||||
rel = call("GET", f"{api}/releases/tags/{tag}")
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code != 404:
|
||||
raise
|
||||
rel = call("POST", f"{api}/releases", json.dumps({"tag_name": tag, "name": tag}).encode())
|
||||
for f in sorted(pathlib.Path("dist").glob("*.zip")):
|
||||
b = uuid.uuid4().hex
|
||||
head = (f"--{b}\r\nContent-Disposition: form-data; name=\"attachment\"; filename=\"{f.name}\"\r\n"
|
||||
"Content-Type: application/zip\r\n\r\n").encode()
|
||||
body = head + f.read_bytes() + f"\r\n--{b}--\r\n".encode()
|
||||
call("POST", f"{api}/releases/{rel['id']}/assets?name={f.name}", body, f"multipart/form-data; boundary={b}")
|
||||
print("attached", f.name)
|
||||
EOF
|
||||
tag="$GITHUB_REF_NAME"
|
||||
# stage tags only (v0.1, v0.2, v1.0); a pre-release such as v0.3-rc.1 leaves release/ alone
|
||||
newest=$(git tag -l 'v*' --sort=-v:refname | grep -E '^v[0-9]+(\.[0-9]+)+$' | head -1 || true)
|
||||
if [ "$tag" != "$newest" ]; then
|
||||
echo "$tag is not the newest stage (${newest:-none}): release/ is left as it is."
|
||||
exit 0
|
||||
fi
|
||||
git config user.name "Release CI"
|
||||
git config user.email "ci@noreply.git.b4l.co.th"
|
||||
REPO_URL=$(git config --get remote.origin.url)
|
||||
AUTHED_URL=${REPO_URL/https:\/\//https:\/\/${GITEA_TOKEN}@}
|
||||
# main may move while this runs: start again from the newest main
|
||||
for i in 1 2 3; do
|
||||
git fetch -q "$AUTHED_URL" main
|
||||
git checkout -q -f -B main FETCH_HEAD
|
||||
rm -rf release
|
||||
mkdir release
|
||||
cp -r /tmp/release/. release/
|
||||
git add -A -- release
|
||||
if git diff --cached --quiet; then echo "release/ already holds $tag."; exit 0; fi
|
||||
git commit -q -m "Release $tag: manufacturing files and documentation [skip ci]"
|
||||
if git push -q "$AUTHED_URL" HEAD:main; then
|
||||
echo "Pushed $(git rev-parse --short HEAD): release/ holds $tag"
|
||||
exit 0
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
echo "::error::Could not push release/ to main"
|
||||
exit 1
|
||||
Reference in new issue
Block a user