isokit/tools/bench_web.py

"""How the web Studio and a static page run on a slow machine.

Serves dist/web (build it first: python tools/build_web.py) and opens it and
an example's static page (isokit build examples/house) in headless Edge with
the CPU slowed down (CDP Emulation.setCPUThrottlingRate; 6x is roughly a
low-power laptop next to a desktop). For each page it prints the load time,
DOM size, the main thread's busy share while idle (animations), frames per
main-thread time per input step and long frames (over 50 ms) while panning,
zooming and hovering, and how long a
rotation, a level switch and an edit take until painted.

    uv run --with playwright python tools/bench_web.py [--rate 6] [--headed] [--anim light|smooth|off]
                                                       [--json out.json] [--only studio|page]

Use --headed for the idle figures: headless Edge renders no frames for CSS
animations when nothing else changes, so their cost only shows in a window.
"""

import argparse
import json
import subprocess
import sys
import time
from pathlib import Path

from playwright.sync_api import sync_playwright

ROOT = Path(__file__).resolve().parent.parent
DIST = ROOT / "dist" / "web"
PAGE = ROOT / "examples" / "house" / "dist" / "index.html"
PORT = 7941
ANIM = None

# in the page: collect long animation frames (over 50 ms, Chromium's LoAF API);
# no requestAnimationFrame loop, which would make headless Chrome render flat out.
# JS_TIMED times an action until the view's SVG has new content and two frames
# have been painted.
JS_SETUP = """() => {
  if (window.__bench) return;
  window.__bench = { loaf: [] };
  try {
    new PerformanceObserver(l => l.getEntries().forEach(e => window.__bench.loaf.push(e.duration)))
      .observe({ type: 'long-animation-frame', buffered: false });
  } catch (e) { /* not supported */ }
}"""
JS_TIMED = """async (code) => {
  const svg = document.querySelector('.iso-stage .iso-svg');
  const t0 = performance.now();
  const changed = new Promise(r => { const mo = new MutationObserver(() => { mo.disconnect(); r(); }); mo.observe(svg, { childList: true }); });
  (0, eval)(code);
  const late = await Promise.race([changed.then(() => false), new Promise(r => setTimeout(() => r(true), 30000))]);
  if (late) return -1;
  await new Promise(r => requestAnimationFrame(() => requestAnimationFrame(r)));
  return performance.now() - t0;
}"""


T0 = time.perf_counter()


def log(what):
    print(f"  [{time.perf_counter() - T0:6.1f}s] {what}", flush=True)


def metrics(cdp):
    m = {x["name"]: x["value"] for x in cdp.send("Performance.getMetrics")["metrics"]}
    return m


def busy(cdp, pg, run, steps=1):
    """Run `run()` (`steps` input events); answer a dict: wall seconds, the main
    thread's busy share, its ms per step, and the long frames (count, longest)."""
    pg.evaluate(JS_SETUP)
    pg.evaluate("window.__bench.loaf = []")
    m0, t0 = metrics(cdp), time.perf_counter()
    run()
    pg.wait_for_timeout(300)
    m1, t1 = metrics(cdp), time.perf_counter()
    loaf = pg.evaluate("window.__bench.loaf")
    wall, task = t1 - t0, m1["TaskDuration"] - m0["TaskDuration"]
    return {"s": wall, "busy": task / wall, "ms_step": 1000 * task / steps, "long": len(loaf), "max_ms": max(loaf or [0])}


def gestures(pg, cdp, stage, out, wheel_mod=None):
    b = stage.bounding_box()
    cx, cy = b["x"] + b["width"] / 2, b["y"] + b["height"] / 2
    out["nodes"] = pg.evaluate("document.querySelectorAll('.iso-stage .iso-svg *').length")
    log("idle")
    out["idle_busy"] = busy(cdp, pg, lambda: pg.wait_for_timeout(4000))["busy"]

    def pan():
        # the middle button: always the camera, in the Studio and on a page
        pg.mouse.move(cx - 150, cy)
        pg.mouse.down(button="middle")
        for i in range(90):
            pg.mouse.move(cx - 150 + 300 * (i % 45) / 45, cy + 60 * ((i // 45) % 2))
            pg.wait_for_timeout(16)
        pg.mouse.up(button="middle")

    def zoom():
        pg.mouse.move(cx, cy)
        if wheel_mod:
            pg.keyboard.down(wheel_mod)
        for i in range(24):
            pg.mouse.wheel(0, -60 if i < 12 else 60)
            pg.wait_for_timeout(30)
        if wheel_mod:
            pg.keyboard.up(wheel_mod)

    def hover():
        for i in range(90):
            pg.mouse.move(b["x"] + b["width"] * (0.2 + 0.6 * i / 90), cy + 40 * ((i // 10) % 2))
            pg.wait_for_timeout(16)

    pg.evaluate("__isokit_view().focus([0, 0, 0], 2.2)")
    pg.wait_for_timeout(600)
    for name, fn, n in [("pan", pan, 90), ("zoom", zoom, 24), ("hover", hover, 90)]:
        log(name)
        r = busy(cdp, pg, fn, n)
        for k, v in r.items():
            out[name + "_" + k] = v
        pg.wait_for_timeout(500)
    log("rotate")
    out["rotate_ms"] = pg.evaluate(JS_TIMED, "__isokit_view().setRotation((__isokit_view().getRotation() + 1) % 4)")
    pg.wait_for_timeout(400)
    log("level")
    # to another level and back
    lv0 = pg.evaluate("__isokit_view().getLevel()")
    other = pg.evaluate("[...document.querySelectorAll('.iso-seg button')].map(b => b.dataset.level).find(l => l !== __isokit_view().getLevel())")
    out["level_ms"] = pg.evaluate(JS_TIMED, f"__isokit_view().setLevel('{other}')")
    pg.wait_for_timeout(400)
    pg.evaluate(JS_TIMED, f"__isokit_view().setLevel('{lv0}')")
    pg.wait_for_timeout(400)


def open_page(b, rate):
    pg = b.new_page(viewport={"width": 1366, "height": 768})
    if ANIM:
        pg.add_init_script(f"try {{ localStorage.setItem('isokit-anim', '{ANIM}'); }} catch (e) {{}}")
    pg.bring_to_front()
    cdp = pg.context.new_cdp_session(pg)
    cdp.send("Performance.enable")
    cdp.send("Emulation.setCPUThrottlingRate", {"rate": rate})
    errs = []
    pg.on("pageerror", lambda e: errs.append(str(e)))
    return pg, cdp, errs


def bench_studio(b, rate):
    out = {}
    pg, cdp, errs = open_page(b, rate)
    t0 = time.perf_counter()
    pg.goto(f"http://127.0.0.1:{PORT}/")
    pg.wait_for_selector(".iso-svg [data-obj]", timeout=120000)
    out["load_s"] = time.perf_counter() - t0
    pg.add_init_script("window.__isokit_view = () => window.__studio.view")
    pg.evaluate("window.__isokit_view = () => window.__studio.view")
    pg.locator("#wb-projects").click()
    t0 = time.perf_counter()
    pg.locator("#wb-nhouse").click()
    pg.wait_for_load_state()
    pg.wait_for_function("window.__studio && __studio.S.doc && __studio.S.doc.objects.length > 50 && document.querySelector('.iso-svg [data-obj]')", timeout=120000)
    out["open_house_s"] = time.perf_counter() - t0
    pg.evaluate("window.__isokit_view = () => window.__studio.view")
    pg.wait_for_timeout(1000)
    gestures(pg, cdp, pg.locator(".iso-stage"), out)
    out["edit_ms"] = pg.evaluate(JS_TIMED, "__studio.op({op: 'add', type: 'tree', x: 14, y: -5})")
    pg.wait_for_timeout(300)
    out["select_ms"] = pg.evaluate("""() => { const t0 = performance.now(); __studio.select('sofa'); return new Promise(r => requestAnimationFrame(() => requestAnimationFrame(() => r(performance.now() - t0)))); }""")
    out["errors"] = errs
    pg.close()
    return out


def bench_page(b, rate):
    out = {}
    pg, cdp, errs = open_page(b, rate)
    t0 = time.perf_counter()
    pg.goto(PAGE.as_uri())
    pg.wait_for_selector(".iso-svg [data-obj]", timeout=120000)
    out["load_s"] = time.perf_counter() - t0
    pg.evaluate("window.__isokit_view = () => window.__isokit.view")
    pg.wait_for_timeout(1000)
    gestures(pg, cdp, pg.locator(".iso-stage"), out, wheel_mod="Control")
    out["errors"] = errs
    pg.close()
    return out


ROWS = [
    ("load_s", "load (s)", "{:.2f}"),
    ("open_house_s", "open house demo (s)", "{:.2f}"),
    ("nodes", "SVG nodes", "{:.0f}"),
    ("idle_busy", "idle: main thread busy", "{:.0%}"),
    ("pan_ms_step", "pan: main thread ms per move", "{:.1f}"),
    ("pan_busy", "pan: busy", "{:.0%}"),
    ("pan_long", "pan: frames over 50 ms", "{:.0f}"),
    ("pan_max_ms", "pan: longest frame (ms)", "{:.0f}"),
    ("zoom_ms_step", "zoom: ms per wheel step", "{:.1f}"),
    ("zoom_long", "zoom: frames over 50 ms", "{:.0f}"),
    ("zoom_max_ms", "zoom: longest frame (ms)", "{:.0f}"),
    ("hover_ms_step", "hover: ms per move", "{:.1f}"),
    ("hover_long", "hover: frames over 50 ms", "{:.0f}"),
    ("rotate_ms", "rotate until painted (ms)", "{:.0f}"),
    ("level_ms", "level switch until painted (ms)", "{:.0f}"),
    ("edit_ms", "edit until painted (ms)", "{:.0f}"),
    ("select_ms", "select until painted (ms)", "{:.0f}"),
]


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--rate", type=float, default=6.0)
    ap.add_argument("--json")
    ap.add_argument("--only", choices=["studio", "page"])
    ap.add_argument("--headed", action="store_true")
    ap.add_argument("--anim", choices=["light", "smooth", "off"])
    a = ap.parse_args()
    global ANIM
    ANIM = a.anim
    if not (DIST / "index.html").exists():
        sys.exit("no dist/web: run python tools/build_web.py first")
    srv = subprocess.Popen([sys.executable, "-m", "http.server", str(PORT), "--bind", "127.0.0.1", "--directory", str(DIST)],
                           stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
    res = {}
    try:
        time.sleep(0.8)
        with sync_playwright() as p:
            b = p.chromium.launch(channel="msedge", headless=not a.headed, args=["--window-position=0,0"])
            if a.only != "page":
                res["studio"] = bench_studio(b, a.rate)
            if a.only != "studio" and PAGE.exists():
                res["page"] = bench_page(b, a.rate)
            b.close()
    finally:
        srv.terminate()
    cols = list(res)
    print(f"CPU slowed {a.rate:g}x, {'window' if a.headed else 'headless'}, animation {a.anim or 'default'}")
    print(f"{'':34}" + "".join(f"{c:>12}" for c in cols))
    for k, lab, fmt in ROWS:
        if any(k in res[c] for c in cols):
            print(f"{lab:34}" + "".join(f"{(fmt.format(res[c][k]) if k in res[c] else '-'):>12}" for c in cols))
    for c in cols:
        if res[c].get("errors"):
            print(c, "errors:", res[c]["errors"])
    if a.json:
        Path(a.json).write_text(json.dumps(res, indent=1))


if __name__ == "__main__":
    main()