grabowski
1a1b62d475
Deploy web Studio / deploy (push) Successful in 3m3s
✦ Agent in the Studio's top bar opens a floating chat window. The model (any OpenAI-compatible endpoint with tool calling; default B4L's inference server, https://inference.p0.contact/v1) gets the 15 MCP tools plus two page tools (select_object, set_view) and the Studio's state (selection, level, rotation) in its system prompt. Answers and reasoning stream in; tool calls show as rows with arguments and results; object ids link to the object; Stop ends a turn; Undo takes the agent's edits back. The window moves by its title bar and resizes at its top-left corner. - studio/agent.js: shared by both front ends (waits for studio.js's isokit:studio event); MCP over /mcp or isokit.mcp in the web Studio - isokit studio: /api/agent/{config,models,chat} relay the calls on their own threads (ureq, chunked and flushed per read), so the key stays on the machine and the endpoint needs no CORS; settings from ISOKIT_LLM_URL / _KEY / _MODEL or agent.json in the user's config folder - web Studio: calls the endpoint from the page with a key kept in the browser; build_web.py --agent-url names a same-site relay as default; the deploy builds with /llm/v1, which deploy/Caddyfile forwards to the inference server (its nginx refuses the CORS preflight) - tools/mock_llm.py + tools/agent_checks.py: the agent window checks in both check_studio.py and check_web.py
161 lines
8.1 KiB
Python
161 lines
8.1 KiB
Python
"""Agent window checks, shared by check_studio.py (isokit studio) and
|
|
check_web.py (web Studio). The page's agent must use the mock model
|
|
(tools/mock_llm.py, key test-key); `start_mock()` runs one."""
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
MOCK_PORT = 7932
|
|
MOCK = f"http://127.0.0.1:{MOCK_PORT}"
|
|
|
|
|
|
def start_mock():
|
|
p = subprocess.Popen([sys.executable, str(Path(__file__).with_name("mock_llm.py")), str(MOCK_PORT)],
|
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
|
p.stdout.readline() # the banner: listening
|
|
return p
|
|
|
|
|
|
def seen():
|
|
return json.loads(urllib.request.urlopen(MOCK + "/_requests").read())
|
|
|
|
|
|
def chats():
|
|
return [r for r in seen() if r.get("path") == "/v1/chat/completions"]
|
|
|
|
|
|
def run(pg, objects, shot, check, can_set_key):
|
|
"""objects(): the scene's objects now. can_set_key: the key is set in the
|
|
window (web), not by the Studio server's environment."""
|
|
pg.locator("#agent").click()
|
|
check(pg.locator("#agwin").is_visible(), "✦ Agent opens the agent window")
|
|
pg.wait_for_function("() => /mock-1/.test(document.getElementById('agwho').textContent)", timeout=8000)
|
|
check(pg.locator("#agwho .ag-on").count() == 1, "the window connects to the model: " + pg.locator("#agwho").inner_text())
|
|
check(pg.locator(".ag-sug").count() >= 3, "an empty chat offers suggestions")
|
|
shot("agent-empty.jpg")
|
|
|
|
# a turn with two tool calls, then an answer
|
|
n0 = len(objects())
|
|
pg.locator("#agtext").fill("Plant a tree at 6, 4 please")
|
|
pg.locator("#agtext").press("Enter")
|
|
pg.wait_for_selector(".ag-bot:not(.ag-live) .ag-text code.ag-obj", timeout=15000)
|
|
objs = objects()
|
|
new = [o for o in objs if o.get("name") == "Agent tree"]
|
|
check(len(objs) == n0 + 1 and len(new) == 1 and new[0]["x"] == 6 and new[0]["y"] == 4,
|
|
"the agent's add_object lands in the scene at (6, 4)")
|
|
tid = new[0]["id"] if new else "?"
|
|
check(pg.locator(".ag-tool.ag-ok").count() == 2, "both tool calls show as done")
|
|
check(pg.locator(f'.ag-tool code.ag-obj[data-obj="{tid}"]').count() == 1, "the add_object row links the new object")
|
|
check(pg.locator(f'#objects li[data-id="{tid}"]').count() == 1, "the object list shows the agent's tree at once")
|
|
check(pg.locator(f'.iso-svg g[data-obj="{tid}"]').count() >= 1, "the map draws the agent's tree")
|
|
check(pg.locator(".ag-think summary").first.inner_text().startswith("Thought for"), "the reasoning folds away: "
|
|
+ pg.locator(".ag-think summary").first.inner_text())
|
|
txt = pg.locator(".ag-bot .ag-text").last.inner_text()
|
|
check("Planted" in txt and tid in txt, "the answer streams in: " + txt.replace("\n", " ")[:70])
|
|
check(pg.locator(".ag-bot .ag-text li").count() == 2 and pg.locator(".ag-bot .ag-text b").count() == 1,
|
|
"markdown list and bold render")
|
|
shot("agent-chat.jpg")
|
|
|
|
c = chats()
|
|
first, second = c[-2]["body"], c[-1]["body"]
|
|
sysmsg = first["messages"][0]
|
|
names = [t["function"]["name"] for t in first.get("tools", [])]
|
|
check(all(r["auth"] == "Bearer test-key" for r in c[-2:]), "the key reaches the model")
|
|
check(sysmsg["role"] == "system" and "isokit Studio" in sysmsg["content"] and "Nothing is selected" in sysmsg["content"]
|
|
and "project_info" in sysmsg["content"], "the system prompt has the MCP instructions and the Studio state")
|
|
check(len(names) == 17 and {"add_object", "apply_ops", "select_object", "set_view"} <= set(names),
|
|
f"the model gets 15 MCP tools + 2 page tools ({len(names)})")
|
|
check(first.get("stream") is True and first.get("model") == "mock-1", "streamed, first offered model by default")
|
|
tool_ids = [m.get("tool_call_id") for m in second["messages"] if m["role"] == "tool"]
|
|
check(tool_ids == ["call_a", "call_b"], f"tool results go back with their call ids {tool_ids}")
|
|
|
|
# the id in the answer selects the object
|
|
pg.locator(f'.ag-bot .ag-text code.ag-obj[data-obj="{tid}"]').click()
|
|
pg.wait_for_timeout(500)
|
|
check(("id " + tid) in pg.locator("#right .st-sub").inner_text(), "clicking the id selects the agent's tree")
|
|
# the selection reaches the model in the next turn; a plain answer
|
|
pg.locator("#agtext").fill("hello")
|
|
pg.locator("#agtext").press("Enter")
|
|
pg.wait_for_function("() => !document.querySelector('.ag-busy')", timeout=10000)
|
|
check(('"id":"%s"' % tid) in chats()[-1]["body"]["messages"][0]["content"], "the system prompt names the selection")
|
|
|
|
# Undo takes the agent's edit back
|
|
pg.locator("#undo").click()
|
|
pg.wait_for_timeout(700)
|
|
check(not any(o.get("name") == "Agent tree" for o in objects()), "Undo removes the agent's tree")
|
|
|
|
# Stop a long answer
|
|
pg.locator("#agtext").fill("tell me a slow story")
|
|
pg.locator("#agtext").press("Enter")
|
|
pg.wait_for_timeout(900)
|
|
check(pg.locator("#agsend").inner_text() == "Stop", "while the agent works, Send turns into Stop")
|
|
pg.locator("#agsend").click()
|
|
pg.wait_for_selector(".ag-note:has-text('Stopped')", timeout=5000)
|
|
check(pg.locator("#agsend").inner_text() == "Send" and "word1" in pg.locator(".ag-bot .ag-text").last.inner_text(),
|
|
"Stop ends the turn and keeps what came")
|
|
|
|
# settings: pick another model
|
|
def settings():
|
|
pg.wait_for_timeout(900) # a good save folds the form away after a moment
|
|
if pg.locator("#agsetf").is_hidden():
|
|
pg.locator("#agset").click()
|
|
|
|
def save():
|
|
pg.locator('#agsetf button[type="submit"]').click()
|
|
pg.wait_for_timeout(700)
|
|
|
|
settings()
|
|
check(pg.locator("#agsetf").is_visible(), "⚙ shows the settings")
|
|
opts = pg.locator("#agmodel option").all_inner_texts()
|
|
check("mock-2" in opts, f"the model list comes from the endpoint {opts}")
|
|
pg.locator("#agmodel").select_option("mock-2")
|
|
save()
|
|
pg.wait_for_function("() => /mock-2/.test(document.getElementById('agwho').textContent)", timeout=5000)
|
|
pg.locator("#agtext").fill("hello again")
|
|
pg.locator("#agtext").press("Enter")
|
|
pg.wait_for_function("() => !document.querySelector('.ag-busy')", timeout=10000)
|
|
check(chats()[-1]["body"]["model"] == "mock-2", "the chosen model is used")
|
|
if can_set_key:
|
|
settings()
|
|
pg.locator("#agkey").fill("wrong-key")
|
|
save()
|
|
check("401" in pg.locator("#agsetmsg").inner_text(), "a wrong key shows the endpoint's refusal: " + pg.locator("#agsetmsg").inner_text())
|
|
check(pg.locator("#agwho .ag-off").count() == 1, "the status dot turns red")
|
|
shot("agent-badkey.jpg")
|
|
pg.locator("#agkey").fill("test-key")
|
|
save()
|
|
check(pg.locator("#agwho .ag-on").count() == 1, "the right key connects again")
|
|
else:
|
|
settings()
|
|
check(pg.locator("#agurl").is_disabled() and "ISOKIT_LLM_KEY" in pg.locator("#agkey").get_attribute("placeholder"),
|
|
"endpoint and key set by the environment are shown as such")
|
|
shot("agent-settings.jpg")
|
|
settings()
|
|
pg.locator("#agmodel").select_option("")
|
|
save()
|
|
check("mock-1" in pg.locator("#agwho").inner_text(), "back to the first offered model")
|
|
|
|
# move and resize the window
|
|
box = pg.locator("#agwin").bounding_box()
|
|
hb = pg.locator(".ag-logo").bounding_box()
|
|
pg.mouse.move(hb["x"] + 5, hb["y"] + 5)
|
|
pg.mouse.down()
|
|
pg.mouse.move(hb["x"] - 95, hb["y"] - 35, steps=5)
|
|
pg.mouse.up()
|
|
box2 = pg.locator("#agwin").bounding_box()
|
|
check(abs(box2["x"] - box["x"] + 100) < 3 and abs(box2["y"] - box["y"] + 40) < 3, "the window moves by its title bar")
|
|
gb = pg.locator("#aggrip").bounding_box()
|
|
pg.mouse.move(gb["x"] + 4, gb["y"] + 4)
|
|
pg.mouse.down()
|
|
pg.mouse.move(gb["x"] - 56, gb["y"] - 26, steps=5)
|
|
pg.mouse.up()
|
|
box3 = pg.locator("#agwin").bounding_box()
|
|
check(abs(box3["width"] - box2["width"] - 60) < 3 and abs(box3["height"] - box2["height"] - 30) < 3, "the top-left grip resizes it")
|
|
|
|
pg.locator("#agnew").click()
|
|
check(pg.locator(".ag-empty").count() == 1 and pg.locator(".ag-tool").count() == 0, "New chat clears the conversation")
|
|
pg.locator("#agclose").click()
|
|
check(pg.locator("#agwin").is_hidden() and pg.locator("#agent").get_attribute("aria-pressed") == "false", "✕ closes the window")
|