grabowski
84309a7a08
Deploy web Studio / deploy (push) Successful in 3m33s
Found while recording a demo with Qwen3.8 on the inference server:
- When the model wrote its answer and then called select_object, it was
asked again with nothing left to say and rambled ("the user hasn't sent
a message yet"), sometimes in Chinese. A step of page tools only
(select_object, set_view) after a written answer now ends the turn; the
system prompt says so, and to answer in the language of the latest message.
- The system prompt names the world directions of the user's screen right
and up, and the 2:1 foreshortening, so "left of the hall" and shapes meant
to be read on screen come out right (asked for a giant W of people, the
model now draws it upright and readable).
- mock_llm.py "show me X" + check: answer and select_object in one step end
the turn without another request, and the history stays valid.
173 lines
8.9 KiB
Python
173 lines
8.9 KiB
Python
"""Agent window checks, shared by check_studio.py (isokit studio) and
|
|
check_web.py (web Studio). The page's agent must use the mock model
|
|
(tools/mock_llm.py, key test-key); `start_mock()` runs one."""
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
MOCK_PORT = 7932
|
|
MOCK = f"http://127.0.0.1:{MOCK_PORT}"
|
|
|
|
|
|
def start_mock():
|
|
p = subprocess.Popen([sys.executable, str(Path(__file__).with_name("mock_llm.py")), str(MOCK_PORT)],
|
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
|
p.stdout.readline() # the banner: listening
|
|
return p
|
|
|
|
|
|
def seen():
|
|
return json.loads(urllib.request.urlopen(MOCK + "/_requests").read())
|
|
|
|
|
|
def chats():
|
|
return [r for r in seen() if r.get("path") == "/v1/chat/completions"]
|
|
|
|
|
|
def run(pg, objects, shot, check, can_set_key):
|
|
"""objects(): the scene's objects now. can_set_key: the key is set in the
|
|
window (web), not by the Studio server's environment."""
|
|
pg.locator("#agent").click()
|
|
check(pg.locator("#agwin").is_visible(), "✦ Agent opens the agent window")
|
|
pg.wait_for_function("() => /mock-1/.test(document.getElementById('agwho').textContent)", timeout=8000)
|
|
check(pg.locator("#agwho .ag-on").count() == 1, "the window connects to the model: " + pg.locator("#agwho").inner_text())
|
|
check(pg.locator(".ag-sug").count() >= 3, "an empty chat offers suggestions")
|
|
shot("agent-empty.jpg")
|
|
|
|
# a turn with two tool calls, then an answer
|
|
n0 = len(objects())
|
|
pg.locator("#agtext").fill("Plant a tree at 6, 4 please")
|
|
pg.locator("#agtext").press("Enter")
|
|
pg.wait_for_selector(".ag-bot:not(.ag-live) .ag-text code.ag-obj", timeout=15000)
|
|
objs = objects()
|
|
new = [o for o in objs if o.get("name") == "Agent tree"]
|
|
check(len(objs) == n0 + 1 and len(new) == 1 and new[0]["x"] == 6 and new[0]["y"] == 4,
|
|
"the agent's add_object lands in the scene at (6, 4)")
|
|
tid = new[0]["id"] if new else "?"
|
|
check(pg.locator(".ag-tool.ag-ok").count() == 2, "both tool calls show as done")
|
|
check(pg.locator(f'.ag-tool code.ag-obj[data-obj="{tid}"]').count() == 1, "the add_object row links the new object")
|
|
check(pg.locator(f'#objects li[data-id="{tid}"]').count() == 1, "the object list shows the agent's tree at once")
|
|
check(pg.locator(f'.iso-svg g[data-obj="{tid}"]').count() >= 1, "the map draws the agent's tree")
|
|
check(pg.locator(".ag-think summary").first.inner_text().startswith("Thought for"), "the reasoning folds away: "
|
|
+ pg.locator(".ag-think summary").first.inner_text())
|
|
txt = pg.locator(".ag-bot .ag-text").last.inner_text()
|
|
check("Planted" in txt and tid in txt, "the answer streams in: " + txt.replace("\n", " ")[:70])
|
|
check(pg.locator(".ag-bot .ag-text li").count() == 2 and pg.locator(".ag-bot .ag-text b").count() == 1,
|
|
"markdown list and bold render")
|
|
shot("agent-chat.jpg")
|
|
|
|
c = chats()
|
|
first, second = c[-2]["body"], c[-1]["body"]
|
|
sysmsg = first["messages"][0]
|
|
names = [t["function"]["name"] for t in first.get("tools", [])]
|
|
check(all(r["auth"] == "Bearer test-key" for r in c[-2:]), "the key reaches the model")
|
|
check(sysmsg["role"] == "system" and "isokit Studio" in sysmsg["content"] and "Nothing is selected" in sysmsg["content"]
|
|
and "project_info" in sysmsg["content"], "the system prompt has the MCP instructions and the Studio state")
|
|
check(len(names) == 17 and {"add_object", "apply_ops", "select_object", "set_view"} <= set(names),
|
|
f"the model gets 15 MCP tools + 2 page tools ({len(names)})")
|
|
check(first.get("stream") is True and first.get("model") == "mock-1", "streamed, first offered model by default")
|
|
tool_ids = [m.get("tool_call_id") for m in second["messages"] if m["role"] == "tool"]
|
|
check(tool_ids == ["call_a", "call_b"], f"tool results go back with their call ids {tool_ids}")
|
|
|
|
# the id in the answer selects the object
|
|
pg.locator(f'.ag-bot .ag-text code.ag-obj[data-obj="{tid}"]').click()
|
|
pg.wait_for_timeout(500)
|
|
check(("id " + tid) in pg.locator("#right .st-sub").inner_text(), "clicking the id selects the agent's tree")
|
|
# the selection reaches the model in the next turn; a plain answer
|
|
pg.locator("#agtext").fill("hello")
|
|
pg.locator("#agtext").press("Enter")
|
|
pg.wait_for_function("() => !document.querySelector('.ag-busy')", timeout=10000)
|
|
check(('"id":"%s"' % tid) in chats()[-1]["body"]["messages"][0]["content"], "the system prompt names the selection")
|
|
|
|
# an answer that ends by pointing at an object ends the turn there
|
|
n_chat = len(chats())
|
|
pg.locator("#agtext").fill("show me " + tid)
|
|
pg.locator("#agtext").press("Enter")
|
|
pg.wait_for_function("() => !document.querySelector('.ag-busy')", timeout=10000)
|
|
pg.wait_for_timeout(600)
|
|
check(len(chats()) == n_chat + 1 and ("id " + tid) in pg.locator("#right .st-sub").inner_text(),
|
|
"answer + select_object ends the turn (no extra request), the object is selected")
|
|
tail = pg.evaluate("() => __agent.A.msgs.slice(-2).map(m => m.role + ':' + (m.tool_call_id || (m.tool_calls || []).length))")
|
|
check(tail == ["assistant:1", "tool:call_s"], f"the history stays valid for the next request {tail}")
|
|
|
|
# Undo takes the agent's edit back
|
|
pg.locator("#undo").click()
|
|
pg.wait_for_timeout(700)
|
|
check(not any(o.get("name") == "Agent tree" for o in objects()), "Undo removes the agent's tree")
|
|
|
|
# Stop a long answer
|
|
pg.locator("#agtext").fill("tell me a slow story")
|
|
pg.locator("#agtext").press("Enter")
|
|
pg.wait_for_timeout(900)
|
|
check(pg.locator("#agsend").inner_text() == "Stop", "while the agent works, Send turns into Stop")
|
|
pg.locator("#agsend").click()
|
|
pg.wait_for_selector(".ag-note:has-text('Stopped')", timeout=5000)
|
|
check(pg.locator("#agsend").inner_text() == "Send" and "word1" in pg.locator(".ag-bot .ag-text").last.inner_text(),
|
|
"Stop ends the turn and keeps what came")
|
|
|
|
# settings: pick another model
|
|
def settings():
|
|
pg.wait_for_timeout(900) # a good save folds the form away after a moment
|
|
if pg.locator("#agsetf").is_hidden():
|
|
pg.locator("#agset").click()
|
|
|
|
def save():
|
|
pg.locator('#agsetf button[type="submit"]').click()
|
|
pg.wait_for_timeout(700)
|
|
|
|
settings()
|
|
check(pg.locator("#agsetf").is_visible(), "⚙ shows the settings")
|
|
opts = pg.locator("#agmodel option").all_inner_texts()
|
|
check("mock-2" in opts, f"the model list comes from the endpoint {opts}")
|
|
pg.locator("#agmodel").select_option("mock-2")
|
|
save()
|
|
pg.wait_for_function("() => /mock-2/.test(document.getElementById('agwho').textContent)", timeout=5000)
|
|
pg.locator("#agtext").fill("hello again")
|
|
pg.locator("#agtext").press("Enter")
|
|
pg.wait_for_function("() => !document.querySelector('.ag-busy')", timeout=10000)
|
|
check(chats()[-1]["body"]["model"] == "mock-2", "the chosen model is used")
|
|
if can_set_key:
|
|
settings()
|
|
pg.locator("#agkey").fill("wrong-key")
|
|
save()
|
|
check("refused the API key" in pg.locator("#agsetmsg").inner_text() and "invalid api key" in pg.locator("#agsetmsg").inner_text(),
|
|
"a wrong key shows the endpoint's refusal: " + pg.locator("#agsetmsg").inner_text())
|
|
check(pg.locator("#agwho .ag-off").count() == 1, "the status dot turns red")
|
|
shot("agent-badkey.jpg")
|
|
pg.locator("#agkey").fill("test-key")
|
|
save()
|
|
check(pg.locator("#agwho .ag-on").count() == 1, "the right key connects again")
|
|
else:
|
|
settings()
|
|
check(pg.locator("#agurl").is_disabled() and "ISOKIT_LLM_KEY" in pg.locator("#agkey").get_attribute("placeholder"),
|
|
"endpoint and key set by the environment are shown as such")
|
|
shot("agent-settings.jpg")
|
|
settings()
|
|
pg.locator("#agmodel").select_option("")
|
|
save()
|
|
check("mock-1" in pg.locator("#agwho").inner_text(), "back to the first offered model")
|
|
|
|
# move and resize the window
|
|
box = pg.locator("#agwin").bounding_box()
|
|
hb = pg.locator(".ag-logo").bounding_box()
|
|
pg.mouse.move(hb["x"] + 5, hb["y"] + 5)
|
|
pg.mouse.down()
|
|
pg.mouse.move(hb["x"] - 95, hb["y"] - 35, steps=5)
|
|
pg.mouse.up()
|
|
box2 = pg.locator("#agwin").bounding_box()
|
|
check(abs(box2["x"] - box["x"] + 100) < 3 and abs(box2["y"] - box["y"] + 40) < 3, "the window moves by its title bar")
|
|
gb = pg.locator("#aggrip").bounding_box()
|
|
pg.mouse.move(gb["x"] + 4, gb["y"] + 4)
|
|
pg.mouse.down()
|
|
pg.mouse.move(gb["x"] - 56, gb["y"] - 26, steps=5)
|
|
pg.mouse.up()
|
|
box3 = pg.locator("#agwin").bounding_box()
|
|
check(abs(box3["width"] - box2["width"] - 60) < 3 and abs(box3["height"] - box2["height"] - 30) < 3, "the top-left grip resizes it")
|
|
|
|
pg.locator("#agnew").click()
|
|
check(pg.locator(".ag-empty").count() == 1 and pg.locator(".ag-tool").count() == 0, "New chat clears the conversation")
|
|
pg.locator("#agclose").click()
|
|
check(pg.locator("#agwin").is_hidden() and pg.locator("#agent").get_attribute("aria-pressed") == "false", "✕ closes the window")
|