diff --git a/studio/agent.js b/studio/agent.js index 29d35d5..f0ad29c 100644 --- a/studio/agent.js +++ b/studio/agent.js @@ -251,6 +251,15 @@ ["z", "rot", "name", "tag", "props", "levels", "locked", "hidden"].forEach(function (k) { if (o[k] !== undefined && o[k] !== null && o[k] !== "") c[k] = o[k]; }); return c; } + /** World directions of the user's screen right and up, for "left of", "above", shapes meant to be read. */ + function screenAxes(rot) { + function dir(sx, sy) { + var a = IsoKit.unproject(rot, sx, sy, 0), o = IsoKit.unproject(rot, 0, 0, 0), x = a[0] - o[0], y = a[1] - o[1], l = Math.hypot(x, y) || 1; + return "(" + +(x / l).toFixed(3) + ", " + +(y / l).toFixed(3) + ")"; + } + return "On the user's screen, right is the world direction " + dir(1, 0) + " and up is " + dir(0, -1) + + " (unit vectors in x, y). The view is isometric: a metre along screen-up looks half as long as a metre along screen-right, so a shape that should read correctly on screen (a letter, a logo) needs twice the depth in world metres along screen-up."; + } function system() { var s = st(), S = s.S, sel = S.sel && s.obj(S.sel), v = s.view; return { @@ -259,9 +268,9 @@ "You are the agent inside the isokit Studio, the page where the user edits an isometric scene. You change the scene with the tools; every edit appears on the user's map at once, and the user can take any edit back with Undo (Ctrl+Z).", A.instructions, "How to work: look before you build (list_objects shows what is where with world boxes; list_prefabs with a type gives its props). Place things inside the site, on free ground, and use the boxes to avoid overlaps. Put related edits into one apply_ops call so they undo as one step. After bigger changes, check with render_view or check_project. Use select_object to show the user what you mean. Ask back when a request is unclear instead of guessing at big changes.", - "Answer in the user's language, short and plain, without tables: say what you changed and give object ids in backticks (the user can click them).", + "Answer in the language of the user's latest message, short and plain, without tables: say what you changed and give object ids in backticks (the user can click them). To show the result, write the answer first and call select_object in the same step: that ends your turn.", "The Studio right now: project \"" + S.meta.title + "\", " + (S.doc.objects || []).length + " objects; view level \"" + v.getLevel() + "\", rotation " + v.getRotation() + "; snap step " + (s.snapStep() || "off") + ". " + - (sel ? "Selected (\"this\", \"it\"): " + JSON.stringify(compact(sel)) : "Nothing is selected.") + (sel ? "Selected (\"this\", \"it\"): " + JSON.stringify(compact(sel)) : "Nothing is selected.") + " " + screenAxes(v.getRotation()) ].filter(Boolean).join("\n\n") }; } @@ -290,6 +299,9 @@ return null; } pending = out.tool_calls.slice(); + // pointing at things (select_object, set_view) after a written answer ends the turn: asked + // again with nothing left to say, models ramble (Qwen even switches language) + var gesture = !!m.content.trim() && out.tool_calls.every(function (c) { return PAGE[c["function"].name]; }); return out.tool_calls.reduce(function (p, c) { return p.then(function () { if (ctl.signal.aborted) return; @@ -300,7 +312,7 @@ pending.shift(); }); }); - }, Promise.resolve()).then(follow).then(function () { if (!ctl.signal.aborted) return turn(); }); + }, Promise.resolve()).then(follow).then(function () { if (!ctl.signal.aborted && !gesture) return turn(); }); }, function (e) { view.done(null); throw e; diff --git a/tools/agent_checks.py b/tools/agent_checks.py index 1fc0240..95b5287 100644 --- a/tools/agent_checks.py +++ b/tools/agent_checks.py @@ -81,6 +81,17 @@ def run(pg, objects, shot, check, can_set_key): pg.wait_for_function("() => !document.querySelector('.ag-busy')", timeout=10000) check(('"id":"%s"' % tid) in chats()[-1]["body"]["messages"][0]["content"], "the system prompt names the selection") + # an answer that ends by pointing at an object ends the turn there + n_chat = len(chats()) + pg.locator("#agtext").fill("show me " + tid) + pg.locator("#agtext").press("Enter") + pg.wait_for_function("() => !document.querySelector('.ag-busy')", timeout=10000) + pg.wait_for_timeout(600) + check(len(chats()) == n_chat + 1 and ("id " + tid) in pg.locator("#right .st-sub").inner_text(), + "answer + select_object ends the turn (no extra request), the object is selected") + tail = pg.evaluate("() => __agent.A.msgs.slice(-2).map(m => m.role + ':' + (m.tool_call_id || (m.tool_calls || []).length))") + check(tail == ["assistant:1", "tool:call_s"], f"the history stays valid for the next request {tail}") + # Undo takes the agent's edit back pg.locator("#undo").click() pg.wait_for_timeout(700) diff --git a/tools/mock_llm.py b/tools/mock_llm.py index 03ff42e..194d625 100644 --- a/tools/mock_llm.py +++ b/tools/mock_llm.py @@ -7,6 +7,8 @@ Wants the key `test-key`. Answers like a model that uses tools: list_objects {type: tree} and add_object {type: tree, x: X, y: Y, name: "Agent tree"} - tool results -> a short answer naming the new object's id in backticks - a user message containing "slow" -> a long, slow answer (to test Stop) +- a user message "show me X" -> an answer and a select_object call for X in one step + (that ends the turn: no request follows) - anything else -> a plain answer Streams (SSE) when asked to. Every page origin may call it (CORS, also the preflight). GET /_requests lists the requests seen (headers and bodies). @@ -76,6 +78,12 @@ class H(BaseHTTPRequestHandler): if last.get("role") == "user" and "slow" in last.get("content", ""): deltas = [{"content": "word%d " % i} for i in range(80)] return self.stream(body, deltas, finish, pause=0.1) + show = re.search(r"show me (\S+)", last.get("content", "")) if last.get("role") == "user" else None + if show: + deltas = [{"content": "Here is `%s`." % show.group(1)}, + {"tool_calls": [{"index": 0, "id": "call_s", "type": "function", + "function": {"name": "select_object", "arguments": json.dumps({"id": show.group(1)})}}]}] + return self.stream(body, deltas, "tool_calls") m = re.search(r"tree at (-?[\d.]+),\s*(-?[\d.]+)", last.get("content", "")) if last.get("role") == "user" else None if m: x, y = float(m.group(1)), float(m.group(2))