agent window: screen directions, pointing ends the turn
Deploy web Studio / deploy (push) Successful in 3m33s

Found while recording a demo with Qwen3.8 on the inference server:

- When the model wrote its answer and then called select_object, it was
  asked again with nothing left to say and rambled ("the user hasn't sent
  a message yet"), sometimes in Chinese. A step of page tools only
  (select_object, set_view) after a written answer now ends the turn; the
  system prompt says so, and to answer in the language of the latest message.
- The system prompt names the world directions of the user's screen right
  and up, and the 2:1 foreshortening, so "left of the hall" and shapes meant
  to be read on screen come out right (asked for a giant W of people, the
  model now draws it upright and readable).
- mock_llm.py "show me X" + check: answer and select_object in one step end
  the turn without another request, and the history stays valid.
This commit is contained in:
grabowski committed 2026-10-23 20:29:17 +07:00
1 parent 36f7a9f440
commit 84309a7a08
3 files changed
+34 -3

No files matched your search

+15 -3
View File
@@ -251,6 +251,15 @@
["z", "rot", "name", "tag", "props", "levels", "locked", "hidden"].forEach(function (k) { if (o[k] !== undefined && o[k] !== null && o[k] !== "") c[k] = o[k]; });
return c;
}
/** World directions of the user's screen right and up, for "left of", "above", shapes meant to be read. */
function screenAxes(rot) {
function dir(sx, sy) {
var a = IsoKit.unproject(rot, sx, sy, 0), o = IsoKit.unproject(rot, 0, 0, 0), x = a[0] - o[0], y = a[1] - o[1], l = Math.hypot(x, y) || 1;
return "(" + +(x / l).toFixed(3) + ", " + +(y / l).toFixed(3) + ")";
}
return "On the user's screen, right is the world direction " + dir(1, 0) + " and up is " + dir(0, -1) +
" (unit vectors in x, y). The view is isometric: a metre along screen-up looks half as long as a metre along screen-right, so a shape that should read correctly on screen (a letter, a logo) needs twice the depth in world metres along screen-up.";
}
function system() {
var s = st(), S = s.S, sel = S.sel && s.obj(S.sel), v = s.view;
return {
@@ -259,9 +268,9 @@
"You are the agent inside the isokit Studio, the page where the user edits an isometric scene. You change the scene with the tools; every edit appears on the user's map at once, and the user can take any edit back with Undo (Ctrl+Z).",
A.instructions,
"How to work: look before you build (list_objects shows what is where with world boxes; list_prefabs with a type gives its props). Place things inside the site, on free ground, and use the boxes to avoid overlaps. Put related edits into one apply_ops call so they undo as one step. After bigger changes, check with render_view or check_project. Use select_object to show the user what you mean. Ask back when a request is unclear instead of guessing at big changes.",
"Answer in the user's language, short and plain, without tables: say what you changed and give object ids in backticks (the user can click them).",
"Answer in the language of the user's latest message, short and plain, without tables: say what you changed and give object ids in backticks (the user can click them). To show the result, write the answer first and call select_object in the same step: that ends your turn.",
"The Studio right now: project \"" + S.meta.title + "\", " + (S.doc.objects || []).length + " objects; view level \"" + v.getLevel() + "\", rotation " + v.getRotation() + "; snap step " + (s.snapStep() || "off") + ". " +
(sel ? "Selected (\"this\", \"it\"): " + JSON.stringify(compact(sel)) : "Nothing is selected.")
(sel ? "Selected (\"this\", \"it\"): " + JSON.stringify(compact(sel)) : "Nothing is selected.") + " " + screenAxes(v.getRotation())
].filter(Boolean).join("\n\n")
};
}
@@ -290,6 +299,9 @@
return null;
}
pending = out.tool_calls.slice();
// pointing at things (select_object, set_view) after a written answer ends the turn: asked
// again with nothing left to say, models ramble (Qwen even switches language)
var gesture = !!m.content.trim() && out.tool_calls.every(function (c) { return PAGE[c["function"].name]; });
return out.tool_calls.reduce(function (p, c) {
return p.then(function () {
if (ctl.signal.aborted) return;
@@ -300,7 +312,7 @@
pending.shift();
});
});
}, Promise.resolve()).then(follow).then(function () { if (!ctl.signal.aborted) return turn(); });
}, Promise.resolve()).then(follow).then(function () { if (!ctl.signal.aborted && !gesture) return turn(); });
}, function (e) {
view.done(null);
throw e;