Canvas: fix A2UI v0.8 rendering

This commit is contained in:
Peter Steinberger
2025-12-17 13:20:27 +01:00
parent 81a9439eb2
commit 9eaa45a291
14 changed files with 301 additions and 134 deletions

View File

@@ -93,7 +93,7 @@ final class CanvasManager {
func eval(sessionKey: String, javaScript: String) async throws -> String { func eval(sessionKey: String, javaScript: String) async throws -> String {
_ = try self.show(sessionKey: sessionKey, path: nil) _ = try self.show(sessionKey: sessionKey, path: nil)
guard let controller = self.panelController else { return "" } guard let controller = self.panelController else { return "" }
return await controller.eval(javaScript: javaScript) return try await controller.eval(javaScript: javaScript)
} }
func snapshot(sessionKey: String, outPath: String?) async throws -> String { func snapshot(sessionKey: String, outPath: String?) async throws -> String {

View File

@@ -171,11 +171,11 @@ final class CanvasWindowController: NSWindowController, WKNavigationDelegate, NS
self.webView.loadFileURL(fileURL, allowingReadAccessTo: accessDir) self.webView.loadFileURL(fileURL, allowingReadAccessTo: accessDir)
} }
func eval(javaScript: String) async -> String { func eval(javaScript: String) async throws -> String {
await withCheckedContinuation { cont in try await withCheckedThrowingContinuation { cont in
self.webView.evaluateJavaScript(javaScript) { result, error in self.webView.evaluateJavaScript(javaScript) { result, error in
if let error { if let error {
cont.resume(returning: "error: \(error.localizedDescription)") cont.resume(throwing: error)
return return
} }
if let result { if let result {

View File

@@ -247,9 +247,12 @@ enum ControlRequestHandler {
case .reset: case .reset:
js = """ js = """
(() => { (() => {
if (!globalThis.clawdisA2UI) { return "missing clawdisA2UI"; } try {
globalThis.clawdisA2UI.reset(); if (!globalThis.clawdisA2UI) { return JSON.stringify({ ok: false, error: "missing clawdisA2UI" }); }
return "ok"; return JSON.stringify(globalThis.clawdisA2UI.reset());
} catch (e) {
return JSON.stringify({ ok: false, error: String(e?.message ?? e), stack: e?.stack });
}
})() })()
""" """
@@ -257,43 +260,100 @@ enum ControlRequestHandler {
guard let jsonl, !jsonl.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { guard let jsonl, !jsonl.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
return Response(ok: false, message: "missing jsonl") return Response(ok: false, message: "missing jsonl")
} }
let messages: [Any] let items: [ParsedJSONLItem]
do { do {
messages = try Self.parseJSONL(jsonl) items = try Self.parseJSONL(jsonl)
} catch { } catch {
return Response(ok: false, message: "invalid jsonl: \(error.localizedDescription)") return Response(ok: false, message: "invalid jsonl: \(error.localizedDescription)")
} }
do {
try Self.validateA2UIV0_8(items)
} catch {
return Response(ok: false, message: error.localizedDescription)
}
let messages = items.map(\.value)
let data = try JSONSerialization.data(withJSONObject: messages, options: []) let data = try JSONSerialization.data(withJSONObject: messages, options: [])
let json = String(data: data, encoding: .utf8) ?? "[]" let json = String(data: data, encoding: .utf8) ?? "[]"
js = """ js = """
(() => { (() => {
if (!globalThis.clawdisA2UI) { return "missing clawdisA2UI"; } try {
const messages = \(json); if (!globalThis.clawdisA2UI) { return JSON.stringify({ ok: false, error: "missing clawdisA2UI" }); }
globalThis.clawdisA2UI.applyMessages(messages); const messages = \(json);
return "ok"; return JSON.stringify(globalThis.clawdisA2UI.applyMessages(messages));
} catch (e) {
return JSON.stringify({ ok: false, error: String(e?.message ?? e), stack: e?.stack });
}
})() })()
""" """
} }
let result = try await CanvasManager.shared.eval(sessionKey: session, javaScript: js) let result = try await CanvasManager.shared.eval(sessionKey: session, javaScript: js)
return Response(ok: true, payload: Data(result.utf8))
let payload = Data(result.utf8)
if let obj = try? JSONSerialization.jsonObject(with: payload, options: []) as? [String: Any],
let ok = obj["ok"] as? Bool
{
let error = obj["error"] as? String
return Response(ok: ok, message: ok ? "" : (error ?? "A2UI error"), payload: payload)
}
return Response(ok: true, payload: payload)
} catch { } catch {
return Response(ok: false, message: error.localizedDescription) return Response(ok: false, message: error.localizedDescription)
} }
} }
private static func parseJSONL(_ text: String) throws -> [Any] { private struct ParsedJSONLItem {
var out: [Any] = [] let lineNumber: Int
for rawLine in text.split(whereSeparator: \.isNewline) { let value: Any
let line = rawLine.trimmingCharacters(in: .whitespacesAndNewlines) }
private static func parseJSONL(_ text: String) throws -> [ParsedJSONLItem] {
var out: [ParsedJSONLItem] = []
var lineNumber = 0
for rawLine in text.split(omittingEmptySubsequences: false, whereSeparator: \.isNewline) {
lineNumber += 1
let line = String(rawLine).trimmingCharacters(in: .whitespacesAndNewlines)
if line.isEmpty { continue } if line.isEmpty { continue }
let data = Data(line.utf8) let data = Data(line.utf8)
let obj = try JSONSerialization.jsonObject(with: data, options: []) let obj = try JSONSerialization.jsonObject(with: data, options: [])
out.append(obj) out.append(ParsedJSONLItem(lineNumber: lineNumber, value: obj))
} }
return out return out
} }
private static func validateA2UIV0_8(_ items: [ParsedJSONLItem]) throws {
let allowed = Set(["beginRendering", "surfaceUpdate", "dataModelUpdate", "deleteSurface"])
for item in items {
guard let dict = item.value as? [String: Any] else {
throw NSError(domain: "A2UI", code: 1, userInfo: [
NSLocalizedDescriptionKey: "A2UI JSONL line \(item.lineNumber): expected a JSON object",
])
}
if dict.keys.contains("createSurface") {
throw NSError(domain: "A2UI", code: 2, userInfo: [
NSLocalizedDescriptionKey: """
A2UI JSONL line \(item.lineNumber): looks like A2UI v0.9 (`createSurface`).
Canvas currently supports A2UI v0.8 server→client messages (`beginRendering`, `surfaceUpdate`, `dataModelUpdate`, `deleteSurface`).
""",
])
}
let matched = dict.keys.filter { allowed.contains($0) }
if matched.count != 1 {
let found = dict.keys.sorted().joined(separator: ", ")
throw NSError(domain: "A2UI", code: 3, userInfo: [
NSLocalizedDescriptionKey: """
A2UI JSONL line \(item.lineNumber): expected exactly one of \(allowed.sorted().joined(separator: ", ")); found: \(found)
""",
])
}
}
}
private static func waitForCanvasA2UI(session: String, timeoutMs: Int) async -> Bool { private static func waitForCanvasA2UI(session: String, timeoutMs: Int) async -> Bool {
let clock = ContinuousClock() let clock = ContinuousClock()
let deadline = clock.now.advanced(by: .milliseconds(timeoutMs)) let deadline = clock.now.advanced(by: .milliseconds(timeoutMs))

File diff suppressed because one or more lines are too long

View File

@@ -515,7 +515,7 @@ struct ClawdisCLI {
Canvas: Canvas:
clawdis-mac canvas show [--session <key>] [--target </...|https://...|file://...>] clawdis-mac canvas show [--session <key>] [--target </...|https://...|file://...>]
[--x <screenX> --y <screenY>] [--width <w> --height <h>] [--x <screenX> --y <screenY>] [--width <w> --height <h>]
clawdis-mac canvas a2ui push --jsonl <path> [--session <key>] clawdis-mac canvas a2ui push --jsonl <path> [--session <key>] # A2UI v0.8 JSONL
clawdis-mac canvas a2ui reset [--session <key>] clawdis-mac canvas a2ui reset [--session <key>]
clawdis-mac canvas hide [--session <key>] clawdis-mac canvas hide [--session <key>]
clawdis-mac canvas eval --js <code> [--session <key>] clawdis-mac canvas eval --js <code> [--session <key>]

View File

@@ -103,13 +103,27 @@ Related:
### Canvas A2UI ### Canvas A2UI
Canvas includes a built-in A2UI renderer (Lit-based). The agent can drive it with JSONL “message” objects: Canvas includes a built-in **A2UI v0.8** renderer (Lit-based). The agent can drive it with JSONL **server→client protocol messages** (one JSON object per line):
- `clawdis-mac canvas a2ui push --jsonl <path> [--session <key>]` - `clawdis-mac canvas a2ui push --jsonl <path> [--session <key>]`
- `clawdis-mac canvas a2ui reset [--session <key>]` - `clawdis-mac canvas a2ui reset [--session <key>]`
`push` expects a JSONL file where **each line is a single JSON object** (parsed and forwarded to the in-page A2UI renderer). `push` expects a JSONL file where **each line is a single JSON object** (parsed and forwarded to the in-page A2UI renderer).
Minimal example (v0.8):
```bash
cat > /tmp/a2ui-v0.8.jsonl <<'EOF'
{"surfaceUpdate":{"surfaceId":"main","components":[{"id":"root","component":{"Column":{"children":{"explicitList":["title","content"]}}}},{"id":"title","component":{"Text":{"text":{"literalString":"Canvas (A2UI v0.8)"},"usageHint":"h1"}}},{"id":"content","component":{"Text":{"text":{"literalString":"If you can read this, `canvas a2ui push` works."},"usageHint":"body"}}}]}}
{"beginRendering":{"surfaceId":"main","root":"root"}}
EOF
clawdis-mac canvas a2ui push --jsonl /tmp/a2ui-v0.8.jsonl --session main
```
Notes:
- This does **not** support the A2UI v0.9 examples using `createSurface`.
## Triggering agent runs from Canvas (deep links) ## Triggering agent runs from Canvas (deep links)
Canvas can trigger new agent runs via the macOS app deep-link scheme: Canvas can trigger new agent runs via the macOS app deep-link scheme:

File diff suppressed because one or more lines are too long

View File

@@ -1 +1 @@
{"version":3,"file":"model-processor.d.ts","sourceRoot":"","sources":["../../../../src/0.8/data/model-processor.ts"],"names":[],"mappings":"AAgBA,OAAO,EACL,qBAAqB,EACrB,gBAAgB,EAKhB,SAAS,EAIT,OAAO,EAGP,gBAAgB,EAGjB,MAAM,gBAAgB,CAAC;AA0BxB;;;GAGG;AACH,qBAAa,oBAAqB,YAAW,gBAAgB;;IAUzD,QAAQ,CAAC,IAAI,EAAE;QACb,OAAO,EAAE,cAAc,CAAC;QACxB,SAAS,EAAE,gBAAgB,CAAC;QAC5B,OAAO,EAAE,cAAc,CAAC;QACxB,OAAO,EAAE,iBAAiB,CAAC;KAC5B;IAdH,MAAM,CAAC,QAAQ,CAAC,kBAAkB,cAAc;gBASrC,IAAI,GAAE;QACb,OAAO,EAAE,cAAc,CAAC;QACxB,SAAS,EAAE,gBAAgB,CAAC;QAC5B,OAAO,EAAE,cAAc,CAAC;QACxB,OAAO,EAAE,iBAAiB,CAAC;KACwC;IAUvE,WAAW,IAAI,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC;IAI3C,aAAa;IAIb,eAAe,CAAC,QAAQ,EAAE,qBAAqB,EAAE,GAAG,IAAI;IA6BxD;;;;OAIG;IACH,OAAO,CACL,IAAI,EAAE,gBAAgB,EACtB,YAAY,EAAE,MAAM,EACpB,SAAS,SAA0C,GAClD,SAAS,GAAG,IAAI;IAkBnB,OAAO,CACL,IAAI,EAAE,gBAAgB,GAAG,IAAI,EAC7B,YAAY,EAAE,MAAM,EACpB,KAAK,EAAE,SAAS,EAChB,SAAS,SAA0C,GAClD,IAAI;IAuBP,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,eAAe,CAAC,EAAE,MAAM,GAAG,MAAM;CAkqB5D"} {"version":3,"file":"model-processor.d.ts","sourceRoot":"","sources":["../../../../src/0.8/data/model-processor.ts"],"names":[],"mappings":"AAgBA,OAAO,EACL,qBAAqB,EACrB,gBAAgB,EAKhB,SAAS,EAIT,OAAO,EAGP,gBAAgB,EAGjB,MAAM,gBAAgB,CAAC;AA0BxB;;;GAGG;AACH,qBAAa,oBAAqB,YAAW,gBAAgB;;IAUzD,QAAQ,CAAC,IAAI,EAAE;QACb,OAAO,EAAE,cAAc,CAAC;QACxB,SAAS,EAAE,gBAAgB,CAAC;QAC5B,OAAO,EAAE,cAAc,CAAC;QACxB,OAAO,EAAE,iBAAiB,CAAC;KAC5B;IAdH,MAAM,CAAC,QAAQ,CAAC,kBAAkB,cAAc;gBASrC,IAAI,GAAE;QACb,OAAO,EAAE,cAAc,CAAC;QACxB,SAAS,EAAE,gBAAgB,CAAC;QAC5B,OAAO,EAAE,cAAc,CAAC;QACxB,OAAO,EAAE,iBAAiB,CAAC;KACwC;IAUvE,WAAW,IAAI,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC;IAI3C,aAAa;IAIb,eAAe,CAAC,QAAQ,EAAE,qBAAqB,EAAE,GAAG,IAAI;IA6BxD;;;;OAIG;IACH,OAAO,CACL,IAAI,EAAE,gBAAgB,EACtB,YAAY,EAAE,MAAM,EACpB,SAAS,SAA0C,GAClD,SAAS,GAAG,IAAI;IAkBnB,OAAO,CACL,IAAI,EAAE,gBAAgB,GAAG,IAAI,EAC7B,YAAY,EAAE,MAAM,EACpB,KAAK,EAAE,SAAS,EAChB,SAAS,SAA0C,GAClD,IAAI;IAuBP,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,eAAe,CAAC,EAAE,MAAM,GAAG,MAAM;CA8qB5D"}

View File

@@ -360,7 +360,7 @@ export class A2uiMessageProcessor {
const resolvedProperties = new this.#objCtor(); const resolvedProperties = new this.#objCtor();
if (isObject(unresolvedProperties)) { if (isObject(unresolvedProperties)) {
for (const [key, value] of Object.entries(unresolvedProperties)) { for (const [key, value] of Object.entries(unresolvedProperties)) {
resolvedProperties[key] = this.#resolvePropertyValue(value, surface, visited, dataContextPath, idSuffix); resolvedProperties[key] = this.#resolvePropertyValue(value, surface, visited, dataContextPath, idSuffix, key);
} }
} }
visited.delete(fullId); visited.delete(fullId);
@@ -549,9 +549,13 @@ export class A2uiMessageProcessor {
* a child node (a string that matches a component ID), an explicitList of * a child node (a string that matches a component ID), an explicitList of
* children, or a template, these will be built out here. * children, or a template, these will be built out here.
*/ */
#resolvePropertyValue(value, surface, visited, dataContextPath, idSuffix = "") { #resolvePropertyValue(value, surface, visited, dataContextPath, idSuffix = "", propertyKey = null) {
const isComponentIdReferenceKey = (key) => key === "child" || key.endsWith("Child");
// 1. If it's a string that matches a component ID, build that node. // 1. If it's a string that matches a component ID, build that node.
if (typeof value === "string" && surface.components.has(value)) { if (typeof value === "string" &&
propertyKey &&
isComponentIdReferenceKey(propertyKey) &&
surface.components.has(value)) {
return this.#buildNodeRecursive(value, surface, visited, dataContextPath, idSuffix); return this.#buildNodeRecursive(value, surface, visited, dataContextPath, idSuffix);
} }
// 2. If it's a ComponentArrayReference (e.g., a `children` property), // 2. If it's a ComponentArrayReference (e.g., a `children` property),
@@ -597,7 +601,7 @@ export class A2uiMessageProcessor {
} }
// 3. If it's a plain array, resolve each of its items. // 3. If it's a plain array, resolve each of its items.
if (Array.isArray(value)) { if (Array.isArray(value)) {
return value.map((item) => this.#resolvePropertyValue(item, surface, visited, dataContextPath, idSuffix)); return value.map((item) => this.#resolvePropertyValue(item, surface, visited, dataContextPath, idSuffix, propertyKey));
} }
// 4. If it's a plain object, resolve each of its properties. // 4. If it's a plain object, resolve each of its properties.
if (isObject(value)) { if (isObject(value)) {
@@ -617,7 +621,7 @@ export class A2uiMessageProcessor {
newObj[key] = propertyValue; newObj[key] = propertyValue;
continue; continue;
} }
newObj[key] = this.#resolvePropertyValue(propertyValue, surface, visited, dataContextPath, idSuffix); newObj[key] = this.#resolvePropertyValue(propertyValue, surface, visited, dataContextPath, idSuffix, key);
} }
return newObj; return newObj;
} }

File diff suppressed because one or more lines are too long

View File

@@ -314,6 +314,43 @@ describe("A2uiMessageProcessor", () => {
assert.strictEqual(plainTree.properties.children[0].id, "child"); assert.strictEqual(plainTree.properties.children[0].id, "child");
assert.strictEqual(plainTree.properties.children[0].type, "Text"); assert.strictEqual(plainTree.properties.children[0].type, "Text");
}); });
it("should not treat enum-like strings as child component IDs", () => {
processor.processMessages([
{
surfaceUpdate: {
surfaceId: "@default",
components: [
{
id: "root",
component: {
Column: { children: { explicitList: ["body"] } },
},
},
{
id: "body",
component: {
Text: {
text: { literalString: "Hello" },
usageHint: "body",
},
},
},
],
},
},
{
beginRendering: {
root: "root",
surfaceId: "@default",
},
},
]);
const tree = processor.getSurfaces().get("@default")?.componentTree;
const plainTree = toPlainObject(tree);
assert.strictEqual(plainTree.id, "root");
assert.strictEqual(plainTree.properties.children[0].id, "body");
assert.strictEqual(plainTree.properties.children[0].type, "Text");
});
it("should throw an error on circular dependencies", () => { it("should throw an error on circular dependencies", () => {
// First, load the components // First, load the components
processor.processMessages([ processor.processMessages([

File diff suppressed because one or more lines are too long

View File

@@ -507,7 +507,8 @@ export class A2uiMessageProcessor implements MessageProcessor {
surface, surface,
visited, visited,
dataContextPath, dataContextPath,
idSuffix idSuffix,
key
); );
} }
} }
@@ -725,10 +726,19 @@ export class A2uiMessageProcessor implements MessageProcessor {
surface: Surface, surface: Surface,
visited: Set<string>, visited: Set<string>,
dataContextPath: string, dataContextPath: string,
idSuffix = "" idSuffix = "",
propertyKey: string | null = null
): ResolvedValue { ): ResolvedValue {
const isComponentIdReferenceKey = (key: string) =>
key === "child" || key.endsWith("Child");
// 1. If it's a string that matches a component ID, build that node. // 1. If it's a string that matches a component ID, build that node.
if (typeof value === "string" && surface.components.has(value)) { if (
typeof value === "string" &&
propertyKey &&
isComponentIdReferenceKey(propertyKey) &&
surface.components.has(value)
) {
return this.#buildNodeRecursive( return this.#buildNodeRecursive(
value, value,
surface, surface,
@@ -814,7 +824,8 @@ export class A2uiMessageProcessor implements MessageProcessor {
surface, surface,
visited, visited,
dataContextPath, dataContextPath,
idSuffix idSuffix,
propertyKey
) )
); );
} }
@@ -843,7 +854,8 @@ export class A2uiMessageProcessor implements MessageProcessor {
surface, surface,
visited, visited,
dataContextPath, dataContextPath,
idSuffix idSuffix,
key
); );
} }
return newObj; return newObj;

View File

@@ -375,6 +375,45 @@ describe("A2uiMessageProcessor", () => {
assert.strictEqual(plainTree.properties.children[0].type, "Text"); assert.strictEqual(plainTree.properties.children[0].type, "Text");
}); });
it("should not treat enum-like strings as child component IDs", () => {
processor.processMessages([
{
surfaceUpdate: {
surfaceId: "@default",
components: [
{
id: "root",
component: {
Column: { children: { explicitList: ["body"] } },
},
},
{
id: "body",
component: {
Text: {
text: { literalString: "Hello" },
usageHint: "body",
},
},
},
],
},
},
{
beginRendering: {
root: "root",
surfaceId: "@default",
},
},
]);
const tree = processor.getSurfaces().get("@default")?.componentTree;
const plainTree = toPlainObject(tree);
assert.strictEqual(plainTree.id, "root");
assert.strictEqual(plainTree.properties.children[0].id, "body");
assert.strictEqual(plainTree.properties.children[0].type, "Text");
});
it("should throw an error on circular dependencies", () => { it("should throw an error on circular dependencies", () => {
// First, load the components // First, load the components
processor.processMessages([ processor.processMessages([