iOS: support canvas.a2ui push/reset

This commit is contained in:
Peter Steinberger
2025-12-18 10:44:32 +01:00
parent 0913329b03
commit 6f58a9d643
6 changed files with 109 additions and 7 deletions

View File

@@ -179,6 +179,8 @@ final class BridgeConnectionController {
ClawdisCanvasCommand.navigate.rawValue,
ClawdisCanvasCommand.evalJS.rawValue,
ClawdisCanvasCommand.snapshot.rawValue,
ClawdisCanvasA2UICommand.push.rawValue,
ClawdisCanvasA2UICommand.reset.rawValue,
]
let caps = Set(self.currentCaps())

View File

@@ -316,6 +316,56 @@ final class NodeAppModel {
let payload = try Self.encodePayload(["format": "png", "base64": base64])
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
case ClawdisCanvasA2UICommand.reset.rawValue:
try self.screen.showA2UI()
if await !self.screen.waitForA2UIReady(timeoutMs: 5000) {
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: ClawdisNodeError(code: .unavailable, message: "A2UI not ready"))
}
let json = try await self.screen.eval(javaScript: """
(() => {
if (!globalThis.clawdisA2UI) return JSON.stringify({ ok: false, error: "missing clawdisA2UI" });
return JSON.stringify(globalThis.clawdisA2UI.reset());
})()
""")
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: json)
case ClawdisCanvasA2UICommand.push.rawValue, ClawdisCanvasA2UICommand.pushJSONL.rawValue:
let messages: [AnyCodable]
if command == ClawdisCanvasA2UICommand.pushJSONL.rawValue {
let params = try Self.decodeParams(ClawdisCanvasA2UIPushJSONLParams.self, from: req.paramsJSON)
messages = try ClawdisCanvasA2UIJSONL.decodeMessagesFromJSONL(params.jsonl)
} else {
let params = try Self.decodeParams(ClawdisCanvasA2UIPushParams.self, from: req.paramsJSON)
messages = params.messages
}
try self.screen.showA2UI()
if await !self.screen.waitForA2UIReady(timeoutMs: 5000) {
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: ClawdisNodeError(code: .unavailable, message: "A2UI not ready"))
}
let messagesJSON = try ClawdisCanvasA2UIJSONL.encodeMessagesJSONArray(messages)
let js = """
(() => {
try {
if (!globalThis.clawdisA2UI) return JSON.stringify({ ok: false, error: "missing clawdisA2UI" });
const messages = \(messagesJSON);
return JSON.stringify(globalThis.clawdisA2UI.applyMessages(messages));
} catch (e) {
return JSON.stringify({ ok: false, error: String(e?.message ?? e) });
}
})()
"""
let resultJSON = try await self.screen.eval(javaScript: js)
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: resultJSON)
case ClawdisCameraCommand.snap.rawValue:
let params = (try? Self.decodeParams(ClawdisCameraSnapParams.self, from: req.paramsJSON)) ??
ClawdisCameraSnapParams()

View File

@@ -43,19 +43,64 @@ final class ScreenController {
func navigate(to urlString: String) {
self.urlString = urlString
if !urlString.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
// `canvas.navigate` is expected to show web content; default to WEB mode.
self.mode = .web
}
self.reload()
}
func reload() {
switch self.mode {
case .web:
guard let url = URL(string: self.urlString.trimmingCharacters(in: .whitespacesAndNewlines)) else { return }
self.webView.load(URLRequest(url: url))
let trimmed = self.urlString.trimmingCharacters(in: .whitespacesAndNewlines)
guard let url = URL(string: trimmed) else { return }
if url.isFileURL {
self.webView.loadFileURL(url, allowingReadAccessTo: url.deletingLastPathComponent())
} else {
self.webView.load(URLRequest(url: url))
}
case .canvas:
self.webView.loadHTMLString(Self.canvasScaffoldHTML, baseURL: nil)
}
}
func showA2UI() throws {
guard let url = ClawdisKitResources.bundle.url(
forResource: "index",
withExtension: "html",
subdirectory: "CanvasA2UI")
else {
throw NSError(domain: "Canvas", code: 10, userInfo: [
NSLocalizedDescriptionKey: "A2UI resources missing (CanvasA2UI/index.html)",
])
}
self.mode = .web
self.urlString = url.absoluteString
self.reload()
}
func waitForA2UIReady(timeoutMs: Int) async -> Bool {
let clock = ContinuousClock()
let deadline = clock.now.advanced(by: .milliseconds(timeoutMs))
while clock.now < deadline {
do {
let res = try await self.eval(javaScript: """
(() => {
try {
return !!globalThis.clawdisA2UI && typeof globalThis.clawdisA2UI.applyMessages === 'function';
} catch (_) { return false; }
})()
""")
if res == "true" { return true }
} catch {
// ignore; page likely still loading
}
try? await Task.sleep(nanoseconds: 120_000_000)
}
return false
}
func eval(javaScript: String) async throws -> String {
try await withCheckedThrowingContinuation { cont in
self.webView.evaluateJavaScript(javaScript) { result, error in

View File

@@ -305,6 +305,8 @@ struct SettingsTab: View {
ClawdisCanvasCommand.navigate.rawValue,
ClawdisCanvasCommand.evalJS.rawValue,
ClawdisCanvasCommand.snapshot.rawValue,
ClawdisCanvasA2UICommand.push.rawValue,
ClawdisCanvasA2UICommand.reset.rawValue,
]
let caps = Set(self.currentCaps())

View File

@@ -17,10 +17,9 @@ import WebKit
#expect(scrollView.bounces == false)
}
@Test @MainActor func webModeRejectsInvalidURLStrings() {
@Test @MainActor func navigateDefaultsToWebMode() {
let screen = ScreenController()
screen.navigate(to: "about:blank")
screen.setMode(.web)
screen.navigate(to: "not a url")
#expect(screen.mode == .web)
}

View File

@@ -119,10 +119,14 @@ Add to `src/gateway/protocol/schema.ts` (and regenerate Swift models):
### Node command set (canvas)
These are values for `node.invoke.command`:
- `canvas.show` / `canvas.hide`
- `canvas.navigate` with `{ url }` (Canvas URL or https URL)
- `canvas.navigate` with `{ url }` (Canvas URL or https URL; switches mode to `"web"`)
- `canvas.eval` with `{ javaScript }`
- `canvas.snapshot` with `{ maxWidth?, quality?, format? }`
- `canvas.setMode` with `{ mode: "canvas" | "web" }`
- `canvas.setMode` with `{ mode: "canvas" | "web" }` (use `"canvas"` to return to the scaffold)
- A2UI (mobile + macOS canvas):
- `canvas.a2ui.push` with `{ messages: [...] }` (A2UI v0.8 server→client messages)
- `canvas.a2ui.pushJSONL` with `{ jsonl: "..." }` (legacy alias)
- `canvas.a2ui.reset`
Result pattern:
- Request is a standard `req/res` with `ok` / `error`.