import crypto from "node:crypto"; import fs from "node:fs/promises"; import { Type } from "@sinclair/typebox"; import { writeBase64ToFile } from "../../cli/nodes-camera.js"; import { canvasSnapshotTempPath, parseCanvasSnapshotPayload, } from "../../cli/nodes-canvas.js"; import { imageMimeFromFormat } from "../../media/mime.js"; import { type AnyAgentTool, imageResult, jsonResult, readStringParam, } from "./common.js"; import { callGatewayTool, type GatewayCallOptions } from "./gateway.js"; import { resolveNodeId } from "./nodes-utils.js"; const CanvasToolSchema = Type.Union([ Type.Object({ action: Type.Literal("present"), gatewayUrl: Type.Optional(Type.String()), gatewayToken: Type.Optional(Type.String()), timeoutMs: Type.Optional(Type.Number()), node: Type.Optional(Type.String()), target: Type.Optional(Type.String()), x: Type.Optional(Type.Number()), y: Type.Optional(Type.Number()), width: Type.Optional(Type.Number()), height: Type.Optional(Type.Number()), }), Type.Object({ action: Type.Literal("hide"), gatewayUrl: Type.Optional(Type.String()), gatewayToken: Type.Optional(Type.String()), timeoutMs: Type.Optional(Type.Number()), node: Type.Optional(Type.String()), }), Type.Object({ action: Type.Literal("navigate"), gatewayUrl: Type.Optional(Type.String()), gatewayToken: Type.Optional(Type.String()), timeoutMs: Type.Optional(Type.Number()), node: Type.Optional(Type.String()), url: Type.String(), }), Type.Object({ action: Type.Literal("eval"), gatewayUrl: Type.Optional(Type.String()), gatewayToken: Type.Optional(Type.String()), timeoutMs: Type.Optional(Type.Number()), node: Type.Optional(Type.String()), javaScript: Type.String(), }), Type.Object({ action: Type.Literal("snapshot"), gatewayUrl: Type.Optional(Type.String()), gatewayToken: Type.Optional(Type.String()), timeoutMs: Type.Optional(Type.Number()), node: Type.Optional(Type.String()), format: Type.Optional( Type.Union([ Type.Literal("png"), Type.Literal("jpg"), Type.Literal("jpeg"), ]), ), maxWidth: Type.Optional(Type.Number()), quality: Type.Optional(Type.Number()), delayMs: Type.Optional(Type.Number()), }), Type.Object({ action: Type.Literal("a2ui_push"), gatewayUrl: Type.Optional(Type.String()), gatewayToken: Type.Optional(Type.String()), timeoutMs: Type.Optional(Type.Number()), node: Type.Optional(Type.String()), jsonl: Type.Optional(Type.String()), jsonlPath: Type.Optional(Type.String()), }), Type.Object({ action: Type.Literal("a2ui_reset"), gatewayUrl: Type.Optional(Type.String()), gatewayToken: Type.Optional(Type.String()), timeoutMs: Type.Optional(Type.Number()), node: Type.Optional(Type.String()), }), ]); export function createCanvasTool(): AnyAgentTool { return { label: "Canvas", name: "canvas", description: "Control node canvases (present/hide/navigate/eval/snapshot/A2UI). Use snapshot to capture the rendered UI.", parameters: CanvasToolSchema, execute: async (_toolCallId, args) => { const params = args as Record; const action = readStringParam(params, "action", { required: true }); const gatewayOpts: GatewayCallOptions = { gatewayUrl: readStringParam(params, "gatewayUrl", { trim: false }), gatewayToken: readStringParam(params, "gatewayToken", { trim: false }), timeoutMs: typeof params.timeoutMs === "number" ? params.timeoutMs : undefined, }; const nodeId = await resolveNodeId( gatewayOpts, readStringParam(params, "node", { trim: true }), true, ); const invoke = async ( command: string, invokeParams?: Record, ) => await callGatewayTool("node.invoke", gatewayOpts, { nodeId, command, params: invokeParams, idempotencyKey: crypto.randomUUID(), }); switch (action) { case "present": { const placement = { x: typeof params.x === "number" ? params.x : undefined, y: typeof params.y === "number" ? params.y : undefined, width: typeof params.width === "number" ? params.width : undefined, height: typeof params.height === "number" ? params.height : undefined, }; const invokeParams: Record = {}; if (typeof params.target === "string" && params.target.trim()) { invokeParams.url = params.target.trim(); } if ( Number.isFinite(placement.x) || Number.isFinite(placement.y) || Number.isFinite(placement.width) || Number.isFinite(placement.height) ) { invokeParams.placement = placement; } await invoke("canvas.present", invokeParams); return jsonResult({ ok: true }); } case "hide": await invoke("canvas.hide", undefined); return jsonResult({ ok: true }); case "navigate": { const url = readStringParam(params, "url", { required: true }); await invoke("canvas.navigate", { url }); return jsonResult({ ok: true }); } case "eval": { const javaScript = readStringParam(params, "javaScript", { required: true, }); const raw = (await invoke("canvas.eval", { javaScript })) as { payload?: { result?: string }; }; const result = raw?.payload?.result; if (result) { return { content: [{ type: "text", text: result }], details: { result }, }; } return jsonResult({ ok: true }); } case "snapshot": { const formatRaw = typeof params.format === "string" ? params.format.toLowerCase() : "png"; const format = formatRaw === "jpg" || formatRaw === "jpeg" ? "jpeg" : "png"; const maxWidth = typeof params.maxWidth === "number" && Number.isFinite(params.maxWidth) ? params.maxWidth : undefined; const quality = typeof params.quality === "number" && Number.isFinite(params.quality) ? params.quality : undefined; const raw = (await invoke("canvas.snapshot", { format, maxWidth, quality, })) as { payload?: unknown }; const payload = parseCanvasSnapshotPayload(raw?.payload); const filePath = canvasSnapshotTempPath({ ext: payload.format === "jpeg" ? "jpg" : payload.format, }); await writeBase64ToFile(filePath, payload.base64); const mimeType = imageMimeFromFormat(payload.format) ?? "image/png"; return await imageResult({ label: "canvas:snapshot", path: filePath, base64: payload.base64, mimeType, details: { format: payload.format }, }); } case "a2ui_push": { const jsonl = typeof params.jsonl === "string" && params.jsonl.trim() ? params.jsonl : typeof params.jsonlPath === "string" && params.jsonlPath.trim() ? await fs.readFile(params.jsonlPath.trim(), "utf8") : ""; if (!jsonl.trim()) throw new Error("jsonl or jsonlPath required"); await invoke("canvas.a2ui.pushJSONL", { jsonl }); return jsonResult({ ok: true }); } case "a2ui_reset": await invoke("canvas.a2ui.reset", undefined); return jsonResult({ ok: true }); default: throw new Error(`Unknown action: ${action}`); } }, }; }