refactor(browser): simplify control API
This commit is contained in:
Peter Steinberger
@@ -1,273 +0,0 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import type { BrowserRouteContext } from "../server-context.js";
|
||||
|
||||
const pw = vi.hoisted(() => ({
|
||||
armDialogViaPlaywright: vi.fn().mockResolvedValue(undefined),
|
||||
armFileUploadViaPlaywright: vi.fn().mockResolvedValue(undefined),
|
||||
clickViaPlaywright: vi.fn().mockResolvedValue(undefined),
|
||||
closePageViaPlaywright: vi.fn().mockResolvedValue(undefined),
|
||||
dragViaPlaywright: vi.fn().mockResolvedValue(undefined),
|
||||
evaluateViaPlaywright: vi.fn().mockResolvedValue("result"),
|
||||
fillFormViaPlaywright: vi.fn().mockResolvedValue(undefined),
|
||||
hoverViaPlaywright: vi.fn().mockResolvedValue(undefined),
|
||||
navigateViaPlaywright: vi
|
||||
.fn()
|
||||
.mockResolvedValue({ url: "https://example.com" }),
|
||||
pressKeyViaPlaywright: vi.fn().mockResolvedValue(undefined),
|
||||
resizeViewportViaPlaywright: vi.fn().mockResolvedValue(undefined),
|
||||
selectOptionViaPlaywright: vi.fn().mockResolvedValue(undefined),
|
||||
typeViaPlaywright: vi.fn().mockResolvedValue(undefined),
|
||||
waitForViaPlaywright: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
vi.mock("../pw-ai.js", () => pw);
|
||||
|
||||
import { handleBrowserActionCore } from "./actions-core.js";
|
||||
|
||||
const baseTab = {
|
||||
targetId: "tab1",
|
||||
title: "One",
|
||||
url: "https://example.com",
|
||||
};
|
||||
|
||||
function createRes() {
|
||||
return {
|
||||
statusCode: 200,
|
||||
body: undefined as unknown,
|
||||
status(code: number) {
|
||||
this.statusCode = code;
|
||||
return this;
|
||||
},
|
||||
json(payload: unknown) {
|
||||
this.body = payload;
|
||||
return this;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createCtx(
|
||||
overrides: Partial<BrowserRouteContext> = {},
|
||||
): BrowserRouteContext {
|
||||
return {
|
||||
state: () => {
|
||||
throw new Error("unused");
|
||||
},
|
||||
ensureBrowserAvailable: vi.fn().mockResolvedValue(undefined),
|
||||
ensureTabAvailable: vi.fn().mockResolvedValue(baseTab),
|
||||
isReachable: vi.fn().mockResolvedValue(true),
|
||||
listTabs: vi
|
||||
.fn()
|
||||
.mockResolvedValue([
|
||||
baseTab,
|
||||
{ targetId: "tab2", title: "Two", url: "https://example.com/2" },
|
||||
]),
|
||||
openTab: vi.fn().mockResolvedValue({
|
||||
targetId: "newtab",
|
||||
title: "",
|
||||
url: "about:blank",
|
||||
type: "page",
|
||||
}),
|
||||
focusTab: vi.fn().mockResolvedValue(undefined),
|
||||
closeTab: vi.fn().mockResolvedValue(undefined),
|
||||
stopRunningBrowser: vi.fn().mockResolvedValue({ stopped: true }),
|
||||
mapTabError: vi.fn().mockReturnValue(null),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
async function callAction(
|
||||
action: Parameters<typeof handleBrowserActionCore>[0]["action"],
|
||||
args: Record<string, unknown> = {},
|
||||
ctxOverride?: Partial<BrowserRouteContext>,
|
||||
) {
|
||||
const res = createRes();
|
||||
const ctx = createCtx(ctxOverride);
|
||||
const handled = await handleBrowserActionCore({
|
||||
action,
|
||||
args,
|
||||
targetId: "",
|
||||
cdpPort: 18792,
|
||||
ctx,
|
||||
res,
|
||||
});
|
||||
return { res, ctx, handled };
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe("handleBrowserActionCore", () => {
|
||||
it("dispatches core browser actions", async () => {
|
||||
const cases = [
|
||||
{
|
||||
action: "close" as const,
|
||||
args: {},
|
||||
fn: pw.closePageViaPlaywright,
|
||||
expectArgs: { cdpPort: 18792, targetId: "tab1" },
|
||||
expectBody: { ok: true, targetId: "tab1", url: baseTab.url },
|
||||
},
|
||||
{
|
||||
action: "resize" as const,
|
||||
args: { width: 800, height: 600 },
|
||||
fn: pw.resizeViewportViaPlaywright,
|
||||
expectArgs: {
|
||||
cdpPort: 18792,
|
||||
targetId: "tab1",
|
||||
width: 800,
|
||||
height: 600,
|
||||
},
|
||||
expectBody: { ok: true, targetId: "tab1", url: baseTab.url },
|
||||
},
|
||||
{
|
||||
action: "dialog" as const,
|
||||
args: { accept: true, promptText: "ok" },
|
||||
fn: pw.armDialogViaPlaywright,
|
||||
expectArgs: {
|
||||
cdpPort: 18792,
|
||||
targetId: "tab1",
|
||||
accept: true,
|
||||
promptText: "ok",
|
||||
},
|
||||
expectBody: { ok: true },
|
||||
},
|
||||
{
|
||||
action: "evaluate" as const,
|
||||
args: { function: "() => 1", ref: "1" },
|
||||
fn: pw.evaluateViaPlaywright,
|
||||
expectArgs: {
|
||||
cdpPort: 18792,
|
||||
targetId: "tab1",
|
||||
fn: "() => 1",
|
||||
ref: "1",
|
||||
},
|
||||
expectBody: { ok: true, result: "result" },
|
||||
},
|
||||
{
|
||||
action: "upload" as const,
|
||||
args: { paths: ["/tmp/file.txt"] },
|
||||
fn: pw.armFileUploadViaPlaywright,
|
||||
expectArgs: {
|
||||
cdpPort: 18792,
|
||||
targetId: "tab1",
|
||||
paths: ["/tmp/file.txt"],
|
||||
},
|
||||
expectBody: { ok: true, targetId: "tab1" },
|
||||
},
|
||||
{
|
||||
action: "fill" as const,
|
||||
args: { fields: [{ ref: "1", value: "x" }] },
|
||||
fn: pw.fillFormViaPlaywright,
|
||||
expectArgs: {
|
||||
cdpPort: 18792,
|
||||
targetId: "tab1",
|
||||
fields: [{ ref: "1", value: "x" }],
|
||||
},
|
||||
expectBody: { ok: true, targetId: "tab1" },
|
||||
},
|
||||
{
|
||||
action: "press" as const,
|
||||
args: { key: "Enter" },
|
||||
fn: pw.pressKeyViaPlaywright,
|
||||
expectArgs: { cdpPort: 18792, targetId: "tab1", key: "Enter" },
|
||||
expectBody: { ok: true, targetId: "tab1" },
|
||||
},
|
||||
{
|
||||
action: "type" as const,
|
||||
args: { ref: "2", text: "hi", submit: true, slowly: true },
|
||||
fn: pw.typeViaPlaywright,
|
||||
expectArgs: {
|
||||
cdpPort: 18792,
|
||||
targetId: "tab1",
|
||||
ref: "2",
|
||||
text: "hi",
|
||||
submit: true,
|
||||
slowly: true,
|
||||
},
|
||||
expectBody: { ok: true, targetId: "tab1" },
|
||||
},
|
||||
{
|
||||
action: "navigate" as const,
|
||||
args: { url: "https://example.com" },
|
||||
fn: pw.navigateViaPlaywright,
|
||||
expectArgs: {
|
||||
cdpPort: 18792,
|
||||
targetId: "tab1",
|
||||
url: "https://example.com",
|
||||
},
|
||||
expectBody: { ok: true, targetId: "tab1", url: baseTab.url },
|
||||
},
|
||||
{
|
||||
action: "click" as const,
|
||||
args: {
|
||||
ref: "1",
|
||||
doubleClick: true,
|
||||
button: "right",
|
||||
modifiers: ["Shift"],
|
||||
},
|
||||
fn: pw.clickViaPlaywright,
|
||||
expectArgs: {
|
||||
cdpPort: 18792,
|
||||
targetId: "tab1",
|
||||
ref: "1",
|
||||
doubleClick: true,
|
||||
button: "right",
|
||||
modifiers: ["Shift"],
|
||||
},
|
||||
expectBody: { ok: true, targetId: "tab1", url: baseTab.url },
|
||||
},
|
||||
{
|
||||
action: "drag" as const,
|
||||
args: { startRef: "1", endRef: "2" },
|
||||
fn: pw.dragViaPlaywright,
|
||||
expectArgs: {
|
||||
cdpPort: 18792,
|
||||
targetId: "tab1",
|
||||
startRef: "1",
|
||||
endRef: "2",
|
||||
},
|
||||
expectBody: { ok: true, targetId: "tab1" },
|
||||
},
|
||||
{
|
||||
action: "hover" as const,
|
||||
args: { ref: "3" },
|
||||
fn: pw.hoverViaPlaywright,
|
||||
expectArgs: { cdpPort: 18792, targetId: "tab1", ref: "3" },
|
||||
expectBody: { ok: true, targetId: "tab1" },
|
||||
},
|
||||
{
|
||||
action: "select" as const,
|
||||
args: { ref: "4", values: ["A"] },
|
||||
fn: pw.selectOptionViaPlaywright,
|
||||
expectArgs: {
|
||||
cdpPort: 18792,
|
||||
targetId: "tab1",
|
||||
ref: "4",
|
||||
values: ["A"],
|
||||
},
|
||||
expectBody: { ok: true, targetId: "tab1" },
|
||||
},
|
||||
{
|
||||
action: "wait" as const,
|
||||
args: { time: 500, text: "ok", textGone: "bye" },
|
||||
fn: pw.waitForViaPlaywright,
|
||||
expectArgs: {
|
||||
cdpPort: 18792,
|
||||
targetId: "tab1",
|
||||
time: 500,
|
||||
text: "ok",
|
||||
textGone: "bye",
|
||||
},
|
||||
expectBody: { ok: true, targetId: "tab1" },
|
||||
},
|
||||
];
|
||||
|
||||
for (const item of cases) {
|
||||
const { res, handled } = await callAction(item.action, item.args);
|
||||
expect(handled).toBe(true);
|
||||
expect(item.fn).toHaveBeenCalledWith(item.expectArgs);
|
||||
expect(res.body).toEqual(item.expectBody);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -1,307 +0,0 @@
|
||||
import type express from "express";
|
||||
|
||||
import {
|
||||
armDialogViaPlaywright,
|
||||
armFileUploadViaPlaywright,
|
||||
clickViaPlaywright,
|
||||
closePageViaPlaywright,
|
||||
dragViaPlaywright,
|
||||
evaluateViaPlaywright,
|
||||
fillFormViaPlaywright,
|
||||
hoverViaPlaywright,
|
||||
navigateViaPlaywright,
|
||||
pressKeyViaPlaywright,
|
||||
resizeViewportViaPlaywright,
|
||||
selectOptionViaPlaywright,
|
||||
typeViaPlaywright,
|
||||
waitForViaPlaywright,
|
||||
} from "../pw-ai.js";
|
||||
import type { BrowserRouteContext } from "../server-context.js";
|
||||
import {
|
||||
jsonError,
|
||||
toBoolean,
|
||||
toNumber,
|
||||
toStringArray,
|
||||
toStringOrEmpty,
|
||||
} from "./utils.js";
|
||||
|
||||
type MouseButton = "left" | "right" | "middle";
|
||||
type KeyboardModifier = "Alt" | "Control" | "ControlOrMeta" | "Meta" | "Shift";
|
||||
|
||||
function normalizeMouseButton(value: unknown): MouseButton | undefined {
|
||||
const raw = toStringOrEmpty(value);
|
||||
if (raw === "left" || raw === "right" || raw === "middle") return raw;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function normalizeModifiers(value: unknown): KeyboardModifier[] | undefined {
|
||||
const raw = toStringArray(value);
|
||||
if (!raw?.length) return undefined;
|
||||
const normalized = raw.filter(
|
||||
(m): m is KeyboardModifier =>
|
||||
m === "Alt" ||
|
||||
m === "Control" ||
|
||||
m === "ControlOrMeta" ||
|
||||
m === "Meta" ||
|
||||
m === "Shift",
|
||||
);
|
||||
return normalized.length ? normalized : undefined;
|
||||
}
|
||||
|
||||
export type BrowserActionCore =
|
||||
| "click"
|
||||
| "close"
|
||||
| "dialog"
|
||||
| "drag"
|
||||
| "evaluate"
|
||||
| "fill"
|
||||
| "hover"
|
||||
| "navigate"
|
||||
| "press"
|
||||
| "resize"
|
||||
| "select"
|
||||
| "type"
|
||||
| "upload"
|
||||
| "wait";
|
||||
|
||||
type ActionCoreParams = {
|
||||
action: BrowserActionCore;
|
||||
args: Record<string, unknown>;
|
||||
targetId: string;
|
||||
cdpPort: number;
|
||||
ctx: BrowserRouteContext;
|
||||
res: express.Response;
|
||||
};
|
||||
|
||||
export async function handleBrowserActionCore(
|
||||
params: ActionCoreParams,
|
||||
): Promise<boolean> {
|
||||
const { action, args, targetId, cdpPort, ctx, res } = params;
|
||||
const target = targetId || undefined;
|
||||
|
||||
switch (action) {
|
||||
case "close": {
|
||||
const tab = await ctx.ensureTabAvailable(target);
|
||||
await closePageViaPlaywright({ cdpPort, targetId: tab.targetId });
|
||||
res.json({ ok: true, targetId: tab.targetId, url: tab.url });
|
||||
return true;
|
||||
}
|
||||
case "resize": {
|
||||
const width = toNumber(args.width);
|
||||
const height = toNumber(args.height);
|
||||
if (!width || !height) {
|
||||
jsonError(res, 400, "width and height are required");
|
||||
return true;
|
||||
}
|
||||
const tab = await ctx.ensureTabAvailable(target);
|
||||
await resizeViewportViaPlaywright({
|
||||
cdpPort,
|
||||
targetId: tab.targetId,
|
||||
width,
|
||||
height,
|
||||
});
|
||||
res.json({ ok: true, targetId: tab.targetId, url: tab.url });
|
||||
return true;
|
||||
}
|
||||
case "dialog": {
|
||||
const accept = toBoolean(args.accept);
|
||||
if (accept === undefined) {
|
||||
jsonError(res, 400, "accept is required");
|
||||
return true;
|
||||
}
|
||||
const promptText = toStringOrEmpty(args.promptText) || undefined;
|
||||
const tab = await ctx.ensureTabAvailable(target);
|
||||
await armDialogViaPlaywright({
|
||||
cdpPort,
|
||||
targetId: tab.targetId,
|
||||
accept,
|
||||
promptText,
|
||||
});
|
||||
res.json({ ok: true });
|
||||
return true;
|
||||
}
|
||||
case "evaluate": {
|
||||
const fn = toStringOrEmpty(args.function);
|
||||
if (!fn) {
|
||||
jsonError(res, 400, "function is required");
|
||||
return true;
|
||||
}
|
||||
const ref = toStringOrEmpty(args.ref) || undefined;
|
||||
const tab = await ctx.ensureTabAvailable(target);
|
||||
const result = await evaluateViaPlaywright({
|
||||
cdpPort,
|
||||
targetId: tab.targetId,
|
||||
fn,
|
||||
ref,
|
||||
});
|
||||
res.json({ ok: true, result });
|
||||
return true;
|
||||
}
|
||||
case "upload": {
|
||||
const paths = toStringArray(args.paths) ?? [];
|
||||
const tab = await ctx.ensureTabAvailable(target);
|
||||
await armFileUploadViaPlaywright({
|
||||
cdpPort,
|
||||
targetId: tab.targetId,
|
||||
paths: paths.length ? paths : undefined,
|
||||
});
|
||||
res.json({ ok: true, targetId: tab.targetId });
|
||||
return true;
|
||||
}
|
||||
case "fill": {
|
||||
const fields = Array.isArray(args.fields)
|
||||
? (args.fields as Array<Record<string, unknown>>)
|
||||
: null;
|
||||
if (!fields?.length) {
|
||||
jsonError(res, 400, "fields are required");
|
||||
return true;
|
||||
}
|
||||
const tab = await ctx.ensureTabAvailable(target);
|
||||
await fillFormViaPlaywright({
|
||||
cdpPort,
|
||||
targetId: tab.targetId,
|
||||
fields,
|
||||
});
|
||||
res.json({ ok: true, targetId: tab.targetId });
|
||||
return true;
|
||||
}
|
||||
case "press": {
|
||||
const key = toStringOrEmpty(args.key);
|
||||
if (!key) {
|
||||
jsonError(res, 400, "key is required");
|
||||
return true;
|
||||
}
|
||||
const tab = await ctx.ensureTabAvailable(target);
|
||||
await pressKeyViaPlaywright({
|
||||
cdpPort,
|
||||
targetId: tab.targetId,
|
||||
key,
|
||||
});
|
||||
res.json({ ok: true, targetId: tab.targetId });
|
||||
return true;
|
||||
}
|
||||
case "type": {
|
||||
const ref = toStringOrEmpty(args.ref);
|
||||
const text = toStringOrEmpty(args.text);
|
||||
if (!ref || !text) {
|
||||
jsonError(res, 400, "ref and text are required");
|
||||
return true;
|
||||
}
|
||||
const submit = toBoolean(args.submit) ?? false;
|
||||
const slowly = toBoolean(args.slowly) ?? false;
|
||||
const tab = await ctx.ensureTabAvailable(target);
|
||||
await typeViaPlaywright({
|
||||
cdpPort,
|
||||
targetId: tab.targetId,
|
||||
ref,
|
||||
text,
|
||||
submit,
|
||||
slowly,
|
||||
});
|
||||
res.json({ ok: true, targetId: tab.targetId });
|
||||
return true;
|
||||
}
|
||||
case "navigate": {
|
||||
const url = toStringOrEmpty(args.url);
|
||||
if (!url) {
|
||||
jsonError(res, 400, "url is required");
|
||||
return true;
|
||||
}
|
||||
const tab = await ctx.ensureTabAvailable(target);
|
||||
const result = await navigateViaPlaywright({
|
||||
cdpPort,
|
||||
targetId: tab.targetId,
|
||||
url,
|
||||
});
|
||||
res.json({ ok: true, targetId: tab.targetId, ...result });
|
||||
return true;
|
||||
}
|
||||
case "click": {
|
||||
const ref = toStringOrEmpty(args.ref);
|
||||
if (!ref) {
|
||||
jsonError(res, 400, "ref is required");
|
||||
return true;
|
||||
}
|
||||
const doubleClick = toBoolean(args.doubleClick) ?? false;
|
||||
const button = normalizeMouseButton(args.button);
|
||||
const modifiers = normalizeModifiers(args.modifiers);
|
||||
const tab = await ctx.ensureTabAvailable(target);
|
||||
await clickViaPlaywright({
|
||||
cdpPort,
|
||||
targetId: tab.targetId,
|
||||
ref,
|
||||
doubleClick,
|
||||
button,
|
||||
modifiers,
|
||||
});
|
||||
res.json({ ok: true, targetId: tab.targetId, url: tab.url });
|
||||
return true;
|
||||
}
|
||||
case "drag": {
|
||||
const startRef = toStringOrEmpty(args.startRef);
|
||||
const endRef = toStringOrEmpty(args.endRef);
|
||||
if (!startRef || !endRef) {
|
||||
jsonError(res, 400, "startRef and endRef are required");
|
||||
return true;
|
||||
}
|
||||
const tab = await ctx.ensureTabAvailable(target);
|
||||
await dragViaPlaywright({
|
||||
cdpPort,
|
||||
targetId: tab.targetId,
|
||||
startRef,
|
||||
endRef,
|
||||
});
|
||||
res.json({ ok: true, targetId: tab.targetId });
|
||||
return true;
|
||||
}
|
||||
case "hover": {
|
||||
const ref = toStringOrEmpty(args.ref);
|
||||
if (!ref) {
|
||||
jsonError(res, 400, "ref is required");
|
||||
return true;
|
||||
}
|
||||
const tab = await ctx.ensureTabAvailable(target);
|
||||
await hoverViaPlaywright({
|
||||
cdpPort,
|
||||
targetId: tab.targetId,
|
||||
ref,
|
||||
});
|
||||
res.json({ ok: true, targetId: tab.targetId });
|
||||
return true;
|
||||
}
|
||||
case "select": {
|
||||
const ref = toStringOrEmpty(args.ref);
|
||||
const values = toStringArray(args.values);
|
||||
if (!ref || !values?.length) {
|
||||
jsonError(res, 400, "ref and values are required");
|
||||
return true;
|
||||
}
|
||||
const tab = await ctx.ensureTabAvailable(target);
|
||||
await selectOptionViaPlaywright({
|
||||
cdpPort,
|
||||
targetId: tab.targetId,
|
||||
ref,
|
||||
values,
|
||||
});
|
||||
res.json({ ok: true, targetId: tab.targetId });
|
||||
return true;
|
||||
}
|
||||
case "wait": {
|
||||
const time = toNumber(args.time);
|
||||
const text = toStringOrEmpty(args.text) || undefined;
|
||||
const textGone = toStringOrEmpty(args.textGone) || undefined;
|
||||
const tab = await ctx.ensureTabAvailable(target);
|
||||
await waitForViaPlaywright({
|
||||
cdpPort,
|
||||
targetId: tab.targetId,
|
||||
time,
|
||||
text,
|
||||
textGone,
|
||||
});
|
||||
res.json({ ok: true, targetId: tab.targetId });
|
||||
return true;
|
||||
}
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -1,174 +0,0 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import type { BrowserRouteContext } from "../server-context.js";
|
||||
|
||||
const pw = vi.hoisted(() => ({
|
||||
getConsoleMessagesViaPlaywright: vi.fn().mockResolvedValue([]),
|
||||
pdfViaPlaywright: vi.fn().mockResolvedValue({ buffer: Buffer.from("pdf") }),
|
||||
verifyElementVisibleViaPlaywright: vi.fn().mockResolvedValue(undefined),
|
||||
verifyListVisibleViaPlaywright: vi.fn().mockResolvedValue(undefined),
|
||||
verifyTextVisibleViaPlaywright: vi.fn().mockResolvedValue(undefined),
|
||||
verifyValueViaPlaywright: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
const media = vi.hoisted(() => ({
|
||||
ensureMediaDir: vi.fn().mockResolvedValue(undefined),
|
||||
saveMediaBuffer: vi.fn().mockResolvedValue({ path: "/tmp/fake.pdf" }),
|
||||
}));
|
||||
|
||||
vi.mock("../pw-ai.js", () => pw);
|
||||
vi.mock("../../media/store.js", () => media);
|
||||
|
||||
import { handleBrowserActionExtra } from "./actions-extra.js";
|
||||
|
||||
const baseTab = {
|
||||
targetId: "tab1",
|
||||
title: "One",
|
||||
url: "https://example.com",
|
||||
};
|
||||
|
||||
function createRes() {
|
||||
return {
|
||||
statusCode: 200,
|
||||
body: undefined as unknown,
|
||||
status(code: number) {
|
||||
this.statusCode = code;
|
||||
return this;
|
||||
},
|
||||
json(payload: unknown) {
|
||||
this.body = payload;
|
||||
return this;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createCtx(
|
||||
overrides: Partial<BrowserRouteContext> = {},
|
||||
): BrowserRouteContext {
|
||||
return {
|
||||
state: () => {
|
||||
throw new Error("unused");
|
||||
},
|
||||
ensureBrowserAvailable: vi.fn().mockResolvedValue(undefined),
|
||||
ensureTabAvailable: vi.fn().mockResolvedValue(baseTab),
|
||||
isReachable: vi.fn().mockResolvedValue(true),
|
||||
listTabs: vi.fn().mockResolvedValue([baseTab]),
|
||||
openTab: vi.fn().mockResolvedValue({
|
||||
targetId: "newtab",
|
||||
title: "",
|
||||
url: "about:blank",
|
||||
type: "page",
|
||||
}),
|
||||
focusTab: vi.fn().mockResolvedValue(undefined),
|
||||
closeTab: vi.fn().mockResolvedValue(undefined),
|
||||
stopRunningBrowser: vi.fn().mockResolvedValue({ stopped: true }),
|
||||
mapTabError: vi.fn().mockReturnValue(null),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
async function callAction(
|
||||
action: Parameters<typeof handleBrowserActionExtra>[0]["action"],
|
||||
args: Record<string, unknown> = {},
|
||||
) {
|
||||
const res = createRes();
|
||||
const ctx = createCtx();
|
||||
const handled = await handleBrowserActionExtra({
|
||||
action,
|
||||
args,
|
||||
targetId: "",
|
||||
cdpPort: 18792,
|
||||
ctx,
|
||||
res,
|
||||
});
|
||||
return { res, ctx, handled };
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe("handleBrowserActionExtra", () => {
|
||||
it("dispatches extra browser actions", async () => {
|
||||
const cases = [
|
||||
{
|
||||
action: "console" as const,
|
||||
args: { level: "error" },
|
||||
fn: pw.getConsoleMessagesViaPlaywright,
|
||||
expectArgs: {
|
||||
cdpPort: 18792,
|
||||
targetId: "tab1",
|
||||
level: "error",
|
||||
},
|
||||
expectBody: { ok: true, messages: [], targetId: "tab1" },
|
||||
},
|
||||
{
|
||||
action: "verifyElement" as const,
|
||||
args: { role: "button", accessibleName: "Submit" },
|
||||
fn: pw.verifyElementVisibleViaPlaywright,
|
||||
expectArgs: {
|
||||
cdpPort: 18792,
|
||||
targetId: "tab1",
|
||||
role: "button",
|
||||
accessibleName: "Submit",
|
||||
},
|
||||
expectBody: { ok: true },
|
||||
},
|
||||
{
|
||||
action: "verifyText" as const,
|
||||
args: { text: "Hello" },
|
||||
fn: pw.verifyTextVisibleViaPlaywright,
|
||||
expectArgs: { cdpPort: 18792, targetId: "tab1", text: "Hello" },
|
||||
expectBody: { ok: true },
|
||||
},
|
||||
{
|
||||
action: "verifyList" as const,
|
||||
args: { ref: "1", items: ["a", "b"] },
|
||||
fn: pw.verifyListVisibleViaPlaywright,
|
||||
expectArgs: {
|
||||
cdpPort: 18792,
|
||||
targetId: "tab1",
|
||||
ref: "1",
|
||||
items: ["a", "b"],
|
||||
},
|
||||
expectBody: { ok: true },
|
||||
},
|
||||
{
|
||||
action: "verifyValue" as const,
|
||||
args: { ref: "2", type: "textbox", value: "x" },
|
||||
fn: pw.verifyValueViaPlaywright,
|
||||
expectArgs: {
|
||||
cdpPort: 18792,
|
||||
targetId: "tab1",
|
||||
ref: "2",
|
||||
type: "textbox",
|
||||
value: "x",
|
||||
},
|
||||
expectBody: { ok: true },
|
||||
},
|
||||
];
|
||||
|
||||
for (const item of cases) {
|
||||
const { res, handled } = await callAction(item.action, item.args);
|
||||
expect(handled).toBe(true);
|
||||
expect(item.fn).toHaveBeenCalledWith(item.expectArgs);
|
||||
expect(res.body).toEqual(item.expectBody);
|
||||
}
|
||||
});
|
||||
|
||||
it("stores PDF output", async () => {
|
||||
const { res: pdfRes } = await callAction("pdf");
|
||||
expect(pw.pdfViaPlaywright).toHaveBeenCalledWith({
|
||||
cdpPort: 18792,
|
||||
targetId: "tab1",
|
||||
});
|
||||
expect(media.ensureMediaDir).toHaveBeenCalled();
|
||||
expect(media.saveMediaBuffer).toHaveBeenCalled();
|
||||
expect(pdfRes.body).toMatchObject({
|
||||
ok: true,
|
||||
path: "/tmp/fake.pdf",
|
||||
targetId: "tab1",
|
||||
url: baseTab.url,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,144 +0,0 @@
|
||||
import path from "node:path";
|
||||
|
||||
import type express from "express";
|
||||
|
||||
import { ensureMediaDir, saveMediaBuffer } from "../../media/store.js";
|
||||
import {
|
||||
getConsoleMessagesViaPlaywright,
|
||||
pdfViaPlaywright,
|
||||
verifyElementVisibleViaPlaywright,
|
||||
verifyListVisibleViaPlaywright,
|
||||
verifyTextVisibleViaPlaywright,
|
||||
verifyValueViaPlaywright,
|
||||
} from "../pw-ai.js";
|
||||
import type { BrowserRouteContext } from "../server-context.js";
|
||||
import { jsonError, toStringArray, toStringOrEmpty } from "./utils.js";
|
||||
|
||||
export type BrowserActionExtra =
|
||||
| "console"
|
||||
| "pdf"
|
||||
| "verifyElement"
|
||||
| "verifyList"
|
||||
| "verifyText"
|
||||
| "verifyValue";
|
||||
|
||||
type ActionExtraParams = {
|
||||
action: BrowserActionExtra;
|
||||
args: Record<string, unknown>;
|
||||
targetId: string;
|
||||
cdpPort: number;
|
||||
ctx: BrowserRouteContext;
|
||||
res: express.Response;
|
||||
};
|
||||
|
||||
export async function handleBrowserActionExtra(
|
||||
params: ActionExtraParams,
|
||||
): Promise<boolean> {
|
||||
const { action, args, targetId, cdpPort, ctx, res } = params;
|
||||
const target = targetId || undefined;
|
||||
|
||||
switch (action) {
|
||||
case "console": {
|
||||
const level = toStringOrEmpty(args.level) || undefined;
|
||||
const tab = await ctx.ensureTabAvailable(target);
|
||||
const messages = await getConsoleMessagesViaPlaywright({
|
||||
cdpPort,
|
||||
targetId: tab.targetId,
|
||||
level,
|
||||
});
|
||||
res.json({ ok: true, messages, targetId: tab.targetId });
|
||||
return true;
|
||||
}
|
||||
case "pdf": {
|
||||
const tab = await ctx.ensureTabAvailable(target);
|
||||
const pdf = await pdfViaPlaywright({
|
||||
cdpPort,
|
||||
targetId: tab.targetId,
|
||||
});
|
||||
await ensureMediaDir();
|
||||
const saved = await saveMediaBuffer(
|
||||
pdf.buffer,
|
||||
"application/pdf",
|
||||
"browser",
|
||||
pdf.buffer.byteLength,
|
||||
);
|
||||
res.json({
|
||||
ok: true,
|
||||
path: path.resolve(saved.path),
|
||||
targetId: tab.targetId,
|
||||
url: tab.url,
|
||||
});
|
||||
return true;
|
||||
}
|
||||
case "verifyElement": {
|
||||
const role = toStringOrEmpty(args.role);
|
||||
const accessibleName = toStringOrEmpty(args.accessibleName);
|
||||
if (!role || !accessibleName) {
|
||||
jsonError(res, 400, "role and accessibleName are required");
|
||||
return true;
|
||||
}
|
||||
const tab = await ctx.ensureTabAvailable(target);
|
||||
await verifyElementVisibleViaPlaywright({
|
||||
cdpPort,
|
||||
targetId: tab.targetId,
|
||||
role,
|
||||
accessibleName,
|
||||
});
|
||||
res.json({ ok: true });
|
||||
return true;
|
||||
}
|
||||
case "verifyText": {
|
||||
const text = toStringOrEmpty(args.text);
|
||||
if (!text) {
|
||||
jsonError(res, 400, "text is required");
|
||||
return true;
|
||||
}
|
||||
const tab = await ctx.ensureTabAvailable(target);
|
||||
await verifyTextVisibleViaPlaywright({
|
||||
cdpPort,
|
||||
targetId: tab.targetId,
|
||||
text,
|
||||
});
|
||||
res.json({ ok: true });
|
||||
return true;
|
||||
}
|
||||
case "verifyList": {
|
||||
const ref = toStringOrEmpty(args.ref);
|
||||
const items = toStringArray(args.items);
|
||||
if (!ref || !items?.length) {
|
||||
jsonError(res, 400, "ref and items are required");
|
||||
return true;
|
||||
}
|
||||
const tab = await ctx.ensureTabAvailable(target);
|
||||
await verifyListVisibleViaPlaywright({
|
||||
cdpPort,
|
||||
targetId: tab.targetId,
|
||||
ref,
|
||||
items,
|
||||
});
|
||||
res.json({ ok: true });
|
||||
return true;
|
||||
}
|
||||
case "verifyValue": {
|
||||
const ref = toStringOrEmpty(args.ref);
|
||||
const type = toStringOrEmpty(args.type);
|
||||
const value = toStringOrEmpty(args.value);
|
||||
if (!ref || !type) {
|
||||
jsonError(res, 400, "ref and type are required");
|
||||
return true;
|
||||
}
|
||||
const tab = await ctx.ensureTabAvailable(target);
|
||||
await verifyValueViaPlaywright({
|
||||
cdpPort,
|
||||
targetId: tab.targetId,
|
||||
ref,
|
||||
type,
|
||||
value,
|
||||
});
|
||||
res.json({ ok: true });
|
||||
return true;
|
||||
}
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -1,198 +0,0 @@
|
||||
import type express from "express";
|
||||
|
||||
import type { BrowserRouteContext } from "../server-context.js";
|
||||
import { handleBrowserActionCore } from "./actions-core.js";
|
||||
import { handleBrowserActionExtra } from "./actions-extra.js";
|
||||
import { jsonError, toStringOrEmpty } from "./utils.js";
|
||||
|
||||
function readBody(req: express.Request): Record<string, unknown> {
|
||||
const body = req.body as Record<string, unknown> | undefined;
|
||||
if (!body || typeof body !== "object" || Array.isArray(body)) return {};
|
||||
return body;
|
||||
}
|
||||
|
||||
function readTargetId(value: unknown): string {
|
||||
return toStringOrEmpty(value);
|
||||
}
|
||||
|
||||
function handleActionError(
|
||||
ctx: BrowserRouteContext,
|
||||
res: express.Response,
|
||||
err: unknown,
|
||||
) {
|
||||
const mapped = ctx.mapTabError(err);
|
||||
if (mapped) return jsonError(res, mapped.status, mapped.message);
|
||||
jsonError(res, 500, String(err));
|
||||
}
|
||||
|
||||
async function runCoreAction(
|
||||
ctx: BrowserRouteContext,
|
||||
res: express.Response,
|
||||
action: Parameters<typeof handleBrowserActionCore>[0]["action"],
|
||||
args: Record<string, unknown>,
|
||||
targetId: string,
|
||||
) {
|
||||
try {
|
||||
const cdpPort = ctx.state().cdpPort;
|
||||
await handleBrowserActionCore({
|
||||
action,
|
||||
args,
|
||||
targetId,
|
||||
cdpPort,
|
||||
ctx,
|
||||
res,
|
||||
});
|
||||
} catch (err) {
|
||||
handleActionError(ctx, res, err);
|
||||
}
|
||||
}
|
||||
|
||||
async function runExtraAction(
|
||||
ctx: BrowserRouteContext,
|
||||
res: express.Response,
|
||||
action: Parameters<typeof handleBrowserActionExtra>[0]["action"],
|
||||
args: Record<string, unknown>,
|
||||
targetId: string,
|
||||
) {
|
||||
try {
|
||||
const cdpPort = ctx.state().cdpPort;
|
||||
await handleBrowserActionExtra({
|
||||
action,
|
||||
args,
|
||||
targetId,
|
||||
cdpPort,
|
||||
ctx,
|
||||
res,
|
||||
});
|
||||
} catch (err) {
|
||||
handleActionError(ctx, res, err);
|
||||
}
|
||||
}
|
||||
|
||||
export function registerBrowserActionRoutes(
|
||||
app: express.Express,
|
||||
ctx: BrowserRouteContext,
|
||||
) {
|
||||
app.post("/navigate", async (req, res) => {
|
||||
const body = readBody(req);
|
||||
const targetId = readTargetId(body.targetId);
|
||||
await runCoreAction(ctx, res, "navigate", body, targetId);
|
||||
});
|
||||
|
||||
app.post("/resize", async (req, res) => {
|
||||
const body = readBody(req);
|
||||
const targetId = readTargetId(body.targetId);
|
||||
await runCoreAction(ctx, res, "resize", body, targetId);
|
||||
});
|
||||
|
||||
app.post("/close", async (req, res) => {
|
||||
const body = readBody(req);
|
||||
const targetId = readTargetId(body.targetId);
|
||||
await runCoreAction(ctx, res, "close", body, targetId);
|
||||
});
|
||||
|
||||
app.post("/click", async (req, res) => {
|
||||
const body = readBody(req);
|
||||
const targetId = readTargetId(body.targetId);
|
||||
await runCoreAction(ctx, res, "click", body, targetId);
|
||||
});
|
||||
|
||||
app.post("/type", async (req, res) => {
|
||||
const body = readBody(req);
|
||||
const targetId = readTargetId(body.targetId);
|
||||
await runCoreAction(ctx, res, "type", body, targetId);
|
||||
});
|
||||
|
||||
app.post("/press", async (req, res) => {
|
||||
const body = readBody(req);
|
||||
const targetId = readTargetId(body.targetId);
|
||||
await runCoreAction(ctx, res, "press", body, targetId);
|
||||
});
|
||||
|
||||
app.post("/hover", async (req, res) => {
|
||||
const body = readBody(req);
|
||||
const targetId = readTargetId(body.targetId);
|
||||
await runCoreAction(ctx, res, "hover", body, targetId);
|
||||
});
|
||||
|
||||
app.post("/drag", async (req, res) => {
|
||||
const body = readBody(req);
|
||||
const targetId = readTargetId(body.targetId);
|
||||
await runCoreAction(ctx, res, "drag", body, targetId);
|
||||
});
|
||||
|
||||
app.post("/select", async (req, res) => {
|
||||
const body = readBody(req);
|
||||
const targetId = readTargetId(body.targetId);
|
||||
await runCoreAction(ctx, res, "select", body, targetId);
|
||||
});
|
||||
|
||||
app.post("/upload", async (req, res) => {
|
||||
const body = readBody(req);
|
||||
const targetId = readTargetId(body.targetId);
|
||||
await runCoreAction(ctx, res, "upload", body, targetId);
|
||||
});
|
||||
|
||||
app.post("/fill", async (req, res) => {
|
||||
const body = readBody(req);
|
||||
const targetId = readTargetId(body.targetId);
|
||||
await runCoreAction(ctx, res, "fill", body, targetId);
|
||||
});
|
||||
|
||||
app.post("/dialog", async (req, res) => {
|
||||
const body = readBody(req);
|
||||
const targetId = readTargetId(body.targetId);
|
||||
await runCoreAction(ctx, res, "dialog", body, targetId);
|
||||
});
|
||||
|
||||
app.post("/wait", async (req, res) => {
|
||||
const body = readBody(req);
|
||||
const targetId = readTargetId(body.targetId);
|
||||
await runCoreAction(ctx, res, "wait", body, targetId);
|
||||
});
|
||||
|
||||
app.post("/evaluate", async (req, res) => {
|
||||
const body = readBody(req);
|
||||
const targetId = readTargetId(body.targetId);
|
||||
await runCoreAction(ctx, res, "evaluate", body, targetId);
|
||||
});
|
||||
|
||||
app.get("/console", async (req, res) => {
|
||||
const targetId = readTargetId(req.query.targetId);
|
||||
const level = toStringOrEmpty(req.query.level);
|
||||
const args = level ? { level } : {};
|
||||
await runExtraAction(ctx, res, "console", args, targetId);
|
||||
});
|
||||
|
||||
app.post("/pdf", async (req, res) => {
|
||||
const body = readBody(req);
|
||||
const targetId = readTargetId(body.targetId);
|
||||
await runExtraAction(ctx, res, "pdf", body, targetId);
|
||||
});
|
||||
|
||||
app.post("/verify/element", async (req, res) => {
|
||||
const body = readBody(req);
|
||||
const targetId = readTargetId(body.targetId);
|
||||
await runExtraAction(ctx, res, "verifyElement", body, targetId);
|
||||
});
|
||||
|
||||
app.post("/verify/text", async (req, res) => {
|
||||
const body = readBody(req);
|
||||
const targetId = readTargetId(body.targetId);
|
||||
await runExtraAction(ctx, res, "verifyText", body, targetId);
|
||||
});
|
||||
|
||||
app.post("/verify/list", async (req, res) => {
|
||||
const body = readBody(req);
|
||||
const targetId = readTargetId(body.targetId);
|
||||
await runExtraAction(ctx, res, "verifyList", body, targetId);
|
||||
});
|
||||
|
||||
app.post("/verify/value", async (req, res) => {
|
||||
const body = readBody(req);
|
||||
const targetId = readTargetId(body.targetId);
|
||||
await runExtraAction(ctx, res, "verifyValue", body, targetId);
|
||||
});
|
||||
|
||||
// Intentionally no coordinate-based mouse actions (move/click/drag).
|
||||
}
|
||||
456
src/browser/routes/agent.ts
Normal file
456
src/browser/routes/agent.ts
Normal file
@@ -0,0 +1,456 @@
|
||||
import path from "node:path";
|
||||
|
||||
import type express from "express";
|
||||
|
||||
import { ensureMediaDir, saveMediaBuffer } from "../../media/store.js";
|
||||
import { snapshotAria } from "../cdp.js";
|
||||
import {
|
||||
armDialogViaPlaywright,
|
||||
armFileUploadViaPlaywright,
|
||||
clickViaPlaywright,
|
||||
closePageViaPlaywright,
|
||||
dragViaPlaywright,
|
||||
evaluateViaPlaywright,
|
||||
fillFormViaPlaywright,
|
||||
getConsoleMessagesViaPlaywright,
|
||||
hoverViaPlaywright,
|
||||
navigateViaPlaywright,
|
||||
pdfViaPlaywright,
|
||||
pressKeyViaPlaywright,
|
||||
resizeViewportViaPlaywright,
|
||||
selectOptionViaPlaywright,
|
||||
snapshotAiViaPlaywright,
|
||||
takeScreenshotViaPlaywright,
|
||||
typeViaPlaywright,
|
||||
waitForViaPlaywright,
|
||||
} from "../pw-ai.js";
|
||||
import {
|
||||
DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
|
||||
DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE,
|
||||
normalizeBrowserScreenshot,
|
||||
} from "../screenshot.js";
|
||||
import type { BrowserRouteContext } from "../server-context.js";
|
||||
import {
|
||||
jsonError,
|
||||
toBoolean,
|
||||
toNumber,
|
||||
toStringArray,
|
||||
toStringOrEmpty,
|
||||
} from "./utils.js";
|
||||
|
||||
type ActKind =
|
||||
| "click"
|
||||
| "close"
|
||||
| "drag"
|
||||
| "evaluate"
|
||||
| "fill"
|
||||
| "hover"
|
||||
| "press"
|
||||
| "resize"
|
||||
| "select"
|
||||
| "type"
|
||||
| "wait";
|
||||
|
||||
type ClickButton = "left" | "right" | "middle";
|
||||
type ClickModifier = "Alt" | "Control" | "ControlOrMeta" | "Meta" | "Shift";
|
||||
|
||||
function readBody(req: express.Request): Record<string, unknown> {
|
||||
const body = req.body as Record<string, unknown> | undefined;
|
||||
if (!body || typeof body !== "object" || Array.isArray(body)) return {};
|
||||
return body;
|
||||
}
|
||||
|
||||
function handleRouteError(
|
||||
ctx: BrowserRouteContext,
|
||||
res: express.Response,
|
||||
err: unknown,
|
||||
) {
|
||||
const mapped = ctx.mapTabError(err);
|
||||
if (mapped) return jsonError(res, mapped.status, mapped.message);
|
||||
jsonError(res, 500, String(err));
|
||||
}
|
||||
|
||||
function parseClickButton(raw: string): ClickButton | undefined {
|
||||
if (raw === "left" || raw === "right" || raw === "middle") return raw;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function registerBrowserAgentRoutes(
|
||||
app: express.Express,
|
||||
ctx: BrowserRouteContext,
|
||||
) {
|
||||
app.post("/navigate", async (req, res) => {
|
||||
const body = readBody(req);
|
||||
const url = toStringOrEmpty(body.url);
|
||||
const targetId = toStringOrEmpty(body.targetId) || undefined;
|
||||
if (!url) return jsonError(res, 400, "url is required");
|
||||
try {
|
||||
const tab = await ctx.ensureTabAvailable(targetId);
|
||||
const result = await navigateViaPlaywright({
|
||||
cdpPort: ctx.state().cdpPort,
|
||||
targetId: tab.targetId,
|
||||
url,
|
||||
});
|
||||
res.json({ ok: true, targetId: tab.targetId, ...result });
|
||||
} catch (err) {
|
||||
handleRouteError(ctx, res, err);
|
||||
}
|
||||
});
|
||||
|
||||
app.post("/act", async (req, res) => {
|
||||
const body = readBody(req);
|
||||
const kind = toStringOrEmpty(body.kind) as ActKind;
|
||||
const targetId = toStringOrEmpty(body.targetId) || undefined;
|
||||
|
||||
if (
|
||||
kind !== "click" &&
|
||||
kind !== "close" &&
|
||||
kind !== "drag" &&
|
||||
kind !== "evaluate" &&
|
||||
kind !== "fill" &&
|
||||
kind !== "hover" &&
|
||||
kind !== "press" &&
|
||||
kind !== "resize" &&
|
||||
kind !== "select" &&
|
||||
kind !== "type" &&
|
||||
kind !== "wait"
|
||||
) {
|
||||
return jsonError(res, 400, "kind is required");
|
||||
}
|
||||
|
||||
try {
|
||||
const tab = await ctx.ensureTabAvailable(targetId);
|
||||
const cdpPort = ctx.state().cdpPort;
|
||||
|
||||
switch (kind) {
|
||||
case "click": {
|
||||
const ref = toStringOrEmpty(body.ref);
|
||||
if (!ref) return jsonError(res, 400, "ref is required");
|
||||
const doubleClick = toBoolean(body.doubleClick) ?? false;
|
||||
const buttonRaw = toStringOrEmpty(body.button) || "";
|
||||
const button = buttonRaw ? parseClickButton(buttonRaw) : undefined;
|
||||
if (buttonRaw && !button)
|
||||
return jsonError(res, 400, "button must be left|right|middle");
|
||||
|
||||
const modifiersRaw = toStringArray(body.modifiers) ?? [];
|
||||
const allowedModifiers = new Set<ClickModifier>([
|
||||
"Alt",
|
||||
"Control",
|
||||
"ControlOrMeta",
|
||||
"Meta",
|
||||
"Shift",
|
||||
]);
|
||||
const invalidModifiers = modifiersRaw.filter(
|
||||
(m) => !allowedModifiers.has(m as ClickModifier),
|
||||
);
|
||||
if (invalidModifiers.length)
|
||||
return jsonError(
|
||||
res,
|
||||
400,
|
||||
"modifiers must be Alt|Control|ControlOrMeta|Meta|Shift",
|
||||
);
|
||||
const modifiers = modifiersRaw.length
|
||||
? (modifiersRaw as ClickModifier[])
|
||||
: undefined;
|
||||
await clickViaPlaywright({
|
||||
cdpPort,
|
||||
targetId: tab.targetId,
|
||||
ref,
|
||||
doubleClick,
|
||||
button,
|
||||
modifiers,
|
||||
});
|
||||
return res.json({ ok: true, targetId: tab.targetId, url: tab.url });
|
||||
}
|
||||
case "type": {
|
||||
const ref = toStringOrEmpty(body.ref);
|
||||
if (!ref) return jsonError(res, 400, "ref is required");
|
||||
if (typeof body.text !== "string")
|
||||
return jsonError(res, 400, "text is required");
|
||||
const text = body.text;
|
||||
const submit = toBoolean(body.submit) ?? false;
|
||||
const slowly = toBoolean(body.slowly) ?? false;
|
||||
await typeViaPlaywright({
|
||||
cdpPort,
|
||||
targetId: tab.targetId,
|
||||
ref,
|
||||
text,
|
||||
submit,
|
||||
slowly,
|
||||
});
|
||||
return res.json({ ok: true, targetId: tab.targetId });
|
||||
}
|
||||
case "press": {
|
||||
const key = toStringOrEmpty(body.key);
|
||||
if (!key) return jsonError(res, 400, "key is required");
|
||||
await pressKeyViaPlaywright({ cdpPort, targetId: tab.targetId, key });
|
||||
return res.json({ ok: true, targetId: tab.targetId });
|
||||
}
|
||||
case "hover": {
|
||||
const ref = toStringOrEmpty(body.ref);
|
||||
if (!ref) return jsonError(res, 400, "ref is required");
|
||||
await hoverViaPlaywright({ cdpPort, targetId: tab.targetId, ref });
|
||||
return res.json({ ok: true, targetId: tab.targetId });
|
||||
}
|
||||
case "drag": {
|
||||
const startRef = toStringOrEmpty(body.startRef);
|
||||
const endRef = toStringOrEmpty(body.endRef);
|
||||
if (!startRef || !endRef)
|
||||
return jsonError(res, 400, "startRef and endRef are required");
|
||||
await dragViaPlaywright({
|
||||
cdpPort,
|
||||
targetId: tab.targetId,
|
||||
startRef,
|
||||
endRef,
|
||||
});
|
||||
return res.json({ ok: true, targetId: tab.targetId });
|
||||
}
|
||||
case "select": {
|
||||
const ref = toStringOrEmpty(body.ref);
|
||||
const values = toStringArray(body.values);
|
||||
if (!ref || !values?.length)
|
||||
return jsonError(res, 400, "ref and values are required");
|
||||
await selectOptionViaPlaywright({
|
||||
cdpPort,
|
||||
targetId: tab.targetId,
|
||||
ref,
|
||||
values,
|
||||
});
|
||||
return res.json({ ok: true, targetId: tab.targetId });
|
||||
}
|
||||
case "fill": {
|
||||
const fields = Array.isArray(body.fields)
|
||||
? (body.fields as Array<Record<string, unknown>>)
|
||||
: null;
|
||||
if (!fields?.length)
|
||||
return jsonError(res, 400, "fields are required");
|
||||
await fillFormViaPlaywright({
|
||||
cdpPort,
|
||||
targetId: tab.targetId,
|
||||
fields,
|
||||
});
|
||||
return res.json({ ok: true, targetId: tab.targetId });
|
||||
}
|
||||
case "resize": {
|
||||
const width = toNumber(body.width);
|
||||
const height = toNumber(body.height);
|
||||
if (!width || !height)
|
||||
return jsonError(res, 400, "width and height are required");
|
||||
await resizeViewportViaPlaywright({
|
||||
cdpPort,
|
||||
targetId: tab.targetId,
|
||||
width,
|
||||
height,
|
||||
});
|
||||
return res.json({ ok: true, targetId: tab.targetId, url: tab.url });
|
||||
}
|
||||
case "wait": {
|
||||
const timeMs = toNumber(body.timeMs);
|
||||
const text = toStringOrEmpty(body.text) || undefined;
|
||||
const textGone = toStringOrEmpty(body.textGone) || undefined;
|
||||
await waitForViaPlaywright({
|
||||
cdpPort,
|
||||
targetId: tab.targetId,
|
||||
timeMs,
|
||||
text,
|
||||
textGone,
|
||||
});
|
||||
return res.json({ ok: true, targetId: tab.targetId });
|
||||
}
|
||||
case "evaluate": {
|
||||
const fn = toStringOrEmpty(body.fn);
|
||||
if (!fn) return jsonError(res, 400, "fn is required");
|
||||
const ref = toStringOrEmpty(body.ref) || undefined;
|
||||
const result = await evaluateViaPlaywright({
|
||||
cdpPort,
|
||||
targetId: tab.targetId,
|
||||
fn,
|
||||
ref,
|
||||
});
|
||||
return res.json({
|
||||
ok: true,
|
||||
targetId: tab.targetId,
|
||||
url: tab.url,
|
||||
result,
|
||||
});
|
||||
}
|
||||
case "close": {
|
||||
await closePageViaPlaywright({ cdpPort, targetId: tab.targetId });
|
||||
return res.json({ ok: true, targetId: tab.targetId });
|
||||
}
|
||||
default: {
|
||||
return jsonError(res, 400, "unsupported kind");
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
handleRouteError(ctx, res, err);
|
||||
}
|
||||
});
|
||||
|
||||
app.post("/hooks/file-chooser", async (req, res) => {
|
||||
const body = readBody(req);
|
||||
const targetId = toStringOrEmpty(body.targetId) || undefined;
|
||||
const paths = toStringArray(body.paths) ?? [];
|
||||
const timeoutMs = toNumber(body.timeoutMs);
|
||||
if (!paths.length) return jsonError(res, 400, "paths are required");
|
||||
try {
|
||||
const tab = await ctx.ensureTabAvailable(targetId);
|
||||
await armFileUploadViaPlaywright({
|
||||
cdpPort: ctx.state().cdpPort,
|
||||
targetId: tab.targetId,
|
||||
paths,
|
||||
timeoutMs: timeoutMs ?? undefined,
|
||||
});
|
||||
res.json({ ok: true });
|
||||
} catch (err) {
|
||||
handleRouteError(ctx, res, err);
|
||||
}
|
||||
});
|
||||
|
||||
app.post("/hooks/dialog", async (req, res) => {
|
||||
const body = readBody(req);
|
||||
const targetId = toStringOrEmpty(body.targetId) || undefined;
|
||||
const accept = toBoolean(body.accept);
|
||||
const promptText = toStringOrEmpty(body.promptText) || undefined;
|
||||
const timeoutMs = toNumber(body.timeoutMs);
|
||||
if (accept === undefined) return jsonError(res, 400, "accept is required");
|
||||
try {
|
||||
const tab = await ctx.ensureTabAvailable(targetId);
|
||||
await armDialogViaPlaywright({
|
||||
cdpPort: ctx.state().cdpPort,
|
||||
targetId: tab.targetId,
|
||||
accept,
|
||||
promptText,
|
||||
timeoutMs: timeoutMs ?? undefined,
|
||||
});
|
||||
res.json({ ok: true });
|
||||
} catch (err) {
|
||||
handleRouteError(ctx, res, err);
|
||||
}
|
||||
});
|
||||
|
||||
app.get("/console", async (req, res) => {
|
||||
const targetId =
|
||||
typeof req.query.targetId === "string" ? req.query.targetId.trim() : "";
|
||||
const level = typeof req.query.level === "string" ? req.query.level : "";
|
||||
|
||||
try {
|
||||
const tab = await ctx.ensureTabAvailable(targetId || undefined);
|
||||
const messages = await getConsoleMessagesViaPlaywright({
|
||||
cdpPort: ctx.state().cdpPort,
|
||||
targetId: tab.targetId,
|
||||
level: level.trim() || undefined,
|
||||
});
|
||||
res.json({ ok: true, messages, targetId: tab.targetId });
|
||||
} catch (err) {
|
||||
handleRouteError(ctx, res, err);
|
||||
}
|
||||
});
|
||||
|
||||
app.post("/pdf", async (req, res) => {
|
||||
const body = readBody(req);
|
||||
const targetId = toStringOrEmpty(body.targetId) || undefined;
|
||||
try {
|
||||
const tab = await ctx.ensureTabAvailable(targetId);
|
||||
const pdf = await pdfViaPlaywright({
|
||||
cdpPort: ctx.state().cdpPort,
|
||||
targetId: tab.targetId,
|
||||
});
|
||||
await ensureMediaDir();
|
||||
const saved = await saveMediaBuffer(
|
||||
pdf.buffer,
|
||||
"application/pdf",
|
||||
"browser",
|
||||
pdf.buffer.byteLength,
|
||||
);
|
||||
res.json({
|
||||
ok: true,
|
||||
path: path.resolve(saved.path),
|
||||
targetId: tab.targetId,
|
||||
url: tab.url,
|
||||
});
|
||||
} catch (err) {
|
||||
handleRouteError(ctx, res, err);
|
||||
}
|
||||
});
|
||||
|
||||
app.post("/screenshot", async (req, res) => {
|
||||
const body = readBody(req);
|
||||
const targetId = toStringOrEmpty(body.targetId) || undefined;
|
||||
const fullPage = toBoolean(body.fullPage) ?? false;
|
||||
const ref = toStringOrEmpty(body.ref) || undefined;
|
||||
const element = toStringOrEmpty(body.element) || undefined;
|
||||
const type = body.type === "jpeg" ? "jpeg" : "png";
|
||||
|
||||
try {
|
||||
const tab = await ctx.ensureTabAvailable(targetId);
|
||||
const snap = await takeScreenshotViaPlaywright({
|
||||
cdpPort: ctx.state().cdpPort,
|
||||
targetId: tab.targetId,
|
||||
ref,
|
||||
element,
|
||||
fullPage,
|
||||
type,
|
||||
});
|
||||
|
||||
const normalized = await normalizeBrowserScreenshot(snap.buffer, {
|
||||
maxSide: DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE,
|
||||
maxBytes: DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
|
||||
});
|
||||
await ensureMediaDir();
|
||||
const saved = await saveMediaBuffer(
|
||||
normalized.buffer,
|
||||
normalized.contentType ?? `image/${type}`,
|
||||
"browser",
|
||||
DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
|
||||
);
|
||||
res.json({
|
||||
ok: true,
|
||||
path: path.resolve(saved.path),
|
||||
targetId: tab.targetId,
|
||||
url: tab.url,
|
||||
});
|
||||
} catch (err) {
|
||||
handleRouteError(ctx, res, err);
|
||||
}
|
||||
});
|
||||
|
||||
app.get("/snapshot", async (req, res) => {
|
||||
const targetId =
|
||||
typeof req.query.targetId === "string" ? req.query.targetId.trim() : "";
|
||||
const format = req.query.format === "aria" ? "aria" : "ai";
|
||||
const limit =
|
||||
typeof req.query.limit === "string" ? Number(req.query.limit) : undefined;
|
||||
|
||||
try {
|
||||
const tab = await ctx.ensureTabAvailable(targetId || undefined);
|
||||
if (format === "ai") {
|
||||
const snap = await snapshotAiViaPlaywright({
|
||||
cdpPort: ctx.state().cdpPort,
|
||||
targetId: tab.targetId,
|
||||
});
|
||||
return res.json({
|
||||
ok: true,
|
||||
format,
|
||||
targetId: tab.targetId,
|
||||
url: tab.url,
|
||||
...snap,
|
||||
});
|
||||
}
|
||||
|
||||
const snap = await snapshotAria({
|
||||
wsUrl: tab.wsUrl ?? "",
|
||||
limit,
|
||||
});
|
||||
return res.json({
|
||||
ok: true,
|
||||
format,
|
||||
targetId: tab.targetId,
|
||||
url: tab.url,
|
||||
...snap,
|
||||
});
|
||||
} catch (err) {
|
||||
handleRouteError(ctx, res, err);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -1,9 +1,8 @@
|
||||
import type express from "express";
|
||||
|
||||
import type { BrowserRouteContext } from "../server-context.js";
|
||||
import { registerBrowserActionRoutes } from "./actions.js";
|
||||
import { registerBrowserAgentRoutes } from "./agent.js";
|
||||
import { registerBrowserBasicRoutes } from "./basic.js";
|
||||
import { registerBrowserInspectRoutes } from "./inspect.js";
|
||||
import { registerBrowserTabRoutes } from "./tabs.js";
|
||||
|
||||
export function registerBrowserRoutes(
|
||||
@@ -12,6 +11,5 @@ export function registerBrowserRoutes(
|
||||
) {
|
||||
registerBrowserBasicRoutes(app, ctx);
|
||||
registerBrowserTabRoutes(app, ctx);
|
||||
registerBrowserInspectRoutes(app, ctx);
|
||||
registerBrowserActionRoutes(app, ctx);
|
||||
registerBrowserAgentRoutes(app, ctx);
|
||||
}
|
||||
|
||||
@@ -1,244 +0,0 @@
|
||||
import path from "node:path";
|
||||
|
||||
import type express from "express";
|
||||
import { ensureMediaDir, saveMediaBuffer } from "../../media/store.js";
|
||||
import {
|
||||
captureScreenshot,
|
||||
captureScreenshotPng,
|
||||
getDomText,
|
||||
querySelector,
|
||||
snapshotAria,
|
||||
snapshotDom,
|
||||
} from "../cdp.js";
|
||||
import {
|
||||
snapshotAiViaPlaywright,
|
||||
takeScreenshotViaPlaywright,
|
||||
} from "../pw-ai.js";
|
||||
import {
|
||||
DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
|
||||
DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE,
|
||||
normalizeBrowserScreenshot,
|
||||
} from "../screenshot.js";
|
||||
import type { BrowserRouteContext } from "../server-context.js";
|
||||
import { jsonError, toBoolean, toStringOrEmpty } from "./utils.js";
|
||||
|
||||
export function registerBrowserInspectRoutes(
|
||||
app: express.Express,
|
||||
ctx: BrowserRouteContext,
|
||||
) {
|
||||
app.get("/screenshot", async (req, res) => {
|
||||
const targetId =
|
||||
typeof req.query.targetId === "string" ? req.query.targetId.trim() : "";
|
||||
const fullPage =
|
||||
req.query.fullPage === "true" || req.query.fullPage === "1";
|
||||
|
||||
try {
|
||||
const tab = await ctx.ensureTabAvailable(targetId || undefined);
|
||||
|
||||
let shot: Buffer<ArrayBufferLike> = Buffer.alloc(0);
|
||||
let contentTypeHint: "image/jpeg" | "image/png" = "image/jpeg";
|
||||
try {
|
||||
shot = await captureScreenshot({
|
||||
wsUrl: tab.wsUrl ?? "",
|
||||
fullPage,
|
||||
format: "jpeg",
|
||||
quality: 85,
|
||||
});
|
||||
} catch {
|
||||
contentTypeHint = "image/png";
|
||||
shot = await captureScreenshotPng({
|
||||
wsUrl: tab.wsUrl ?? "",
|
||||
fullPage,
|
||||
});
|
||||
}
|
||||
|
||||
const normalized = await normalizeBrowserScreenshot(shot, {
|
||||
maxSide: DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE,
|
||||
maxBytes: DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
|
||||
});
|
||||
await ensureMediaDir();
|
||||
const saved = await saveMediaBuffer(
|
||||
normalized.buffer,
|
||||
normalized.contentType ?? contentTypeHint,
|
||||
"browser",
|
||||
DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
|
||||
);
|
||||
const filePath = path.resolve(saved.path);
|
||||
res.json({
|
||||
ok: true,
|
||||
path: filePath,
|
||||
targetId: tab.targetId,
|
||||
url: tab.url,
|
||||
});
|
||||
} catch (err) {
|
||||
const mapped = ctx.mapTabError(err);
|
||||
if (mapped) return jsonError(res, mapped.status, mapped.message);
|
||||
jsonError(res, 500, String(err));
|
||||
}
|
||||
});
|
||||
|
||||
app.post("/screenshot", async (req, res) => {
|
||||
const body = req.body as Record<string, unknown>;
|
||||
const targetId = toStringOrEmpty(body?.targetId);
|
||||
const fullPage = toBoolean(body?.fullPage) ?? false;
|
||||
const ref = toStringOrEmpty(body?.ref);
|
||||
const element = toStringOrEmpty(body?.element);
|
||||
const type = body?.type === "jpeg" ? "jpeg" : "png";
|
||||
const filename = toStringOrEmpty(body?.filename);
|
||||
|
||||
try {
|
||||
const tab = await ctx.ensureTabAvailable(targetId || undefined);
|
||||
const snap = await takeScreenshotViaPlaywright({
|
||||
cdpPort: ctx.state().cdpPort,
|
||||
targetId: tab.targetId,
|
||||
ref,
|
||||
element,
|
||||
fullPage,
|
||||
type,
|
||||
});
|
||||
const buffer = snap.buffer;
|
||||
const normalized = await normalizeBrowserScreenshot(buffer, {
|
||||
maxSide: DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE,
|
||||
maxBytes: DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
|
||||
});
|
||||
await ensureMediaDir();
|
||||
const saved = await saveMediaBuffer(
|
||||
normalized.buffer,
|
||||
normalized.contentType ?? `image/${type}`,
|
||||
"browser",
|
||||
DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
|
||||
);
|
||||
const filePath = path.resolve(saved.path);
|
||||
res.json({
|
||||
ok: true,
|
||||
path: filePath,
|
||||
targetId: tab.targetId,
|
||||
url: tab.url,
|
||||
filename: filename || undefined,
|
||||
});
|
||||
} catch (err) {
|
||||
const mapped = ctx.mapTabError(err);
|
||||
if (mapped) return jsonError(res, mapped.status, mapped.message);
|
||||
jsonError(res, 500, String(err));
|
||||
}
|
||||
});
|
||||
|
||||
app.get("/query", async (req, res) => {
|
||||
const selector =
|
||||
typeof req.query.selector === "string" ? req.query.selector.trim() : "";
|
||||
const targetId =
|
||||
typeof req.query.targetId === "string" ? req.query.targetId.trim() : "";
|
||||
const limit =
|
||||
typeof req.query.limit === "string" ? Number(req.query.limit) : undefined;
|
||||
|
||||
if (!selector) return jsonError(res, 400, "selector is required");
|
||||
|
||||
try {
|
||||
const tab = await ctx.ensureTabAvailable(targetId || undefined);
|
||||
const result = await querySelector({
|
||||
wsUrl: tab.wsUrl ?? "",
|
||||
selector,
|
||||
limit,
|
||||
});
|
||||
res.json({ ok: true, targetId: tab.targetId, url: tab.url, ...result });
|
||||
} catch (err) {
|
||||
const mapped = ctx.mapTabError(err);
|
||||
if (mapped) return jsonError(res, mapped.status, mapped.message);
|
||||
jsonError(res, 500, String(err));
|
||||
}
|
||||
});
|
||||
|
||||
app.get("/dom", async (req, res) => {
|
||||
const targetId =
|
||||
typeof req.query.targetId === "string" ? req.query.targetId.trim() : "";
|
||||
const format = req.query.format === "text" ? "text" : "html";
|
||||
const selector =
|
||||
typeof req.query.selector === "string" ? req.query.selector.trim() : "";
|
||||
const maxChars =
|
||||
typeof req.query.maxChars === "string"
|
||||
? Number(req.query.maxChars)
|
||||
: undefined;
|
||||
|
||||
try {
|
||||
const tab = await ctx.ensureTabAvailable(targetId || undefined);
|
||||
const result = await getDomText({
|
||||
wsUrl: tab.wsUrl ?? "",
|
||||
format,
|
||||
maxChars,
|
||||
selector: selector || undefined,
|
||||
});
|
||||
res.json({
|
||||
ok: true,
|
||||
targetId: tab.targetId,
|
||||
url: tab.url,
|
||||
format,
|
||||
...result,
|
||||
});
|
||||
} catch (err) {
|
||||
const mapped = ctx.mapTabError(err);
|
||||
if (mapped) return jsonError(res, mapped.status, mapped.message);
|
||||
jsonError(res, 500, String(err));
|
||||
}
|
||||
});
|
||||
|
||||
app.get("/snapshot", async (req, res) => {
|
||||
const targetId =
|
||||
typeof req.query.targetId === "string" ? req.query.targetId.trim() : "";
|
||||
const format =
|
||||
req.query.format === "domSnapshot"
|
||||
? "domSnapshot"
|
||||
: req.query.format === "ai"
|
||||
? "ai"
|
||||
: "aria";
|
||||
const limit =
|
||||
typeof req.query.limit === "string" ? Number(req.query.limit) : undefined;
|
||||
|
||||
try {
|
||||
const tab = await ctx.ensureTabAvailable(targetId || undefined);
|
||||
|
||||
if (format === "ai") {
|
||||
const snap = await snapshotAiViaPlaywright({
|
||||
cdpPort: ctx.state().cdpPort,
|
||||
targetId: tab.targetId,
|
||||
});
|
||||
return res.json({
|
||||
ok: true,
|
||||
format,
|
||||
targetId: tab.targetId,
|
||||
url: tab.url,
|
||||
...snap,
|
||||
});
|
||||
}
|
||||
|
||||
if (format === "aria") {
|
||||
const snap = await snapshotAria({
|
||||
wsUrl: tab.wsUrl ?? "",
|
||||
limit,
|
||||
});
|
||||
return res.json({
|
||||
ok: true,
|
||||
format,
|
||||
targetId: tab.targetId,
|
||||
url: tab.url,
|
||||
...snap,
|
||||
});
|
||||
}
|
||||
|
||||
const snap = await snapshotDom({
|
||||
wsUrl: tab.wsUrl ?? "",
|
||||
limit,
|
||||
});
|
||||
return res.json({
|
||||
ok: true,
|
||||
format,
|
||||
targetId: tab.targetId,
|
||||
url: tab.url,
|
||||
...snap,
|
||||
});
|
||||
} catch (err) {
|
||||
const mapped = ctx.mapTabError(err);
|
||||
if (mapped) return jsonError(res, mapped.status, mapped.message);
|
||||
jsonError(res, 500, String(err));
|
||||
}
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user