Merge pull request #567 from erikpr1994/fix/gemini-schema-sanitization
fix(agents): remove unsupported JSON Schema keywords for Cloud Code Assist API
This commit is contained in:
@@ -29,6 +29,7 @@
|
|||||||
- Heartbeat: resolve Telegram account IDs from config-only tokens; cron tool accepts canonical `jobId` and legacy `id` for job actions. (#516) — thanks @YuriNachos
|
- Heartbeat: resolve Telegram account IDs from config-only tokens; cron tool accepts canonical `jobId` and legacy `id` for job actions. (#516) — thanks @YuriNachos
|
||||||
- Discord: stop provider when gateway reconnects are exhausted and surface errors. (#514) — thanks @joshp123
|
- Discord: stop provider when gateway reconnects are exhausted and surface errors. (#514) — thanks @joshp123
|
||||||
- Agents: strip empty assistant text blocks from session history to avoid Claude API 400s. (#210)
|
- Agents: strip empty assistant text blocks from session history to avoid Claude API 400s. (#210)
|
||||||
|
- Agents: scrub unsupported JSON Schema keywords from tool schemas for Cloud Code Assist API compatibility. (#567) — thanks @erikpr1994
|
||||||
- Auto-reply: preserve block reply ordering with timeout fallback for streaming. (#503) — thanks @joshp123
|
- Auto-reply: preserve block reply ordering with timeout fallback for streaming. (#503) — thanks @joshp123
|
||||||
- Auto-reply: block reply ordering fix (duplicate PR superseded by #503). (#483) — thanks @AbhisekBasu1
|
- Auto-reply: block reply ordering fix (duplicate PR superseded by #503). (#483) — thanks @AbhisekBasu1
|
||||||
- Auto-reply: avoid splitting outbound chunks inside parentheses. (#499) — thanks @philipp-spiess
|
- Auto-reply: avoid splitting outbound chunks inside parentheses. (#499) — thanks @philipp-spiess
|
||||||
|
|||||||
@@ -50,35 +50,40 @@ beforeEach(() => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
describe("bash tool backgrounding", () => {
|
describe("bash tool backgrounding", () => {
|
||||||
it("backgrounds after yield and can be polled", async () => {
|
it(
|
||||||
const result = await bashTool.execute("call1", {
|
"backgrounds after yield and can be polled",
|
||||||
command: joinCommands([yieldDelayCmd, "echo done"]),
|
async () => {
|
||||||
yieldMs: 10,
|
const result = await bashTool.execute("call1", {
|
||||||
});
|
command: joinCommands([yieldDelayCmd, "echo done"]),
|
||||||
|
yieldMs: 10,
|
||||||
expect(result.details.status).toBe("running");
|
|
||||||
const sessionId = (result.details as { sessionId: string }).sessionId;
|
|
||||||
|
|
||||||
let status = "running";
|
|
||||||
let output = "";
|
|
||||||
const deadline = Date.now() + (process.platform === "win32" ? 8000 : 2000);
|
|
||||||
|
|
||||||
while (Date.now() < deadline && status === "running") {
|
|
||||||
const poll = await processTool.execute("call2", {
|
|
||||||
action: "poll",
|
|
||||||
sessionId,
|
|
||||||
});
|
});
|
||||||
status = (poll.details as { status: string }).status;
|
|
||||||
const textBlock = poll.content.find((c) => c.type === "text");
|
|
||||||
output = textBlock?.text ?? "";
|
|
||||||
if (status === "running") {
|
|
||||||
await sleep(20);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
expect(status).toBe("completed");
|
expect(result.details.status).toBe("running");
|
||||||
expect(output).toContain("done");
|
const sessionId = (result.details as { sessionId: string }).sessionId;
|
||||||
});
|
|
||||||
|
let status = "running";
|
||||||
|
let output = "";
|
||||||
|
const deadline =
|
||||||
|
Date.now() + (process.platform === "win32" ? 8000 : 2000);
|
||||||
|
|
||||||
|
while (Date.now() < deadline && status === "running") {
|
||||||
|
const poll = await processTool.execute("call2", {
|
||||||
|
action: "poll",
|
||||||
|
sessionId,
|
||||||
|
});
|
||||||
|
status = (poll.details as { status: string }).status;
|
||||||
|
const textBlock = poll.content.find((c) => c.type === "text");
|
||||||
|
output = textBlock?.text ?? "";
|
||||||
|
if (status === "running") {
|
||||||
|
await sleep(20);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
expect(status).toBe("completed");
|
||||||
|
expect(output).toContain("done");
|
||||||
|
},
|
||||||
|
isWin ? 15_000 : 5_000,
|
||||||
|
);
|
||||||
|
|
||||||
it("supports explicit background", async () => {
|
it("supports explicit background", async () => {
|
||||||
const result = await bashTool.execute("call1", {
|
const result = await bashTool.execute("call1", {
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import type { AgentTool, AgentToolResult } from "@mariozechner/pi-agent-core";
|
|||||||
import { Type } from "@sinclair/typebox";
|
import { Type } from "@sinclair/typebox";
|
||||||
|
|
||||||
import { logInfo } from "../logger.js";
|
import { logInfo } from "../logger.js";
|
||||||
|
import { sliceUtf16Safe } from "../utils.js";
|
||||||
import {
|
import {
|
||||||
addSession,
|
addSession,
|
||||||
appendOutput,
|
appendOutput,
|
||||||
@@ -1041,7 +1042,7 @@ function chunkString(input: string, limit = CHUNK_LIMIT) {
|
|||||||
function truncateMiddle(str: string, max: number) {
|
function truncateMiddle(str: string, max: number) {
|
||||||
if (str.length <= max) return str;
|
if (str.length <= max) return str;
|
||||||
const half = Math.floor((max - 3) / 2);
|
const half = Math.floor((max - 3) / 2);
|
||||||
return `${str.slice(0, half)}...${str.slice(str.length - half)}`;
|
return `${sliceUtf16Safe(str, 0, half)}...${sliceUtf16Safe(str, -half)}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
function sliceLogLines(
|
function sliceLogLines(
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import { resolveStateDir } from "../config/paths.js";
|
|||||||
import { emitAgentEvent } from "../infra/agent-events.js";
|
import { emitAgentEvent } from "../infra/agent-events.js";
|
||||||
import { createSubsystemLogger } from "../logging.js";
|
import { createSubsystemLogger } from "../logging.js";
|
||||||
import { splitMediaFromOutput } from "../media/parse.js";
|
import { splitMediaFromOutput } from "../media/parse.js";
|
||||||
|
import { truncateUtf16Safe } from "../utils.js";
|
||||||
import type { BlockReplyChunking } from "./pi-embedded-block-chunker.js";
|
import type { BlockReplyChunking } from "./pi-embedded-block-chunker.js";
|
||||||
import { EmbeddedBlockChunker } from "./pi-embedded-block-chunker.js";
|
import { EmbeddedBlockChunker } from "./pi-embedded-block-chunker.js";
|
||||||
import { isMessagingToolDuplicate } from "./pi-embedded-helpers.js";
|
import { isMessagingToolDuplicate } from "./pi-embedded-helpers.js";
|
||||||
@@ -64,7 +65,7 @@ type MessagingToolSend = {
|
|||||||
|
|
||||||
function truncateToolText(text: string): string {
|
function truncateToolText(text: string): string {
|
||||||
if (text.length <= TOOL_RESULT_MAX_CHARS) return text;
|
if (text.length <= TOOL_RESULT_MAX_CHARS) return text;
|
||||||
return `${text.slice(0, TOOL_RESULT_MAX_CHARS)}\n…(truncated)…`;
|
return `${truncateUtf16Safe(text, TOOL_RESULT_MAX_CHARS)}\n…(truncated)…`;
|
||||||
}
|
}
|
||||||
|
|
||||||
function sanitizeToolResult(result: unknown): unknown {
|
function sanitizeToolResult(result: unknown): unknown {
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import path from "node:path";
|
|||||||
|
|
||||||
import sharp from "sharp";
|
import sharp from "sharp";
|
||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import { createClawdbotCodingTools } from "./pi-tools.js";
|
import { __testing, createClawdbotCodingTools } from "./pi-tools.js";
|
||||||
import { createBrowserTool } from "./tools/browser-tool.js";
|
import { createBrowserTool } from "./tools/browser-tool.js";
|
||||||
|
|
||||||
describe("createClawdbotCodingTools", () => {
|
describe("createClawdbotCodingTools", () => {
|
||||||
@@ -64,6 +64,28 @@ describe("createClawdbotCodingTools", () => {
|
|||||||
expect(format?.enum).toEqual(["aria", "ai"]);
|
expect(format?.enum).toEqual(["aria", "ai"]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("inlines local $ref before removing unsupported keywords", () => {
|
||||||
|
const cleaned = __testing.cleanToolSchemaForGemini({
|
||||||
|
type: "object",
|
||||||
|
properties: {
|
||||||
|
foo: { $ref: "#/$defs/Foo" },
|
||||||
|
},
|
||||||
|
$defs: {
|
||||||
|
Foo: { type: "string", enum: ["a", "b"] },
|
||||||
|
},
|
||||||
|
}) as {
|
||||||
|
$defs?: unknown;
|
||||||
|
properties?: Record<string, unknown>;
|
||||||
|
};
|
||||||
|
|
||||||
|
expect(cleaned.$defs).toBeUndefined();
|
||||||
|
expect(cleaned.properties).toBeDefined();
|
||||||
|
expect(cleaned.properties?.foo).toMatchObject({
|
||||||
|
type: "string",
|
||||||
|
enum: ["a", "b"],
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it("preserves action enums in normalized schemas", () => {
|
it("preserves action enums in normalized schemas", () => {
|
||||||
const tools = createClawdbotCodingTools();
|
const tools = createClawdbotCodingTools();
|
||||||
const toolNames = [
|
const toolNames = [
|
||||||
@@ -331,4 +353,52 @@ describe("createClawdbotCodingTools", () => {
|
|||||||
expect(tools.some((tool) => tool.name === "Bash")).toBe(true);
|
expect(tools.some((tool) => tool.name === "Bash")).toBe(true);
|
||||||
expect(tools.some((tool) => tool.name === "browser")).toBe(false);
|
expect(tools.some((tool) => tool.name === "browser")).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("removes unsupported JSON Schema keywords for Cloud Code Assist API compatibility", () => {
|
||||||
|
const tools = createClawdbotCodingTools();
|
||||||
|
|
||||||
|
// Helper to recursively check schema for unsupported keywords
|
||||||
|
const unsupportedKeywords = new Set([
|
||||||
|
"patternProperties",
|
||||||
|
"additionalProperties",
|
||||||
|
"$schema",
|
||||||
|
"$id",
|
||||||
|
"$ref",
|
||||||
|
"$defs",
|
||||||
|
"definitions",
|
||||||
|
]);
|
||||||
|
|
||||||
|
const findUnsupportedKeywords = (
|
||||||
|
schema: unknown,
|
||||||
|
path: string,
|
||||||
|
): string[] => {
|
||||||
|
const found: string[] = [];
|
||||||
|
if (!schema || typeof schema !== "object") return found;
|
||||||
|
if (Array.isArray(schema)) {
|
||||||
|
schema.forEach((item, i) => {
|
||||||
|
found.push(...findUnsupportedKeywords(item, `${path}[${i}]`));
|
||||||
|
});
|
||||||
|
return found;
|
||||||
|
}
|
||||||
|
for (const [key, value] of Object.entries(
|
||||||
|
schema as Record<string, unknown>,
|
||||||
|
)) {
|
||||||
|
if (unsupportedKeywords.has(key)) {
|
||||||
|
found.push(`${path}.${key}`);
|
||||||
|
}
|
||||||
|
if (value && typeof value === "object") {
|
||||||
|
found.push(...findUnsupportedKeywords(value, `${path}.${key}`));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return found;
|
||||||
|
};
|
||||||
|
|
||||||
|
for (const tool of tools) {
|
||||||
|
const violations = findUnsupportedKeywords(
|
||||||
|
tool.parameters,
|
||||||
|
`${tool.name}.parameters`,
|
||||||
|
);
|
||||||
|
expect(violations).toEqual([]);
|
||||||
|
}
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -195,12 +195,122 @@ function tryFlattenLiteralAnyOf(
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
function cleanSchemaForGemini(schema: unknown): unknown {
|
// Keywords that Cloud Code Assist API rejects (not compliant with their JSON Schema subset)
|
||||||
|
const UNSUPPORTED_SCHEMA_KEYWORDS = new Set([
|
||||||
|
"patternProperties",
|
||||||
|
"additionalProperties",
|
||||||
|
"$schema",
|
||||||
|
"$id",
|
||||||
|
"$ref",
|
||||||
|
"$defs",
|
||||||
|
"definitions",
|
||||||
|
]);
|
||||||
|
|
||||||
|
type SchemaDefs = Map<string, unknown>;
|
||||||
|
|
||||||
|
function extendSchemaDefs(
|
||||||
|
defs: SchemaDefs | undefined,
|
||||||
|
schema: Record<string, unknown>,
|
||||||
|
): SchemaDefs | undefined {
|
||||||
|
const defsEntry =
|
||||||
|
schema.$defs &&
|
||||||
|
typeof schema.$defs === "object" &&
|
||||||
|
!Array.isArray(schema.$defs)
|
||||||
|
? (schema.$defs as Record<string, unknown>)
|
||||||
|
: undefined;
|
||||||
|
const legacyDefsEntry =
|
||||||
|
schema.definitions &&
|
||||||
|
typeof schema.definitions === "object" &&
|
||||||
|
!Array.isArray(schema.definitions)
|
||||||
|
? (schema.definitions as Record<string, unknown>)
|
||||||
|
: undefined;
|
||||||
|
|
||||||
|
if (!defsEntry && !legacyDefsEntry) return defs;
|
||||||
|
|
||||||
|
const next = defs ? new Map(defs) : new Map<string, unknown>();
|
||||||
|
if (defsEntry) {
|
||||||
|
for (const [key, value] of Object.entries(defsEntry)) next.set(key, value);
|
||||||
|
}
|
||||||
|
if (legacyDefsEntry) {
|
||||||
|
for (const [key, value] of Object.entries(legacyDefsEntry))
|
||||||
|
next.set(key, value);
|
||||||
|
}
|
||||||
|
return next;
|
||||||
|
}
|
||||||
|
|
||||||
|
function decodeJsonPointerSegment(segment: string): string {
|
||||||
|
return segment.replaceAll("~1", "/").replaceAll("~0", "~");
|
||||||
|
}
|
||||||
|
|
||||||
|
function tryResolveLocalRef(
|
||||||
|
ref: string,
|
||||||
|
defs: SchemaDefs | undefined,
|
||||||
|
): unknown | undefined {
|
||||||
|
if (!defs) return undefined;
|
||||||
|
const match = ref.match(/^#\/(?:\$defs|definitions)\/(.+)$/);
|
||||||
|
if (!match) return undefined;
|
||||||
|
const name = decodeJsonPointerSegment(match[1] ?? "");
|
||||||
|
if (!name) return undefined;
|
||||||
|
return defs.get(name);
|
||||||
|
}
|
||||||
|
|
||||||
|
function cleanSchemaForGeminiWithDefs(
|
||||||
|
schema: unknown,
|
||||||
|
defs: SchemaDefs | undefined,
|
||||||
|
refStack: Set<string> | undefined,
|
||||||
|
): unknown {
|
||||||
if (!schema || typeof schema !== "object") return schema;
|
if (!schema || typeof schema !== "object") return schema;
|
||||||
if (Array.isArray(schema)) return schema.map(cleanSchemaForGemini);
|
if (Array.isArray(schema)) {
|
||||||
|
return schema.map((item) =>
|
||||||
|
cleanSchemaForGeminiWithDefs(item, defs, refStack),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
const obj = schema as Record<string, unknown>;
|
const obj = schema as Record<string, unknown>;
|
||||||
|
const nextDefs = extendSchemaDefs(defs, obj);
|
||||||
|
|
||||||
|
const refValue = typeof obj.$ref === "string" ? obj.$ref : undefined;
|
||||||
|
if (refValue) {
|
||||||
|
if (refStack?.has(refValue)) {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
const resolved = tryResolveLocalRef(refValue, nextDefs);
|
||||||
|
if (resolved) {
|
||||||
|
const nextRefStack = refStack ? new Set(refStack) : new Set<string>();
|
||||||
|
nextRefStack.add(refValue);
|
||||||
|
|
||||||
|
const cleaned = cleanSchemaForGeminiWithDefs(
|
||||||
|
resolved,
|
||||||
|
nextDefs,
|
||||||
|
nextRefStack,
|
||||||
|
);
|
||||||
|
if (!cleaned || typeof cleaned !== "object" || Array.isArray(cleaned)) {
|
||||||
|
return cleaned;
|
||||||
|
}
|
||||||
|
|
||||||
|
const result: Record<string, unknown> = {
|
||||||
|
...(cleaned as Record<string, unknown>),
|
||||||
|
};
|
||||||
|
for (const key of ["description", "title", "default", "examples"]) {
|
||||||
|
if (key in obj && obj[key] !== undefined) {
|
||||||
|
result[key] = obj[key];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
const result: Record<string, unknown> = {};
|
||||||
|
for (const key of ["description", "title", "default", "examples"]) {
|
||||||
|
if (key in obj && obj[key] !== undefined) {
|
||||||
|
result[key] = obj[key];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
const hasAnyOf = "anyOf" in obj && Array.isArray(obj.anyOf);
|
const hasAnyOf = "anyOf" in obj && Array.isArray(obj.anyOf);
|
||||||
|
const hasOneOf = "oneOf" in obj && Array.isArray(obj.oneOf);
|
||||||
|
|
||||||
// Try to flatten anyOf of literals to a single enum BEFORE processing
|
// Try to flatten anyOf of literals to a single enum BEFORE processing
|
||||||
// This handles Type.Union([Type.Literal("a"), Type.Literal("b")]) patterns
|
// This handles Type.Union([Type.Literal("a"), Type.Literal("b")]) patterns
|
||||||
@@ -221,14 +331,28 @@ function cleanSchemaForGemini(schema: unknown): unknown {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Try to flatten oneOf of literals similarly
|
||||||
|
if (hasOneOf) {
|
||||||
|
const flattened = tryFlattenLiteralAnyOf(obj.oneOf as unknown[]);
|
||||||
|
if (flattened) {
|
||||||
|
const result: Record<string, unknown> = {
|
||||||
|
type: flattened.type,
|
||||||
|
enum: flattened.enum,
|
||||||
|
};
|
||||||
|
for (const key of ["description", "title", "default", "examples"]) {
|
||||||
|
if (key in obj && obj[key] !== undefined) {
|
||||||
|
result[key] = obj[key];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const cleaned: Record<string, unknown> = {};
|
const cleaned: Record<string, unknown> = {};
|
||||||
|
|
||||||
for (const [key, value] of Object.entries(obj)) {
|
for (const [key, value] of Object.entries(obj)) {
|
||||||
// Skip unsupported schema features for Gemini:
|
// Skip keywords that Cloud Code Assist API doesn't support
|
||||||
// - patternProperties: not in OpenAPI 3.0 subset
|
if (UNSUPPORTED_SCHEMA_KEYWORDS.has(key)) {
|
||||||
// - const: convert to enum with single value instead
|
|
||||||
if (key === "patternProperties") {
|
|
||||||
// Gemini doesn't support patternProperties - skip it
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -238,8 +362,8 @@ function cleanSchemaForGemini(schema: unknown): unknown {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Skip 'type' if we have 'anyOf' — Gemini doesn't allow both
|
// Skip 'type' if we have 'anyOf' or 'oneOf' — Gemini doesn't allow both
|
||||||
if (key === "type" && hasAnyOf) {
|
if (key === "type" && (hasAnyOf || hasOneOf)) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -247,27 +371,29 @@ function cleanSchemaForGemini(schema: unknown): unknown {
|
|||||||
// Recursively clean nested properties
|
// Recursively clean nested properties
|
||||||
const props = value as Record<string, unknown>;
|
const props = value as Record<string, unknown>;
|
||||||
cleaned[key] = Object.fromEntries(
|
cleaned[key] = Object.fromEntries(
|
||||||
Object.entries(props).map(([k, v]) => [k, cleanSchemaForGemini(v)]),
|
Object.entries(props).map(([k, v]) => [
|
||||||
|
k,
|
||||||
|
cleanSchemaForGeminiWithDefs(v, nextDefs, refStack),
|
||||||
|
]),
|
||||||
);
|
);
|
||||||
} else if (key === "items" && value && typeof value === "object") {
|
} else if (key === "items" && value && typeof value === "object") {
|
||||||
// Recursively clean array items schema
|
// Recursively clean array items schema
|
||||||
cleaned[key] = cleanSchemaForGemini(value);
|
cleaned[key] = cleanSchemaForGeminiWithDefs(value, nextDefs, refStack);
|
||||||
} else if (key === "anyOf" && Array.isArray(value)) {
|
} else if (key === "anyOf" && Array.isArray(value)) {
|
||||||
// Clean each anyOf variant
|
// Clean each anyOf variant
|
||||||
cleaned[key] = value.map((variant) => cleanSchemaForGemini(variant));
|
cleaned[key] = value.map((variant) =>
|
||||||
|
cleanSchemaForGeminiWithDefs(variant, nextDefs, refStack),
|
||||||
|
);
|
||||||
} else if (key === "oneOf" && Array.isArray(value)) {
|
} else if (key === "oneOf" && Array.isArray(value)) {
|
||||||
// Clean each oneOf variant
|
// Clean each oneOf variant
|
||||||
cleaned[key] = value.map((variant) => cleanSchemaForGemini(variant));
|
cleaned[key] = value.map((variant) =>
|
||||||
|
cleanSchemaForGeminiWithDefs(variant, nextDefs, refStack),
|
||||||
|
);
|
||||||
} else if (key === "allOf" && Array.isArray(value)) {
|
} else if (key === "allOf" && Array.isArray(value)) {
|
||||||
// Clean each allOf variant
|
// Clean each allOf variant
|
||||||
cleaned[key] = value.map((variant) => cleanSchemaForGemini(variant));
|
cleaned[key] = value.map((variant) =>
|
||||||
} else if (
|
cleanSchemaForGeminiWithDefs(variant, nextDefs, refStack),
|
||||||
key === "additionalProperties" &&
|
);
|
||||||
value &&
|
|
||||||
typeof value === "object"
|
|
||||||
) {
|
|
||||||
// Recursively clean additionalProperties schema
|
|
||||||
cleaned[key] = cleanSchemaForGemini(value);
|
|
||||||
} else {
|
} else {
|
||||||
cleaned[key] = value;
|
cleaned[key] = value;
|
||||||
}
|
}
|
||||||
@@ -276,6 +402,18 @@ function cleanSchemaForGemini(schema: unknown): unknown {
|
|||||||
return cleaned;
|
return cleaned;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function cleanSchemaForGemini(schema: unknown): unknown {
|
||||||
|
if (!schema || typeof schema !== "object") return schema;
|
||||||
|
if (Array.isArray(schema)) return schema.map(cleanSchemaForGemini);
|
||||||
|
|
||||||
|
const defs = extendSchemaDefs(undefined, schema as Record<string, unknown>);
|
||||||
|
return cleanSchemaForGeminiWithDefs(schema, defs, undefined);
|
||||||
|
}
|
||||||
|
|
||||||
|
function cleanToolSchemaForGemini(schema: Record<string, unknown>): unknown {
|
||||||
|
return cleanSchemaForGemini(schema);
|
||||||
|
}
|
||||||
|
|
||||||
function normalizeToolParameters(tool: AnyAgentTool): AnyAgentTool {
|
function normalizeToolParameters(tool: AnyAgentTool): AnyAgentTool {
|
||||||
const schema =
|
const schema =
|
||||||
tool.parameters && typeof tool.parameters === "object"
|
tool.parameters && typeof tool.parameters === "object"
|
||||||
@@ -613,6 +751,10 @@ function createClawdbotReadTool(base: AnyAgentTool): AnyAgentTool {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export const __testing = {
|
||||||
|
cleanToolSchemaForGemini,
|
||||||
|
} as const;
|
||||||
|
|
||||||
export function createClawdbotCodingTools(options?: {
|
export function createClawdbotCodingTools(options?: {
|
||||||
bash?: BashToolDefaults & ProcessToolDefaults;
|
bash?: BashToolDefaults & ProcessToolDefaults;
|
||||||
messageProvider?: string;
|
messageProvider?: string;
|
||||||
|
|||||||
@@ -1202,16 +1202,16 @@ export type AgentDefaultsConfig = {
|
|||||||
every?: string;
|
every?: string;
|
||||||
/** Heartbeat model override (provider/model). */
|
/** Heartbeat model override (provider/model). */
|
||||||
model?: string;
|
model?: string;
|
||||||
/** Delivery target (last|whatsapp|telegram|discord|signal|imessage|none). */
|
/** Delivery target (last|whatsapp|telegram|discord|slack|msteams|signal|imessage|none). */
|
||||||
target?:
|
target?:
|
||||||
| "last"
|
| "last"
|
||||||
| "whatsapp"
|
| "whatsapp"
|
||||||
| "telegram"
|
| "telegram"
|
||||||
| "discord"
|
| "discord"
|
||||||
| "slack"
|
| "slack"
|
||||||
|
| "msteams"
|
||||||
| "signal"
|
| "signal"
|
||||||
| "imessage"
|
| "imessage"
|
||||||
| "msteams"
|
|
||||||
| "none";
|
| "none";
|
||||||
/** Optional delivery override (E.164 for WhatsApp, chat id for Telegram). */
|
/** Optional delivery override (E.164 for WhatsApp, chat id for Telegram). */
|
||||||
to?: string;
|
to?: string;
|
||||||
|
|||||||
@@ -601,6 +601,7 @@ const HeartbeatSchema = z
|
|||||||
z.literal("telegram"),
|
z.literal("telegram"),
|
||||||
z.literal("discord"),
|
z.literal("discord"),
|
||||||
z.literal("slack"),
|
z.literal("slack"),
|
||||||
|
z.literal("msteams"),
|
||||||
z.literal("signal"),
|
z.literal("signal"),
|
||||||
z.literal("imessage"),
|
z.literal("imessage"),
|
||||||
z.literal("none"),
|
z.literal("none"),
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ import {
|
|||||||
import { registerAgentRunContext } from "../infra/agent-events.js";
|
import { registerAgentRunContext } from "../infra/agent-events.js";
|
||||||
import { parseTelegramTarget } from "../telegram/targets.js";
|
import { parseTelegramTarget } from "../telegram/targets.js";
|
||||||
import { resolveTelegramToken } from "../telegram/token.js";
|
import { resolveTelegramToken } from "../telegram/token.js";
|
||||||
import { normalizeE164 } from "../utils.js";
|
import { normalizeE164, truncateUtf16Safe } from "../utils.js";
|
||||||
import type { CronJob } from "./types.js";
|
import type { CronJob } from "./types.js";
|
||||||
|
|
||||||
export type RunCronAgentTurnResult = {
|
export type RunCronAgentTurnResult = {
|
||||||
@@ -68,7 +68,7 @@ function pickSummaryFromOutput(text: string | undefined) {
|
|||||||
const clean = (text ?? "").trim();
|
const clean = (text ?? "").trim();
|
||||||
if (!clean) return undefined;
|
if (!clean) return undefined;
|
||||||
const limit = 2000;
|
const limit = 2000;
|
||||||
return clean.length > limit ? `${clean.slice(0, limit)}…` : clean;
|
return clean.length > limit ? `${truncateUtf16Safe(clean, limit)}…` : clean;
|
||||||
}
|
}
|
||||||
|
|
||||||
function pickSummaryFromPayloads(
|
function pickSummaryFromPayloads(
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import crypto from "node:crypto";
|
import crypto from "node:crypto";
|
||||||
|
|
||||||
|
import { truncateUtf16Safe } from "../utils.js";
|
||||||
import { computeNextRunAtMs } from "./schedule.js";
|
import { computeNextRunAtMs } from "./schedule.js";
|
||||||
import { loadCronStore, saveCronStore } from "./store.js";
|
import { loadCronStore, saveCronStore } from "./store.js";
|
||||||
import type {
|
import type {
|
||||||
@@ -61,7 +62,7 @@ function normalizeOptionalText(raw: unknown) {
|
|||||||
|
|
||||||
function truncateText(input: string, maxLen: number) {
|
function truncateText(input: string, maxLen: number) {
|
||||||
if (input.length <= maxLen) return input;
|
if (input.length <= maxLen) return input;
|
||||||
return `${input.slice(0, Math.max(0, maxLen - 1)).trimEnd()}…`;
|
return `${truncateUtf16Safe(input, Math.max(0, maxLen - 1)).trimEnd()}…`;
|
||||||
}
|
}
|
||||||
|
|
||||||
function inferLegacyName(job: {
|
function inferLegacyName(job: {
|
||||||
|
|||||||
@@ -61,6 +61,7 @@ import {
|
|||||||
} from "../routing/resolve-route.js";
|
} from "../routing/resolve-route.js";
|
||||||
import { resolveThreadSessionKeys } from "../routing/session-key.js";
|
import { resolveThreadSessionKeys } from "../routing/session-key.js";
|
||||||
import type { RuntimeEnv } from "../runtime.js";
|
import type { RuntimeEnv } from "../runtime.js";
|
||||||
|
import { truncateUtf16Safe } from "../utils.js";
|
||||||
import { loadWebMedia } from "../web/media.js";
|
import { loadWebMedia } from "../web/media.js";
|
||||||
import { resolveDiscordAccount } from "./accounts.js";
|
import { resolveDiscordAccount } from "./accounts.js";
|
||||||
import { chunkDiscordText } from "./chunk.js";
|
import { chunkDiscordText } from "./chunk.js";
|
||||||
@@ -1017,7 +1018,10 @@ export function createDiscordMessageHandler(params: {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (shouldLogVerbose()) {
|
if (shouldLogVerbose()) {
|
||||||
const preview = combinedBody.slice(0, 200).replace(/\n/g, "\\n");
|
const preview = truncateUtf16Safe(combinedBody, 200).replace(
|
||||||
|
/\n/g,
|
||||||
|
"\\n",
|
||||||
|
);
|
||||||
logVerbose(
|
logVerbose(
|
||||||
`discord inbound: channel=${message.channelId} from=${ctxPayload.From} preview="${preview}"`,
|
`discord inbound: channel=${message.channelId} from=${ctxPayload.From} preview="${preview}"`,
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ import {
|
|||||||
} from "../pairing/pairing-store.js";
|
} from "../pairing/pairing-store.js";
|
||||||
import { resolveAgentRoute } from "../routing/resolve-route.js";
|
import { resolveAgentRoute } from "../routing/resolve-route.js";
|
||||||
import type { RuntimeEnv } from "../runtime.js";
|
import type { RuntimeEnv } from "../runtime.js";
|
||||||
|
import { truncateUtf16Safe } from "../utils.js";
|
||||||
import { resolveIMessageAccount } from "./accounts.js";
|
import { resolveIMessageAccount } from "./accounts.js";
|
||||||
import { createIMessageRpcClient } from "./client.js";
|
import { createIMessageRpcClient } from "./client.js";
|
||||||
import { sendMessageIMessage } from "./send.js";
|
import { sendMessageIMessage } from "./send.js";
|
||||||
@@ -413,7 +414,7 @@ export async function monitorIMessageProvider(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (shouldLogVerbose()) {
|
if (shouldLogVerbose()) {
|
||||||
const preview = body.slice(0, 200).replace(/\n/g, "\\n");
|
const preview = truncateUtf16Safe(body, 200).replace(/\n/g, "\\n");
|
||||||
logVerbose(
|
logVerbose(
|
||||||
`imessage inbound: chatId=${chatId ?? "unknown"} from=${ctxPayload.From} len=${body.length} preview="${preview}"`,
|
`imessage inbound: chatId=${chatId ?? "unknown"} from=${ctxPayload.From} len=${body.length} preview="${preview}"`,
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -503,13 +503,19 @@ function formatConsoleLine(opts: {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function writeConsoleLine(level: Level, line: string) {
|
function writeConsoleLine(level: Level, line: string) {
|
||||||
|
const sanitized =
|
||||||
|
process.platform === "win32" && process.env.GITHUB_ACTIONS === "true"
|
||||||
|
? line
|
||||||
|
.replace(/[\uD800-\uDBFF][\uDC00-\uDFFF]/g, "?")
|
||||||
|
.replace(/[\uD800-\uDFFF]/g, "?")
|
||||||
|
: line;
|
||||||
const sink = rawConsole ?? console;
|
const sink = rawConsole ?? console;
|
||||||
if (forceConsoleToStderr || level === "error" || level === "fatal") {
|
if (forceConsoleToStderr || level === "error" || level === "fatal") {
|
||||||
(sink.error ?? console.error)(line);
|
(sink.error ?? console.error)(sanitized);
|
||||||
} else if (level === "warn") {
|
} else if (level === "warn") {
|
||||||
(sink.warn ?? console.warn)(line);
|
(sink.warn ?? console.warn)(sanitized);
|
||||||
} else {
|
} else {
|
||||||
(sink.log ?? console.log)(line);
|
(sink.log ?? console.log)(sanitized);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -55,10 +55,11 @@ export async function monitorMSTeamsProvider(
|
|||||||
const port = msteamsCfg.webhook?.port ?? 3978;
|
const port = msteamsCfg.webhook?.port ?? 3978;
|
||||||
const textLimit = resolveTextChunkLimit(cfg, "msteams");
|
const textLimit = resolveTextChunkLimit(cfg, "msteams");
|
||||||
const MB = 1024 * 1024;
|
const MB = 1024 * 1024;
|
||||||
|
const agentDefaults = cfg.agents?.defaults;
|
||||||
const mediaMaxBytes =
|
const mediaMaxBytes =
|
||||||
typeof cfg.agents?.defaults?.mediaMaxMb === "number" &&
|
typeof agentDefaults?.mediaMaxMb === "number" &&
|
||||||
cfg.agents.defaults.mediaMaxMb > 0
|
agentDefaults.mediaMaxMb > 0
|
||||||
? Math.floor(cfg.agents.defaults.mediaMaxMb * MB)
|
? Math.floor(agentDefaults.mediaMaxMb * MB)
|
||||||
: 8 * MB;
|
: 8 * MB;
|
||||||
const conversationStore =
|
const conversationStore =
|
||||||
opts.conversationStore ?? createMSTeamsConversationStoreFs();
|
opts.conversationStore ?? createMSTeamsConversationStoreFs();
|
||||||
|
|||||||
55
src/utils.ts
55
src/utils.ts
@@ -95,6 +95,61 @@ export function sleep(ms: number) {
|
|||||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isHighSurrogate(codeUnit: number): boolean {
|
||||||
|
return codeUnit >= 0xd800 && codeUnit <= 0xdbff;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isLowSurrogate(codeUnit: number): boolean {
|
||||||
|
return codeUnit >= 0xdc00 && codeUnit <= 0xdfff;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function sliceUtf16Safe(
|
||||||
|
input: string,
|
||||||
|
start: number,
|
||||||
|
end?: number,
|
||||||
|
): string {
|
||||||
|
const len = input.length;
|
||||||
|
|
||||||
|
let from = start < 0 ? Math.max(len + start, 0) : Math.min(start, len);
|
||||||
|
let to =
|
||||||
|
end === undefined
|
||||||
|
? len
|
||||||
|
: end < 0
|
||||||
|
? Math.max(len + end, 0)
|
||||||
|
: Math.min(end, len);
|
||||||
|
|
||||||
|
if (to < from) {
|
||||||
|
const tmp = from;
|
||||||
|
from = to;
|
||||||
|
to = tmp;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (from > 0 && from < len) {
|
||||||
|
const codeUnit = input.charCodeAt(from);
|
||||||
|
if (
|
||||||
|
isLowSurrogate(codeUnit) &&
|
||||||
|
isHighSurrogate(input.charCodeAt(from - 1))
|
||||||
|
) {
|
||||||
|
from += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (to > 0 && to < len) {
|
||||||
|
const codeUnit = input.charCodeAt(to - 1);
|
||||||
|
if (isHighSurrogate(codeUnit) && isLowSurrogate(input.charCodeAt(to))) {
|
||||||
|
to -= 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return input.slice(from, to);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function truncateUtf16Safe(input: string, maxLen: number): string {
|
||||||
|
const limit = Math.max(0, Math.floor(maxLen));
|
||||||
|
if (input.length <= limit) return input;
|
||||||
|
return sliceUtf16Safe(input, 0, limit);
|
||||||
|
}
|
||||||
|
|
||||||
export function resolveUserPath(input: string): string {
|
export function resolveUserPath(input: string): string {
|
||||||
const trimmed = input.trim();
|
const trimmed = input.trim();
|
||||||
if (!trimmed) return trimmed;
|
if (!trimmed) return trimmed;
|
||||||
|
|||||||
@@ -2,18 +2,26 @@ import fs from "node:fs";
|
|||||||
import os from "node:os";
|
import os from "node:os";
|
||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
|
|
||||||
|
import { installWindowsCIOutputSanitizer } from "./windows-ci-output-sanitizer";
|
||||||
|
|
||||||
|
installWindowsCIOutputSanitizer();
|
||||||
|
|
||||||
const originalHome = process.env.HOME;
|
const originalHome = process.env.HOME;
|
||||||
const originalUserProfile = process.env.USERPROFILE;
|
const originalUserProfile = process.env.USERPROFILE;
|
||||||
const originalXdgConfigHome = process.env.XDG_CONFIG_HOME;
|
const originalXdgConfigHome = process.env.XDG_CONFIG_HOME;
|
||||||
const originalXdgDataHome = process.env.XDG_DATA_HOME;
|
const originalXdgDataHome = process.env.XDG_DATA_HOME;
|
||||||
const originalXdgStateHome = process.env.XDG_STATE_HOME;
|
const originalXdgStateHome = process.env.XDG_STATE_HOME;
|
||||||
const originalXdgCacheHome = process.env.XDG_CACHE_HOME;
|
const originalXdgCacheHome = process.env.XDG_CACHE_HOME;
|
||||||
|
const originalStateDir = process.env.CLAWDBOT_STATE_DIR;
|
||||||
const originalTestHome = process.env.CLAWDBOT_TEST_HOME;
|
const originalTestHome = process.env.CLAWDBOT_TEST_HOME;
|
||||||
|
|
||||||
const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "clawdbot-test-home-"));
|
const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "clawdbot-test-home-"));
|
||||||
process.env.HOME = tempHome;
|
process.env.HOME = tempHome;
|
||||||
process.env.USERPROFILE = tempHome;
|
process.env.USERPROFILE = tempHome;
|
||||||
process.env.CLAWDBOT_TEST_HOME = tempHome;
|
process.env.CLAWDBOT_TEST_HOME = tempHome;
|
||||||
|
if (process.platform === "win32") {
|
||||||
|
process.env.CLAWDBOT_STATE_DIR = path.join(tempHome, ".clawdbot");
|
||||||
|
}
|
||||||
process.env.XDG_CONFIG_HOME = path.join(tempHome, ".config");
|
process.env.XDG_CONFIG_HOME = path.join(tempHome, ".config");
|
||||||
process.env.XDG_DATA_HOME = path.join(tempHome, ".local", "share");
|
process.env.XDG_DATA_HOME = path.join(tempHome, ".local", "share");
|
||||||
process.env.XDG_STATE_HOME = path.join(tempHome, ".local", "state");
|
process.env.XDG_STATE_HOME = path.join(tempHome, ".local", "state");
|
||||||
@@ -31,6 +39,7 @@ process.on("exit", () => {
|
|||||||
restoreEnv("XDG_DATA_HOME", originalXdgDataHome);
|
restoreEnv("XDG_DATA_HOME", originalXdgDataHome);
|
||||||
restoreEnv("XDG_STATE_HOME", originalXdgStateHome);
|
restoreEnv("XDG_STATE_HOME", originalXdgStateHome);
|
||||||
restoreEnv("XDG_CACHE_HOME", originalXdgCacheHome);
|
restoreEnv("XDG_CACHE_HOME", originalXdgCacheHome);
|
||||||
|
restoreEnv("CLAWDBOT_STATE_DIR", originalStateDir);
|
||||||
restoreEnv("CLAWDBOT_TEST_HOME", originalTestHome);
|
restoreEnv("CLAWDBOT_TEST_HOME", originalTestHome);
|
||||||
try {
|
try {
|
||||||
fs.rmSync(tempHome, { recursive: true, force: true });
|
fs.rmSync(tempHome, { recursive: true, force: true });
|
||||||
|
|||||||
5
test/vitest-global-setup.ts
Normal file
5
test/vitest-global-setup.ts
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
import { installWindowsCIOutputSanitizer } from "./windows-ci-output-sanitizer";
|
||||||
|
|
||||||
|
export default function globalSetup() {
|
||||||
|
installWindowsCIOutputSanitizer();
|
||||||
|
}
|
||||||
59
test/windows-ci-output-sanitizer.ts
Normal file
59
test/windows-ci-output-sanitizer.ts
Normal file
@@ -0,0 +1,59 @@
|
|||||||
|
import fs from "node:fs";
|
||||||
|
|
||||||
|
function sanitizeWindowsCIOutput(text: string): string {
|
||||||
|
return text
|
||||||
|
.replace(/[\uD800-\uDBFF][\uDC00-\uDFFF]/g, "?")
|
||||||
|
.replace(/[\uD800-\uDFFF]/g, "?");
|
||||||
|
}
|
||||||
|
|
||||||
|
function decodeUtf8Text(chunk: unknown): string | null {
|
||||||
|
if (typeof chunk === "string") return chunk;
|
||||||
|
if (Buffer.isBuffer(chunk)) return chunk.toString("utf-8");
|
||||||
|
if (chunk instanceof Uint8Array) return Buffer.from(chunk).toString("utf-8");
|
||||||
|
if (chunk instanceof ArrayBuffer) return Buffer.from(chunk).toString("utf-8");
|
||||||
|
if (ArrayBuffer.isView(chunk)) {
|
||||||
|
return Buffer.from(
|
||||||
|
chunk.buffer,
|
||||||
|
chunk.byteOffset,
|
||||||
|
chunk.byteLength,
|
||||||
|
).toString("utf-8");
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function installWindowsCIOutputSanitizer(): void {
|
||||||
|
if (process.platform !== "win32") return;
|
||||||
|
if (process.env.GITHUB_ACTIONS !== "true") return;
|
||||||
|
|
||||||
|
const globalKey = "__clawdbotWindowsCIOutputSanitizerInstalled";
|
||||||
|
if ((globalThis as Record<string, unknown>)[globalKey] === true) return;
|
||||||
|
(globalThis as Record<string, unknown>)[globalKey] = true;
|
||||||
|
|
||||||
|
const originalStdoutWrite = process.stdout.write.bind(process.stdout);
|
||||||
|
const originalStderrWrite = process.stderr.write.bind(process.stderr);
|
||||||
|
|
||||||
|
process.stdout.write = ((chunk: unknown, ...args: unknown[]) => {
|
||||||
|
const text = decodeUtf8Text(chunk);
|
||||||
|
if (text !== null)
|
||||||
|
return originalStdoutWrite(sanitizeWindowsCIOutput(text), ...args);
|
||||||
|
return originalStdoutWrite(chunk as never, ...args); // passthrough
|
||||||
|
}) as typeof process.stdout.write;
|
||||||
|
|
||||||
|
process.stderr.write = ((chunk: unknown, ...args: unknown[]) => {
|
||||||
|
const text = decodeUtf8Text(chunk);
|
||||||
|
if (text !== null)
|
||||||
|
return originalStderrWrite(sanitizeWindowsCIOutput(text), ...args);
|
||||||
|
return originalStderrWrite(chunk as never, ...args); // passthrough
|
||||||
|
}) as typeof process.stderr.write;
|
||||||
|
|
||||||
|
const originalWriteSync = fs.writeSync.bind(fs);
|
||||||
|
fs.writeSync = ((fd: number, data: unknown, ...args: unknown[]) => {
|
||||||
|
if (fd === 1 || fd === 2) {
|
||||||
|
const text = decodeUtf8Text(data);
|
||||||
|
if (text !== null) {
|
||||||
|
return originalWriteSync(fd, sanitizeWindowsCIOutput(text), ...args);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return originalWriteSync(fd, data as never, ...(args as never[]));
|
||||||
|
}) as typeof fs.writeSync;
|
||||||
|
}
|
||||||
@@ -4,6 +4,7 @@ export default defineConfig({
|
|||||||
test: {
|
test: {
|
||||||
include: ["src/**/*.test.ts", "test/format-error.test.ts"],
|
include: ["src/**/*.test.ts", "test/format-error.test.ts"],
|
||||||
setupFiles: ["test/setup.ts"],
|
setupFiles: ["test/setup.ts"],
|
||||||
|
globalSetup: ["test/vitest-global-setup.ts"],
|
||||||
exclude: [
|
exclude: [
|
||||||
"dist/**",
|
"dist/**",
|
||||||
"apps/macos/**",
|
"apps/macos/**",
|
||||||
|
|||||||
Reference in New Issue
Block a user