diff --git a/scripts/zai-fallback-repro.ts b/scripts/zai-fallback-repro.ts new file mode 100644 index 000000000..6c8aa1211 --- /dev/null +++ b/scripts/zai-fallback-repro.ts @@ -0,0 +1,185 @@ +import { randomUUID } from "node:crypto"; +import { spawn } from "node:child_process"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +type RunResult = { + code: number | null; + signal: NodeJS.Signals | null; + stdout: string; + stderr: string; +}; + +function pickAnthropicEnv(): { type: "oauth" | "api"; value: string } | null { + const oauth = process.env.ANTHROPIC_OAUTH_TOKEN?.trim(); + if (oauth) return { type: "oauth", value: oauth }; + const api = process.env.ANTHROPIC_API_KEY?.trim(); + if (api) return { type: "api", value: api }; + return null; +} + +function pickZaiKey(): string | null { + return ( + process.env.ZAI_API_KEY?.trim() ?? process.env.Z_AI_API_KEY?.trim() ?? null + ); +} + +async function runCommand( + label: string, + args: string[], + env: NodeJS.ProcessEnv, +): Promise { + return await new Promise((resolve, reject) => { + const child = spawn("pnpm", args, { + env, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + const text = String(chunk); + stdout += text; + process.stdout.write(text); + }); + child.stderr.on("data", (chunk) => { + const text = String(chunk); + stderr += text; + process.stderr.write(text); + }); + child.on("error", (err) => reject(err)); + child.on("close", (code, signal) => { + if (code === 0) { + resolve({ code, signal, stdout, stderr }); + return; + } + resolve({ code, signal, stdout, stderr }); + const summary = signal + ? `${label} exited with signal ${signal}` + : `${label} exited with code ${code}`; + console.error(summary); + }); + }); +} + +async function main() { + const anthropic = pickAnthropicEnv(); + const zaiKey = pickZaiKey(); + if (!anthropic) { + console.error("Missing ANTHROPIC_OAUTH_TOKEN or ANTHROPIC_API_KEY."); + process.exit(1); + } + if (!zaiKey) { + console.error("Missing ZAI_API_KEY or Z_AI_API_KEY."); + process.exit(1); + } + + const baseDir = await fs.mkdtemp( + path.join(os.tmpdir(), "clawdbot-zai-fallback-"), + ); + const stateDir = path.join(baseDir, "state"); + const configPath = path.join(baseDir, "clawdbot.json"); + await fs.mkdir(stateDir, { recursive: true }); + + const config = { + agents: { + defaults: { + model: { + primary: "anthropic/claude-opus-4-5", + fallbacks: ["zai/glm-4.7"], + }, + models: { + "anthropic/claude-opus-4-5": {}, + "zai/glm-4.7": {}, + }, + }, + }, + }; + await fs.writeFile(configPath, JSON.stringify(config, null, 2), "utf8"); + + const sessionId = + process.env.CLAWDBOT_ZAI_FALLBACK_SESSION_ID ?? randomUUID(); + + const baseEnv: NodeJS.ProcessEnv = { + ...process.env, + CLAWDBOT_CONFIG_PATH: configPath, + CLAWDBOT_STATE_DIR: stateDir, + ZAI_API_KEY: zaiKey, + Z_AI_API_KEY: "", + }; + + const envValidAnthropic: NodeJS.ProcessEnv = { + ...baseEnv, + ANTHROPIC_OAUTH_TOKEN: anthropic.type === "oauth" ? anthropic.value : "", + ANTHROPIC_API_KEY: anthropic.type === "api" ? anthropic.value : "", + }; + + const envInvalidAnthropic: NodeJS.ProcessEnv = { + ...baseEnv, + ANTHROPIC_OAUTH_TOKEN: anthropic.type === "oauth" ? "invalid" : "", + ANTHROPIC_API_KEY: anthropic.type === "api" ? "invalid" : "", + }; + + console.log("== Run 1: create tool history (primary only)"); + const toolPrompt = + "Use the bash tool to create a file named zai-fallback-tool.txt with the content tool-ok. " + + "Then use the read tool to display the file contents. Reply with just the file contents."; + const run1 = await runCommand( + "run1", + [ + "clawdbot", + "agent", + "--local", + "--session-id", + sessionId, + "--message", + toolPrompt, + ], + envValidAnthropic, + ); + if (run1.code !== 0) { + process.exit(run1.code ?? 1); + } + + const sessionFile = path.join( + stateDir, + "agents", + "main", + "sessions", + `${sessionId}.jsonl`, + ); + const transcript = await fs.readFile(sessionFile, "utf8").catch(() => ""); + if (!transcript.includes('"toolResult"')) { + console.warn("Warning: no toolResult entries detected in session history."); + } + + console.log("== Run 2: force auth failover to Z.AI"); + const followupPrompt = + "What is the content of zai-fallback-tool.txt? Reply with just the contents."; + const run2 = await runCommand( + "run2", + [ + "clawdbot", + "agent", + "--local", + "--session-id", + sessionId, + "--message", + followupPrompt, + ], + envInvalidAnthropic, + ); + + if (run2.code === 0) { + console.log("PASS: fallback succeeded."); + process.exit(0); + } + + console.error("FAIL: fallback failed."); + process.exit(run2.code ?? 1); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/src/agents/model-compat.test.ts b/src/agents/model-compat.test.ts new file mode 100644 index 000000000..ce6d12d4b --- /dev/null +++ b/src/agents/model-compat.test.ts @@ -0,0 +1,44 @@ +import type { Api, Model } from "@mariozechner/pi-ai"; +import { describe, expect, it } from "vitest"; +import { normalizeModelCompat } from "./model-compat.js"; + +const baseModel = (): Model => + ({ + id: "glm-4.7", + name: "GLM-4.7", + api: "openai-completions", + provider: "zai", + baseUrl: "https://api.z.ai/api/coding/paas/v4", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 8192, + maxTokens: 1024, + }) as Model; + +describe("normalizeModelCompat", () => { + it("forces supportsDeveloperRole off for z.ai models", () => { + const model = baseModel(); + delete (model as { compat?: unknown }).compat; + const normalized = normalizeModelCompat(model); + expect(normalized.compat?.supportsDeveloperRole).toBe(false); + }); + + it("leaves non-zai models untouched", () => { + const model = { + ...baseModel(), + provider: "openai", + baseUrl: "https://api.openai.com/v1", + }; + delete (model as { compat?: unknown }).compat; + const normalized = normalizeModelCompat(model); + expect(normalized.compat).toBeUndefined(); + }); + + it("does not override explicit z.ai compat false", () => { + const model = baseModel(); + model.compat = { supportsDeveloperRole: false }; + const normalized = normalizeModelCompat(model); + expect(normalized.compat?.supportsDeveloperRole).toBe(false); + }); +}); diff --git a/src/agents/model-compat.ts b/src/agents/model-compat.ts new file mode 100644 index 000000000..ce08f33fa --- /dev/null +++ b/src/agents/model-compat.ts @@ -0,0 +1,13 @@ +import type { Api, Model } from "@mariozechner/pi-ai"; + +export function normalizeModelCompat(model: Model): Model { + const baseUrl = model.baseUrl ?? ""; + const isZai = model.provider === "zai" || baseUrl.includes("api.z.ai"); + if (!isZai) return model; + + const compat = model.compat ?? {}; + if (compat.supportsDeveloperRole === false) return model; + + model.compat = { ...compat, supportsDeveloperRole: false }; + return model; +} diff --git a/src/agents/pi-embedded-runner.ts b/src/agents/pi-embedded-runner.ts index 47b4905a8..69b788f8d 100644 --- a/src/agents/pi-embedded-runner.ts +++ b/src/agents/pi-embedded-runner.ts @@ -66,6 +66,7 @@ import { resolveAuthProfileOrder, resolveModelAuthMode, } from "./model-auth.js"; +import { normalizeModelCompat } from "./model-compat.js"; import { ensureClawdbotModelsJson } from "./models-config.js"; import type { MessagingToolSend } from "./pi-embedded-messaging.js"; import { acquireSessionWriteLock } from "./session-write-lock.js"; @@ -762,7 +763,7 @@ function resolveModel( modelRegistry, }; } - return { model, authStorage, modelRegistry }; + return { model: normalizeModelCompat(model), authStorage, modelRegistry }; } export async function compactEmbeddedPiSession(params: { diff --git a/src/gateway/gateway-models.profiles.live.test.ts b/src/gateway/gateway-models.profiles.live.test.ts index a4d76704a..49c187260 100644 --- a/src/gateway/gateway-models.profiles.live.test.ts +++ b/src/gateway/gateway-models.profiles.live.test.ts @@ -27,6 +27,7 @@ const ALL_MODELS = const EXTRA_TOOL_PROBES = process.env.CLAWDBOT_LIVE_GATEWAY_TOOL_PROBE === "1"; const EXTRA_IMAGE_PROBES = process.env.CLAWDBOT_LIVE_GATEWAY_IMAGE_PROBE === "1"; +const ZAI_FALLBACK = process.env.CLAWDBOT_LIVE_GATEWAY_ZAI_FALLBACK === "1"; const PROVIDERS = parseFilter(process.env.CLAWDBOT_LIVE_GATEWAY_PROVIDERS); const describeLive = LIVE && GATEWAY_LIVE ? describe : describe.skip; @@ -559,4 +560,142 @@ describeLive("gateway live (dev agent, profile keys)", () => { }, 20 * 60 * 1000, ); + + it("z.ai fallback handles anthropic tool history", async () => { + if (!ZAI_FALLBACK) return; + const previous = { + configPath: process.env.CLAWDBOT_CONFIG_PATH, + token: process.env.CLAWDBOT_GATEWAY_TOKEN, + skipProviders: process.env.CLAWDBOT_SKIP_PROVIDERS, + skipGmail: process.env.CLAWDBOT_SKIP_GMAIL_WATCHER, + skipCron: process.env.CLAWDBOT_SKIP_CRON, + skipCanvas: process.env.CLAWDBOT_SKIP_CANVAS_HOST, + }; + + process.env.CLAWDBOT_SKIP_PROVIDERS = "1"; + process.env.CLAWDBOT_SKIP_GMAIL_WATCHER = "1"; + process.env.CLAWDBOT_SKIP_CRON = "1"; + process.env.CLAWDBOT_SKIP_CANVAS_HOST = "1"; + + const token = `test-${randomUUID()}`; + process.env.CLAWDBOT_GATEWAY_TOKEN = token; + + const cfg = loadConfig(); + await ensureClawdbotModelsJson(cfg); + + const agentDir = resolveClawdbotAgentDir(); + const authStorage = discoverAuthStorage(agentDir); + const modelRegistry = discoverModels(authStorage, agentDir); + const anthropic = modelRegistry.find( + "anthropic", + "claude-opus-4-5", + ) as Model | null; + const zai = modelRegistry.find("zai", "glm-4.7") as Model | null; + + if (!anthropic || !zai) return; + try { + await getApiKeyForModel({ model: anthropic, cfg }); + await getApiKeyForModel({ model: zai, cfg }); + } catch { + return; + } + + const workspaceDir = resolveUserPath( + cfg.agents?.defaults?.workspace ?? path.join(os.homedir(), "clawd"), + ); + await fs.mkdir(workspaceDir, { recursive: true }); + const nonceA = randomUUID(); + const nonceB = randomUUID(); + const toolProbePath = path.join( + workspaceDir, + `.clawdbot-live-zai-fallback.${nonceA}.txt`, + ); + await fs.writeFile(toolProbePath, `nonceA=${nonceA}\nnonceB=${nonceB}\n`); + + const port = await getFreeGatewayPort(); + const server = await startGatewayServer({ + configPath: cfg.__meta?.path, + port, + token, + }); + + const client = await connectClient({ + url: `ws://127.0.0.1:${port}`, + token, + }); + + try { + const sessionKey = "agent:dev:live-zai-fallback"; + + await client.request>("sessions.patch", { + key: sessionKey, + model: "anthropic/claude-opus-4-5", + }); + await client.request>("sessions.reset", { + key: sessionKey, + }); + + const runId = randomUUID(); + const toolProbe = await client.request( + "agent", + { + sessionKey, + idempotencyKey: `idem-${runId}-tool`, + message: + `Call the tool named \`read\` (or \`Read\` if \`read\` is unavailable) with JSON arguments {"path":"${toolProbePath}"}. ` + + `Then reply with exactly: ${nonceA} ${nonceB}. No extra text.`, + deliver: false, + }, + { expectFinal: true }, + ); + if (toolProbe?.status !== "ok") { + throw new Error( + `anthropic tool probe failed: status=${String(toolProbe?.status)}`, + ); + } + const toolText = extractPayloadText(toolProbe?.result); + if (!toolText.includes(nonceA) || !toolText.includes(nonceB)) { + throw new Error(`anthropic tool probe missing nonce: ${toolText}`); + } + + await client.request>("sessions.patch", { + key: sessionKey, + model: "zai/glm-4.7", + }); + + const followupId = randomUUID(); + const followup = await client.request( + "agent", + { + sessionKey, + idempotencyKey: `idem-${followupId}-followup`, + message: + `What are the values of nonceA and nonceB in "${toolProbePath}"? ` + + `Reply with exactly: ${nonceA} ${nonceB}.`, + deliver: false, + }, + { expectFinal: true }, + ); + if (followup?.status !== "ok") { + throw new Error( + `zai followup failed: status=${String(followup?.status)}`, + ); + } + const followupText = extractPayloadText(followup?.result); + if (!followupText.includes(nonceA) || !followupText.includes(nonceB)) { + throw new Error(`zai followup missing nonce: ${followupText}`); + } + } finally { + client.stop(); + await server.close({ reason: "live test complete" }); + await fs.rm(toolProbePath, { force: true }); + + process.env.CLAWDBOT_CONFIG_PATH = previous.configPath; + process.env.CLAWDBOT_GATEWAY_TOKEN = previous.token; + process.env.CLAWDBOT_SKIP_PROVIDERS = previous.skipProviders; + process.env.CLAWDBOT_SKIP_GMAIL_WATCHER = previous.skipGmail; + process.env.CLAWDBOT_SKIP_CRON = previous.skipCron; + process.env.CLAWDBOT_SKIP_CANVAS_HOST = previous.skipCanvas; + } + }, 180_000); });