diff --git a/CHANGELOG.md b/CHANGELOG.md index d4a7ac352..3ea1fd8c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Plugins: voice-call plugin now real (Twilio/log), adds start/status RPC/CLI/tool + tests. - Docs: add plugins doc + cross-links from tools/skills/gateway config. - Tests: add Docker plugin loader smoke test. +- Config: add `$include` directive for modular config files. (#731) — thanks @pasogott. - Build: set pnpm minimum release age to 2880 minutes (2 days). (#718) — thanks @dan-dr. - macOS: prompt to install the global `clawdbot` CLI when missing in local mode; install via `clawd.bot/install-cli.sh` (no onboarding) and use external launchd/CLI instead of the embedded gateway runtime. - Docs: add gog calendar event color IDs from `gog calendar colors`. (#715) — thanks @mjrussell. diff --git a/docs/gateway/configuration.md b/docs/gateway/configuration.md index 162d1c43e..06ceec2f7 100644 --- a/docs/gateway/configuration.md +++ b/docs/gateway/configuration.md @@ -82,6 +82,132 @@ To prevent the bot from responding to WhatsApp @-mentions in groups (only respon } ``` +## Config Includes (`$include`) + +Split your config into multiple files using the `$include` directive. This is useful for: +- Organizing large configs (e.g., per-client agent definitions) +- Sharing common settings across environments +- Keeping sensitive configs separate + +### Basic usage + +```json5 +// ~/.clawdbot/clawdbot.json +{ + gateway: { port: 18789 }, + + // Include a single file (replaces the key's value) + agents: { "$include": "./agents.json5" }, + + // Include multiple files (deep-merged in order) + broadcast: { + "$include": [ + "./clients/mueller.json5", + "./clients/schmidt.json5" + ] + } +} +``` + +```json5 +// ~/.clawdbot/agents.json5 +{ + defaults: { sandbox: { mode: "all", scope: "session" } }, + list: [ + { id: "main", workspace: "~/clawd" } + ] +} +``` + +### Merge behavior + +- **Single file**: Replaces the object containing `$include` +- **Array of files**: Deep-merges files in order (later files override earlier ones) +- **With sibling keys**: Sibling keys are merged after includes (override included values) +- **Sibling keys + arrays/primitives**: Not supported (included content must be an object) + +```json5 +// Sibling keys override included values +{ + "$include": "./base.json5", // { a: 1, b: 2 } + b: 99 // Result: { a: 1, b: 99 } +} +``` + +### Nested includes + +Included files can themselves contain `$include` directives (up to 10 levels deep): + +```json5 +// clients/mueller.json5 +{ + agents: { "$include": "./mueller/agents.json5" }, + broadcast: { "$include": "./mueller/broadcast.json5" } +} +``` + +### Path resolution + +- **Relative paths**: Resolved relative to the including file +- **Absolute paths**: Used as-is +- **Parent directories**: `../` references work as expected + +```json5 +{ "$include": "./sub/config.json5" } // relative +{ "$include": "/etc/clawdbot/base.json5" } // absolute +{ "$include": "../shared/common.json5" } // parent dir +``` + +### Error handling + +- **Missing file**: Clear error with resolved path +- **Parse error**: Shows which included file failed +- **Circular includes**: Detected and reported with include chain + +### Example: Multi-client legal setup + +```json5 +// ~/.clawdbot/clawdbot.json +{ + gateway: { port: 18789, auth: { token: "secret" } }, + + // Common agent defaults + agents: { + defaults: { + sandbox: { mode: "all", scope: "session" } + }, + // Merge agent lists from all clients + list: { "$include": [ + "./clients/mueller/agents.json5", + "./clients/schmidt/agents.json5" + ]} + }, + + // Merge broadcast configs + broadcast: { "$include": [ + "./clients/mueller/broadcast.json5", + "./clients/schmidt/broadcast.json5" + ]}, + + whatsapp: { groupPolicy: "allowlist" } +} +``` + +```json5 +// ~/.clawdbot/clients/mueller/agents.json5 +[ + { id: "mueller-transcribe", workspace: "~/clients/mueller/transcribe" }, + { id: "mueller-docs", workspace: "~/clients/mueller/docs" } +] +``` + +```json5 +// ~/.clawdbot/clients/mueller/broadcast.json5 +{ + "120363403215116621@g.us": ["mueller-transcribe", "mueller-docs"] +} +``` + ## Common options ### Env vars + `.env` diff --git a/src/config/includes.test.ts b/src/config/includes.test.ts new file mode 100644 index 000000000..f8f85a8c6 --- /dev/null +++ b/src/config/includes.test.ts @@ -0,0 +1,280 @@ +import { describe, expect, it } from "vitest"; + +import { + CircularIncludeError, + ConfigIncludeError, + type IncludeResolver, + resolveConfigIncludes, +} from "./includes.js"; + +function createMockResolver(files: Record): IncludeResolver { + return { + readFile: (filePath: string) => { + if (filePath in files) { + return JSON.stringify(files[filePath]); + } + throw new Error(`ENOENT: no such file: ${filePath}`); + }, + parseJson: JSON.parse, + }; +} + +function resolve( + obj: unknown, + files: Record = {}, + basePath = "/config/clawdbot.json", +) { + return resolveConfigIncludes(obj, basePath, createMockResolver(files)); +} + +describe("resolveConfigIncludes", () => { + it("passes through primitives unchanged", () => { + expect(resolve("hello")).toBe("hello"); + expect(resolve(42)).toBe(42); + expect(resolve(true)).toBe(true); + expect(resolve(null)).toBe(null); + }); + + it("passes through arrays with recursion", () => { + expect(resolve([1, 2, { a: 1 }])).toEqual([1, 2, { a: 1 }]); + }); + + it("passes through objects without $include", () => { + const obj = { foo: "bar", nested: { x: 1 } }; + expect(resolve(obj)).toEqual(obj); + }); + + it("resolves single file $include", () => { + const files = { "/config/agents.json": { list: [{ id: "main" }] } }; + const obj = { agents: { $include: "./agents.json" } }; + expect(resolve(obj, files)).toEqual({ + agents: { list: [{ id: "main" }] }, + }); + }); + + it("resolves absolute path $include", () => { + const files = { "/etc/clawdbot/agents.json": { list: [{ id: "main" }] } }; + const obj = { agents: { $include: "/etc/clawdbot/agents.json" } }; + expect(resolve(obj, files)).toEqual({ + agents: { list: [{ id: "main" }] }, + }); + }); + + it("resolves array $include with deep merge", () => { + const files = { + "/config/a.json": { "group-a": ["agent1"] }, + "/config/b.json": { "group-b": ["agent2"] }, + }; + const obj = { broadcast: { $include: ["./a.json", "./b.json"] } }; + expect(resolve(obj, files)).toEqual({ + broadcast: { + "group-a": ["agent1"], + "group-b": ["agent2"], + }, + }); + }); + + it("deep merges overlapping keys in array $include", () => { + const files = { + "/config/a.json": { agents: { defaults: { workspace: "~/a" } } }, + "/config/b.json": { agents: { list: [{ id: "main" }] } }, + }; + const obj = { $include: ["./a.json", "./b.json"] }; + expect(resolve(obj, files)).toEqual({ + agents: { + defaults: { workspace: "~/a" }, + list: [{ id: "main" }], + }, + }); + }); + + it("merges $include with sibling keys", () => { + const files = { "/config/base.json": { a: 1, b: 2 } }; + const obj = { $include: "./base.json", c: 3 }; + expect(resolve(obj, files)).toEqual({ a: 1, b: 2, c: 3 }); + }); + + it("sibling keys override included values", () => { + const files = { "/config/base.json": { a: 1, b: 2 } }; + const obj = { $include: "./base.json", b: 99 }; + expect(resolve(obj, files)).toEqual({ a: 1, b: 99 }); + }); + + it("throws when sibling keys are used with non-object includes", () => { + const files = { "/config/list.json": ["a", "b"] }; + const obj = { $include: "./list.json", extra: true }; + expect(() => resolve(obj, files)).toThrow(ConfigIncludeError); + expect(() => resolve(obj, files)).toThrow( + /Sibling keys require included content to be an object/, + ); + }); + + it("resolves nested includes", () => { + const files = { + "/config/level1.json": { nested: { $include: "./level2.json" } }, + "/config/level2.json": { deep: "value" }, + }; + const obj = { $include: "./level1.json" }; + expect(resolve(obj, files)).toEqual({ + nested: { deep: "value" }, + }); + }); + + it("throws ConfigIncludeError for missing file", () => { + const obj = { $include: "./missing.json" }; + expect(() => resolve(obj)).toThrow(ConfigIncludeError); + expect(() => resolve(obj)).toThrow(/Failed to read include file/); + }); + + it("throws ConfigIncludeError for invalid JSON", () => { + const resolver: IncludeResolver = { + readFile: () => "{ invalid json }", + parseJson: JSON.parse, + }; + const obj = { $include: "./bad.json" }; + expect(() => + resolveConfigIncludes(obj, "/config/clawdbot.json", resolver), + ).toThrow(ConfigIncludeError); + expect(() => + resolveConfigIncludes(obj, "/config/clawdbot.json", resolver), + ).toThrow(/Failed to parse include file/); + }); + + it("throws CircularIncludeError for circular includes", () => { + const resolver: IncludeResolver = { + readFile: (filePath: string) => { + if (filePath === "/config/a.json") { + return JSON.stringify({ $include: "./b.json" }); + } + if (filePath === "/config/b.json") { + return JSON.stringify({ $include: "./a.json" }); + } + throw new Error(`Unknown file: ${filePath}`); + }, + parseJson: JSON.parse, + }; + const obj = { $include: "./a.json" }; + expect(() => + resolveConfigIncludes(obj, "/config/clawdbot.json", resolver), + ).toThrow(CircularIncludeError); + expect(() => + resolveConfigIncludes(obj, "/config/clawdbot.json", resolver), + ).toThrow(/Circular include detected/); + }); + + it("throws ConfigIncludeError for invalid $include value type", () => { + const obj = { $include: 123 }; + expect(() => resolve(obj)).toThrow(ConfigIncludeError); + expect(() => resolve(obj)).toThrow(/expected string or array/); + }); + + it("throws ConfigIncludeError for invalid array item type", () => { + const files = { "/config/valid.json": { valid: true } }; + const obj = { $include: ["./valid.json", 123] }; + expect(() => resolve(obj, files)).toThrow(ConfigIncludeError); + expect(() => resolve(obj, files)).toThrow(/expected string, got number/); + }); + + it("respects max depth limit", () => { + const files: Record = {}; + for (let i = 0; i < 15; i++) { + files[`/config/level${i}.json`] = { $include: `./level${i + 1}.json` }; + } + files["/config/level15.json"] = { done: true }; + + const obj = { $include: "./level0.json" }; + expect(() => resolve(obj, files)).toThrow(ConfigIncludeError); + expect(() => resolve(obj, files)).toThrow(/Maximum include depth/); + }); + + it("handles relative paths correctly", () => { + const files = { "/config/clients/mueller/agents.json": { id: "mueller" } }; + const obj = { agent: { $include: "./clients/mueller/agents.json" } }; + expect(resolve(obj, files)).toEqual({ + agent: { id: "mueller" }, + }); + }); + + it("resolves parent directory references", () => { + const files = { "/shared/common.json": { shared: true } }; + const obj = { $include: "../../shared/common.json" }; + expect(resolve(obj, files, "/config/sub/clawdbot.json")).toEqual({ + shared: true, + }); + }); +}); + +describe("real-world config patterns", () => { + it("supports per-client agent includes", () => { + const files = { + "/config/clients/mueller.json": { + agents: [ + { + id: "mueller-screenshot", + workspace: "~/clients/mueller/screenshot", + }, + { + id: "mueller-transcribe", + workspace: "~/clients/mueller/transcribe", + }, + ], + broadcast: { + "group-mueller": ["mueller-screenshot", "mueller-transcribe"], + }, + }, + "/config/clients/schmidt.json": { + agents: [ + { + id: "schmidt-screenshot", + workspace: "~/clients/schmidt/screenshot", + }, + ], + broadcast: { "group-schmidt": ["schmidt-screenshot"] }, + }, + }; + + const obj = { + gateway: { port: 18789 }, + $include: ["./clients/mueller.json", "./clients/schmidt.json"], + }; + + expect(resolve(obj, files)).toEqual({ + gateway: { port: 18789 }, + agents: [ + { id: "mueller-screenshot", workspace: "~/clients/mueller/screenshot" }, + { id: "mueller-transcribe", workspace: "~/clients/mueller/transcribe" }, + { id: "schmidt-screenshot", workspace: "~/clients/schmidt/screenshot" }, + ], + broadcast: { + "group-mueller": ["mueller-screenshot", "mueller-transcribe"], + "group-schmidt": ["schmidt-screenshot"], + }, + }); + }); + + it("supports modular config structure", () => { + const files = { + "/config/gateway.json": { gateway: { port: 18789, bind: "loopback" } }, + "/config/providers/whatsapp.json": { + whatsapp: { dmPolicy: "pairing", allowFrom: ["+49123"] }, + }, + "/config/agents/defaults.json": { + agents: { defaults: { sandbox: { mode: "all" } } }, + }, + }; + + const obj = { + $include: [ + "./gateway.json", + "./providers/whatsapp.json", + "./agents/defaults.json", + ], + }; + + expect(resolve(obj, files)).toEqual({ + gateway: { port: 18789, bind: "loopback" }, + whatsapp: { dmPolicy: "pairing", allowFrom: ["+49123"] }, + agents: { defaults: { sandbox: { mode: "all" } } }, + }); + }); +}); diff --git a/src/config/includes.ts b/src/config/includes.ts new file mode 100644 index 000000000..3fa4f0185 --- /dev/null +++ b/src/config/includes.ts @@ -0,0 +1,258 @@ +/** + * Config includes: $include directive for modular configs + * + * @example + * ```json5 + * { + * "$include": "./base.json5", // single file + * "$include": ["./a.json5", "./b.json5"] // merge multiple + * } + * ``` + */ + +import fs from "node:fs"; +import path from "node:path"; + +import JSON5 from "json5"; + +export const INCLUDE_KEY = "$include"; +export const MAX_INCLUDE_DEPTH = 10; + +// ============================================================================ +// Types +// ============================================================================ + +export type IncludeResolver = { + readFile: (path: string) => string; + parseJson: (raw: string) => unknown; +}; + +// ============================================================================ +// Errors +// ============================================================================ + +export class ConfigIncludeError extends Error { + constructor( + message: string, + public readonly includePath: string, + public readonly cause?: Error, + ) { + super(message); + this.name = "ConfigIncludeError"; + } +} + +export class CircularIncludeError extends ConfigIncludeError { + constructor(public readonly chain: string[]) { + super( + `Circular include detected: ${chain.join(" -> ")}`, + chain[chain.length - 1], + ); + this.name = "CircularIncludeError"; + } +} + +// ============================================================================ +// Utilities +// ============================================================================ + +function isPlainObject(value: unknown): value is Record { + return ( + typeof value === "object" && + value !== null && + !Array.isArray(value) && + Object.prototype.toString.call(value) === "[object Object]" + ); +} + +/** Deep merge: arrays concatenate, objects merge recursively, primitives: source wins */ +export function deepMerge(target: unknown, source: unknown): unknown { + if (Array.isArray(target) && Array.isArray(source)) { + return [...target, ...source]; + } + if (isPlainObject(target) && isPlainObject(source)) { + const result: Record = { ...target }; + for (const key of Object.keys(source)) { + result[key] = + key in result ? deepMerge(result[key], source[key]) : source[key]; + } + return result; + } + return source; +} + +// ============================================================================ +// Include Resolver Class +// ============================================================================ + +class IncludeProcessor { + private visited = new Set(); + private depth = 0; + + constructor( + private basePath: string, + private resolver: IncludeResolver, + ) { + this.visited.add(path.normalize(basePath)); + } + + process(obj: unknown): unknown { + if (Array.isArray(obj)) { + return obj.map((item) => this.process(item)); + } + + if (!isPlainObject(obj)) { + return obj; + } + + if (!(INCLUDE_KEY in obj)) { + return this.processObject(obj); + } + + return this.processInclude(obj); + } + + private processObject(obj: Record): Record { + const result: Record = {}; + for (const [key, value] of Object.entries(obj)) { + result[key] = this.process(value); + } + return result; + } + + private processInclude(obj: Record): unknown { + const includeValue = obj[INCLUDE_KEY]; + const otherKeys = Object.keys(obj).filter((k) => k !== INCLUDE_KEY); + const included = this.resolveInclude(includeValue); + + if (otherKeys.length === 0) { + return included; + } + + if (!isPlainObject(included)) { + throw new ConfigIncludeError( + "Sibling keys require included content to be an object", + typeof includeValue === "string" ? includeValue : INCLUDE_KEY, + ); + } + + // Merge included content with sibling keys + const rest: Record = {}; + for (const key of otherKeys) { + rest[key] = this.process(obj[key]); + } + return deepMerge(included, rest); + } + + private resolveInclude(value: unknown): unknown { + if (typeof value === "string") { + return this.loadFile(value); + } + + if (Array.isArray(value)) { + return value.reduce((merged, item) => { + if (typeof item !== "string") { + throw new ConfigIncludeError( + `Invalid $include array item: expected string, got ${typeof item}`, + String(item), + ); + } + return deepMerge(merged, this.loadFile(item)); + }, {}); + } + + throw new ConfigIncludeError( + `Invalid $include value: expected string or array of strings, got ${typeof value}`, + String(value), + ); + } + + private loadFile(includePath: string): unknown { + const resolvedPath = this.resolvePath(includePath); + + this.checkCircular(resolvedPath); + this.checkDepth(includePath); + + const raw = this.readFile(includePath, resolvedPath); + const parsed = this.parseFile(includePath, resolvedPath, raw); + + return this.processNested(resolvedPath, parsed); + } + + private resolvePath(includePath: string): string { + const resolved = path.isAbsolute(includePath) + ? includePath + : path.resolve(path.dirname(this.basePath), includePath); + return path.normalize(resolved); + } + + private checkCircular(resolvedPath: string): void { + if (this.visited.has(resolvedPath)) { + throw new CircularIncludeError([...this.visited, resolvedPath]); + } + } + + private checkDepth(includePath: string): void { + if (this.depth >= MAX_INCLUDE_DEPTH) { + throw new ConfigIncludeError( + `Maximum include depth (${MAX_INCLUDE_DEPTH}) exceeded at: ${includePath}`, + includePath, + ); + } + } + + private readFile(includePath: string, resolvedPath: string): string { + try { + return this.resolver.readFile(resolvedPath); + } catch (err) { + throw new ConfigIncludeError( + `Failed to read include file: ${includePath} (resolved: ${resolvedPath})`, + includePath, + err instanceof Error ? err : undefined, + ); + } + } + + private parseFile( + includePath: string, + resolvedPath: string, + raw: string, + ): unknown { + try { + return this.resolver.parseJson(raw); + } catch (err) { + throw new ConfigIncludeError( + `Failed to parse include file: ${includePath} (resolved: ${resolvedPath})`, + includePath, + err instanceof Error ? err : undefined, + ); + } + } + + private processNested(resolvedPath: string, parsed: unknown): unknown { + const nested = new IncludeProcessor(resolvedPath, this.resolver); + nested.visited = new Set([...this.visited, resolvedPath]); + nested.depth = this.depth + 1; + return nested.process(parsed); + } +} + +// ============================================================================ +// Public API +// ============================================================================ + +const defaultResolver: IncludeResolver = { + readFile: (p) => fs.readFileSync(p, "utf-8"), + parseJson: (raw) => JSON5.parse(raw), +}; + +/** + * Resolves all $include directives in a parsed config object. + */ +export function resolveConfigIncludes( + obj: unknown, + configPath: string, + resolver: IncludeResolver = defaultResolver, +): unknown { + return new IncludeProcessor(configPath, resolver).process(obj); +} diff --git a/src/config/io.ts b/src/config/io.ts index 05fa98c2d..7b1ac6fb0 100644 --- a/src/config/io.ts +++ b/src/config/io.ts @@ -4,6 +4,7 @@ import os from "node:os"; import path from "node:path"; import JSON5 from "json5"; + import { loadShellEnvFallback, resolveShellEnvFallbackTimeoutMs, @@ -21,6 +22,7 @@ import { applySessionDefaults, applyTalkApiKey, } from "./defaults.js"; +import { ConfigIncludeError, resolveConfigIncludes } from "./includes.js"; import { findLegacyConfigIssues } from "./legacy.js"; import { resolveConfigPath, resolveStateDir } from "./paths.js"; import { applyConfigOverrides } from "./runtime-overrides.js"; @@ -32,6 +34,12 @@ import type { import { validateConfigObject } from "./validation.js"; import { ClawdbotSchema } from "./zod-schema.js"; +// Re-export for backwards compatibility +export { + CircularIncludeError, + ConfigIncludeError, +} from "./includes.js"; + const SHELL_ENV_EXPECTED_KEYS = [ "OPENAI_API_KEY", "ANTHROPIC_API_KEY", @@ -148,9 +156,16 @@ export function createConfigIO(overrides: ConfigIoDeps = {}) { } const raw = deps.fs.readFileSync(configPath, "utf-8"); const parsed = deps.json5.parse(raw); - warnOnConfigMiskeys(parsed, deps.logger); - if (typeof parsed !== "object" || parsed === null) return {}; - const validated = ClawdbotSchema.safeParse(parsed); + + // Resolve $include directives before validation + const resolved = resolveConfigIncludes(parsed, configPath, { + readFile: (p) => deps.fs.readFileSync(p, "utf-8"), + parseJson: (raw) => deps.json5.parse(raw), + }); + + warnOnConfigMiskeys(resolved, deps.logger); + if (typeof resolved !== "object" || resolved === null) return {}; + const validated = ClawdbotSchema.safeParse(resolved); if (!validated.success) { deps.logger.error("Invalid config:"); for (const iss of validated.error.issues) { @@ -245,9 +260,33 @@ export function createConfigIO(overrides: ConfigIoDeps = {}) { }; } - const legacyIssues = findLegacyConfigIssues(parsedRes.parsed); + // Resolve $include directives + let resolved: unknown; + try { + resolved = resolveConfigIncludes(parsedRes.parsed, configPath, { + readFile: (p) => deps.fs.readFileSync(p, "utf-8"), + parseJson: (raw) => deps.json5.parse(raw), + }); + } catch (err) { + const message = + err instanceof ConfigIncludeError + ? err.message + : `Include resolution failed: ${String(err)}`; + return { + path: configPath, + exists: true, + raw, + parsed: parsedRes.parsed, + valid: false, + config: {}, + issues: [{ path: "", message }], + legacyIssues: [], + }; + } - const validated = validateConfigObject(parsedRes.parsed); + const legacyIssues = findLegacyConfigIssues(resolved); + + const validated = validateConfigObject(resolved); if (!validated.ok) { return { path: configPath, diff --git a/src/postinstall-patcher.test.ts b/src/postinstall-patcher.test.ts index 44dfe2e01..855905eb5 100644 --- a/src/postinstall-patcher.test.ts +++ b/src/postinstall-patcher.test.ts @@ -16,13 +16,13 @@ describe("postinstall patcher", () => { fs.mkdirSync(target); const filePath = path.join(target, "main.js"); - const original = [ + const original = `${[ "var QRCode = require('./../vendor/QRCode'),", " QRErrorCorrectLevel = require('./../vendor/QRCode/QRErrorCorrectLevel'),", ' black = "\\033[40m \\033[0m",', ' white = "\\033[47m \\033[0m",', " toCell = function (isBlack) {", - ].join("\n") + "\n"; + ].join("\n")}\n`; fs.writeFileSync(filePath, original, "utf-8"); const patchText = `diff --git a/lib/main.js b/lib/main.js @@ -43,13 +43,13 @@ index 0000000..1111111 100644 const updated = fs.readFileSync(filePath, "utf-8"); expect(updated).toBe( - [ + `${[ "var QRCode = require('./../vendor/QRCode/index.js'),", " QRErrorCorrectLevel = require('./../vendor/QRCode/QRErrorCorrectLevel.js'),", ' black = "\\033[40m \\033[0m",', ' white = "\\033[47m \\033[0m",', " toCell = function (isBlack) {", - ].join("\n") + "\n", + ].join("\n")}\n`, ); fs.rmSync(dir, { recursive: true, force: true }); @@ -60,7 +60,7 @@ index 0000000..1111111 100644 const filePath = path.join(dir, "file.txt"); fs.writeFileSync( filePath, - ["alpha", "beta", "gamma", "delta", "epsilon"].join("\n") + "\n", + `${["alpha", "beta", "gamma", "delta", "epsilon"].join("\n")}\n`, "utf-8", );