diff --git a/docs/gateway/configuration.md b/docs/gateway/configuration.md index 162d1c43e..9a0da11d7 100644 --- a/docs/gateway/configuration.md +++ b/docs/gateway/configuration.md @@ -82,6 +82,131 @@ To prevent the bot from responding to WhatsApp @-mentions in groups (only respon } ``` +## Config Includes (`$include`) + +Split your config into multiple files using the `$include` directive. This is useful for: +- Organizing large configs (e.g., per-client agent definitions) +- Sharing common settings across environments +- Keeping sensitive configs separate + +### Basic usage + +```json5 +// ~/.clawdbot/clawdbot.json +{ + gateway: { port: 18789 }, + + // Include a single file (replaces the key's value) + agents: { "$include": "./agents.json5" }, + + // Include multiple files (deep-merged in order) + broadcast: { + "$include": [ + "./clients/mueller.json5", + "./clients/schmidt.json5" + ] + } +} +``` + +```json5 +// ~/.clawdbot/agents.json5 +{ + defaults: { sandbox: { mode: "all", scope: "session" } }, + list: [ + { id: "main", workspace: "~/clawd" } + ] +} +``` + +### Merge behavior + +- **Single file**: Replaces the object containing `$include` +- **Array of files**: Deep-merges files in order (later files override earlier ones) +- **With sibling keys**: Sibling keys are merged after includes (override included values) + +```json5 +// Sibling keys override included values +{ + "$include": "./base.json5", // { a: 1, b: 2 } + b: 99 // Result: { a: 1, b: 99 } +} +``` + +### Nested includes + +Included files can themselves contain `$include` directives (up to 10 levels deep): + +```json5 +// clients/mueller.json5 +{ + agents: { "$include": "./mueller/agents.json5" }, + broadcast: { "$include": "./mueller/broadcast.json5" } +} +``` + +### Path resolution + +- **Relative paths**: Resolved relative to the including file +- **Absolute paths**: Used as-is +- **Parent directories**: `../` references work as expected + +```json5 +{ "$include": "./sub/config.json5" } // relative +{ "$include": "/etc/clawdbot/base.json5" } // absolute +{ "$include": "../shared/common.json5" } // parent dir +``` + +### Error handling + +- **Missing file**: Clear error with resolved path +- **Parse error**: Shows which included file failed +- **Circular includes**: Detected and reported with include chain + +### Example: Multi-client legal setup + +```json5 +// ~/.clawdbot/clawdbot.json +{ + gateway: { port: 18789, auth: { token: "secret" } }, + + // Common agent defaults + agents: { + defaults: { + sandbox: { mode: "all", scope: "session" } + }, + // Merge agent lists from all clients + list: { "$include": [ + "./clients/mueller/agents.json5", + "./clients/schmidt/agents.json5" + ]} + }, + + // Merge broadcast configs + broadcast: { "$include": [ + "./clients/mueller/broadcast.json5", + "./clients/schmidt/broadcast.json5" + ]}, + + whatsapp: { groupPolicy: "allowlist" } +} +``` + +```json5 +// ~/.clawdbot/clients/mueller/agents.json5 +[ + { id: "mueller-transcribe", workspace: "~/clients/mueller/transcribe" }, + { id: "mueller-docs", workspace: "~/clients/mueller/docs" } +] +``` + +```json5 +// ~/.clawdbot/clients/mueller/broadcast.json5 +{ + "120363403215116621@g.us": ["mueller-transcribe", "mueller-docs"] +} +``` + ## Common options ### Env vars + `.env` diff --git a/src/config/io-includes.test.ts b/src/config/io-includes.test.ts new file mode 100644 index 000000000..ad77d2570 --- /dev/null +++ b/src/config/io-includes.test.ts @@ -0,0 +1,304 @@ +import { describe, expect, it } from "vitest"; + +import { + CircularIncludeError, + ConfigIncludeError, + resolveIncludes, +} from "./io.js"; + +function createMockContext( + files: Record, + basePath = "/config/clawdbot.json", +) { + const fsModule = { + readFileSync: (filePath: string) => { + if (filePath in files) { + return JSON.stringify(files[filePath]); + } + const err = new Error(`ENOENT: no such file: ${filePath}`); + (err as NodeJS.ErrnoException).code = "ENOENT"; + throw err; + }, + } as typeof import("node:fs"); + + const json5Module = { + parse: JSON.parse, + } as typeof import("json5"); + + return { + basePath, + visited: new Set([basePath]), + depth: 0, + fsModule, + json5Module, + logger: { error: () => {}, warn: () => {} }, + }; +} + +describe("resolveIncludes", () => { + it("passes through primitives unchanged", () => { + const ctx = createMockContext({}); + expect(resolveIncludes("hello", ctx)).toBe("hello"); + expect(resolveIncludes(42, ctx)).toBe(42); + expect(resolveIncludes(true, ctx)).toBe(true); + expect(resolveIncludes(null, ctx)).toBe(null); + }); + + it("passes through arrays with recursion", () => { + const ctx = createMockContext({}); + expect(resolveIncludes([1, 2, { a: 1 }], ctx)).toEqual([1, 2, { a: 1 }]); + }); + + it("passes through objects without $include", () => { + const ctx = createMockContext({}); + const obj = { foo: "bar", nested: { x: 1 } }; + expect(resolveIncludes(obj, ctx)).toEqual(obj); + }); + + it("resolves single file $include", () => { + const ctx = createMockContext({ + "/config/agents.json": { list: [{ id: "main" }] }, + }); + const obj = { agents: { $include: "./agents.json" } }; + expect(resolveIncludes(obj, ctx)).toEqual({ + agents: { list: [{ id: "main" }] }, + }); + }); + + it("resolves absolute path $include", () => { + const ctx = createMockContext({ + "/etc/clawdbot/agents.json": { list: [{ id: "main" }] }, + }); + const obj = { agents: { $include: "/etc/clawdbot/agents.json" } }; + expect(resolveIncludes(obj, ctx)).toEqual({ + agents: { list: [{ id: "main" }] }, + }); + }); + + it("resolves array $include with deep merge", () => { + const ctx = createMockContext({ + "/config/a.json": { "group-a": ["agent1"] }, + "/config/b.json": { "group-b": ["agent2"] }, + }); + const obj = { broadcast: { $include: ["./a.json", "./b.json"] } }; + expect(resolveIncludes(obj, ctx)).toEqual({ + broadcast: { + "group-a": ["agent1"], + "group-b": ["agent2"], + }, + }); + }); + + it("deep merges overlapping keys in array $include", () => { + const ctx = createMockContext({ + "/config/a.json": { agents: { defaults: { workspace: "~/a" } } }, + "/config/b.json": { agents: { list: [{ id: "main" }] } }, + }); + const obj = { $include: ["./a.json", "./b.json"] }; + expect(resolveIncludes(obj, ctx)).toEqual({ + agents: { + defaults: { workspace: "~/a" }, + list: [{ id: "main" }], + }, + }); + }); + + it("merges $include with sibling keys", () => { + const ctx = createMockContext({ + "/config/base.json": { a: 1, b: 2 }, + }); + const obj = { $include: "./base.json", c: 3 }; + expect(resolveIncludes(obj, ctx)).toEqual({ a: 1, b: 2, c: 3 }); + }); + + it("sibling keys override included values", () => { + const ctx = createMockContext({ + "/config/base.json": { a: 1, b: 2 }, + }); + const obj = { $include: "./base.json", b: 99 }; + expect(resolveIncludes(obj, ctx)).toEqual({ a: 1, b: 99 }); + }); + + it("resolves nested includes", () => { + const ctx = createMockContext({ + "/config/level1.json": { nested: { $include: "./level2.json" } }, + "/config/level2.json": { deep: "value" }, + }); + const obj = { $include: "./level1.json" }; + expect(resolveIncludes(obj, ctx)).toEqual({ + nested: { deep: "value" }, + }); + }); + + it("throws ConfigIncludeError for missing file", () => { + const ctx = createMockContext({}); + const obj = { $include: "./missing.json" }; + expect(() => resolveIncludes(obj, ctx)).toThrow(ConfigIncludeError); + expect(() => resolveIncludes(obj, ctx)).toThrow(/Failed to read include file/); + }); + + it("throws ConfigIncludeError for invalid JSON", () => { + const fsModule = { + readFileSync: () => "{ invalid json }", + } as typeof import("node:fs"); + const json5Module = { + parse: JSON.parse, + } as typeof import("json5"); + const ctx = { + basePath: "/config/clawdbot.json", + visited: new Set(["/config/clawdbot.json"]), + depth: 0, + fsModule, + json5Module, + logger: { error: () => {}, warn: () => {} }, + }; + const obj = { $include: "./bad.json" }; + expect(() => resolveIncludes(obj, ctx)).toThrow(ConfigIncludeError); + expect(() => resolveIncludes(obj, ctx)).toThrow(/Failed to parse include file/); + }); + + it("throws CircularIncludeError for circular includes", () => { + // Create a mock that simulates circular includes + const fsModule = { + readFileSync: (filePath: string) => { + if (filePath === "/config/a.json") { + return JSON.stringify({ $include: "./b.json" }); + } + if (filePath === "/config/b.json") { + return JSON.stringify({ $include: "./a.json" }); + } + throw new Error(`Unknown file: ${filePath}`); + }, + } as typeof import("node:fs"); + const json5Module = { parse: JSON.parse } as typeof import("json5"); + const ctx = { + basePath: "/config/clawdbot.json", + visited: new Set(["/config/clawdbot.json"]), + depth: 0, + fsModule, + json5Module, + logger: { error: () => {}, warn: () => {} }, + }; + const obj = { $include: "./a.json" }; + expect(() => resolveIncludes(obj, ctx)).toThrow(CircularIncludeError); + expect(() => resolveIncludes(obj, ctx)).toThrow(/Circular include detected/); + }); + + it("throws ConfigIncludeError for invalid $include value type", () => { + const ctx = createMockContext({}); + const obj = { $include: 123 }; + expect(() => resolveIncludes(obj, ctx)).toThrow(ConfigIncludeError); + expect(() => resolveIncludes(obj, ctx)).toThrow(/expected string or array/); + }); + + it("throws ConfigIncludeError for invalid array item type", () => { + const ctx = createMockContext({ + "/config/valid.json": { valid: true }, + }); + const obj = { $include: ["./valid.json", 123] }; + expect(() => resolveIncludes(obj, ctx)).toThrow(ConfigIncludeError); + expect(() => resolveIncludes(obj, ctx)).toThrow(/expected string, got number/); + }); + + it("respects max depth limit", () => { + // Create deeply nested includes + const files: Record = {}; + for (let i = 0; i < 15; i++) { + files[`/config/level${i}.json`] = { $include: `./level${i + 1}.json` }; + } + files["/config/level15.json"] = { done: true }; + + const ctx = createMockContext(files); + const obj = { $include: "./level0.json" }; + expect(() => resolveIncludes(obj, ctx)).toThrow(ConfigIncludeError); + expect(() => resolveIncludes(obj, ctx)).toThrow(/Maximum include depth/); + }); + + it("handles relative paths correctly", () => { + const ctx = createMockContext( + { + "/config/clients/mueller/agents.json": { id: "mueller" }, + }, + "/config/clawdbot.json", + ); + const obj = { agent: { $include: "./clients/mueller/agents.json" } }; + expect(resolveIncludes(obj, ctx)).toEqual({ + agent: { id: "mueller" }, + }); + }); + + it("resolves parent directory references", () => { + const ctx = createMockContext( + { + "/shared/common.json": { shared: true }, + }, + "/config/sub/clawdbot.json", + ); + const obj = { $include: "../../shared/common.json" }; + expect(resolveIncludes(obj, ctx)).toEqual({ shared: true }); + }); +}); + +describe("real-world config patterns", () => { + it("supports per-client agent includes", () => { + const ctx = createMockContext({ + "/config/clients/mueller.json": { + agents: [ + { id: "mueller-screenshot", workspace: "~/clients/mueller/screenshot" }, + { id: "mueller-transcribe", workspace: "~/clients/mueller/transcribe" }, + ], + broadcast: { "group-mueller": ["mueller-screenshot", "mueller-transcribe"] }, + }, + "/config/clients/schmidt.json": { + agents: [ + { id: "schmidt-screenshot", workspace: "~/clients/schmidt/screenshot" }, + ], + broadcast: { "group-schmidt": ["schmidt-screenshot"] }, + }, + }); + + const obj = { + gateway: { port: 18789 }, + $include: ["./clients/mueller.json", "./clients/schmidt.json"], + }; + + expect(resolveIncludes(obj, ctx)).toEqual({ + gateway: { port: 18789 }, + agents: [ + { id: "mueller-screenshot", workspace: "~/clients/mueller/screenshot" }, + { id: "mueller-transcribe", workspace: "~/clients/mueller/transcribe" }, + { id: "schmidt-screenshot", workspace: "~/clients/schmidt/screenshot" }, + ], + broadcast: { + "group-mueller": ["mueller-screenshot", "mueller-transcribe"], + "group-schmidt": ["schmidt-screenshot"], + }, + }); + }); + + it("supports modular config structure", () => { + const ctx = createMockContext({ + "/config/gateway.json": { gateway: { port: 18789, bind: "loopback" } }, + "/config/providers/whatsapp.json": { + whatsapp: { dmPolicy: "pairing", allowFrom: ["+49123"] }, + }, + "/config/agents/defaults.json": { + agents: { defaults: { sandbox: { mode: "all" } } }, + }, + }); + + const obj = { + $include: [ + "./gateway.json", + "./providers/whatsapp.json", + "./agents/defaults.json", + ], + }; + + expect(resolveIncludes(obj, ctx)).toEqual({ + gateway: { port: 18789, bind: "loopback" }, + whatsapp: { dmPolicy: "pairing", allowFrom: ["+49123"] }, + agents: { defaults: { sandbox: { mode: "all" } } }, + }); + }); +}); diff --git a/src/config/io.ts b/src/config/io.ts index 05fa98c2d..8a68c8ae8 100644 --- a/src/config/io.ts +++ b/src/config/io.ts @@ -4,6 +4,7 @@ import os from "node:os"; import path from "node:path"; import JSON5 from "json5"; + import { loadShellEnvFallback, resolveShellEnvFallbackTimeoutMs, @@ -128,6 +129,240 @@ export function parseConfigJson5( } } +// ============================================================================ +// Config Includes ($include directive) +// ============================================================================ + +const INCLUDE_KEY = "$include"; +const MAX_INCLUDE_DEPTH = 10; + +export class ConfigIncludeError extends Error { + constructor( + message: string, + public readonly includePath: string, + public readonly cause?: Error, + ) { + super(message); + this.name = "ConfigIncludeError"; + } +} + +export class CircularIncludeError extends ConfigIncludeError { + constructor( + public readonly chain: string[], + ) { + super( + `Circular include detected: ${chain.join(" -> ")}`, + chain[chain.length - 1], + ); + this.name = "CircularIncludeError"; + } +} + +type IncludeContext = { + basePath: string; + visited: Set; + depth: number; + fsModule: typeof fs; + json5Module: typeof JSON5; + logger: Pick; +}; + +function isPlainObject(value: unknown): value is Record { + return ( + typeof value === "object" && + value !== null && + !Array.isArray(value) && + Object.prototype.toString.call(value) === "[object Object]" + ); +} + +function deepMerge(target: unknown, source: unknown): unknown { + if (Array.isArray(target) && Array.isArray(source)) { + return [...target, ...source]; + } + if (isPlainObject(target) && isPlainObject(source)) { + const result: Record = { ...target }; + for (const key of Object.keys(source)) { + if (key in result) { + result[key] = deepMerge(result[key], source[key]); + } else { + result[key] = source[key]; + } + } + return result; + } + return source; +} + +function resolveIncludePath(includePath: string, basePath: string): string { + if (path.isAbsolute(includePath)) { + return includePath; + } + const baseDir = path.dirname(basePath); + return path.resolve(baseDir, includePath); +} + +function loadIncludeFile( + includePath: string, + ctx: IncludeContext, +): unknown { + const resolvedPath = resolveIncludePath(includePath, ctx.basePath); + const normalizedPath = path.normalize(resolvedPath); + + // Check for circular includes + if (ctx.visited.has(normalizedPath)) { + throw new CircularIncludeError([...ctx.visited, normalizedPath]); + } + + // Check depth limit + if (ctx.depth >= MAX_INCLUDE_DEPTH) { + throw new ConfigIncludeError( + `Maximum include depth (${MAX_INCLUDE_DEPTH}) exceeded at: ${includePath}`, + includePath, + ); + } + + // Read and parse the file + let raw: string; + try { + raw = ctx.fsModule.readFileSync(normalizedPath, "utf-8"); + } catch (err) { + throw new ConfigIncludeError( + `Failed to read include file: ${includePath} (resolved: ${normalizedPath})`, + includePath, + err instanceof Error ? err : undefined, + ); + } + + let parsed: unknown; + try { + parsed = ctx.json5Module.parse(raw); + } catch (err) { + throw new ConfigIncludeError( + `Failed to parse include file: ${includePath} (resolved: ${normalizedPath})`, + includePath, + err instanceof Error ? err : undefined, + ); + } + + // Recursively resolve includes in the loaded file + const newCtx: IncludeContext = { + ...ctx, + basePath: normalizedPath, + visited: new Set([...ctx.visited, normalizedPath]), + depth: ctx.depth + 1, + }; + + return resolveIncludes(parsed, newCtx); +} + +function resolveIncludeDirective( + includeValue: unknown, + ctx: IncludeContext, +): unknown { + if (typeof includeValue === "string") { + // Single file include + return loadIncludeFile(includeValue, ctx); + } + + if (Array.isArray(includeValue)) { + // Multiple files - deep merge them + let result: unknown = {}; + for (const item of includeValue) { + if (typeof item !== "string") { + throw new ConfigIncludeError( + `Invalid $include array item: expected string, got ${typeof item}`, + String(item), + ); + } + const loaded = loadIncludeFile(item, ctx); + result = deepMerge(result, loaded); + } + return result; + } + + throw new ConfigIncludeError( + `Invalid $include value: expected string or array of strings, got ${typeof includeValue}`, + String(includeValue), + ); +} + +/** + * Recursively resolves $include directives in the config object. + * + * Supports: + * - `{ "$include": "./path/to/file.json5" }` - replaces object with file contents + * - `{ "$include": ["./a.json5", "./b.json5"] }` - deep merges multiple files + * - Nested includes up to MAX_INCLUDE_DEPTH levels + * + * @example + * ```json5 + * // clawdbot.json + * { + * gateway: { port: 18789 }, + * agents: { "$include": "./agents.json5" }, + * broadcast: { "$include": ["./clients/a.json5", "./clients/b.json5"] } + * } + * ``` + */ +export function resolveIncludes( + obj: unknown, + ctx: IncludeContext, +): unknown { + if (Array.isArray(obj)) { + return obj.map((item) => resolveIncludes(item, ctx)); + } + + if (isPlainObject(obj)) { + // Check if this object is an include directive + if (INCLUDE_KEY in obj) { + const includeValue = obj[INCLUDE_KEY]; + const otherKeys = Object.keys(obj).filter((k) => k !== INCLUDE_KEY); + + if (otherKeys.length > 0) { + // Has other keys besides $include - merge include result with them + const included = resolveIncludeDirective(includeValue, ctx); + const rest: Record = {}; + for (const key of otherKeys) { + rest[key] = resolveIncludes(obj[key], ctx); + } + return deepMerge(included, rest); + } + + // Pure include directive + return resolveIncludeDirective(includeValue, ctx); + } + + // Regular object - recurse into properties + const result: Record = {}; + for (const [key, value] of Object.entries(obj)) { + result[key] = resolveIncludes(value, ctx); + } + return result; + } + + // Primitives pass through unchanged + return obj; +} + +/** + * Creates an include context for resolving $include directives. + */ +function createIncludeContext( + configPath: string, + deps: Required, +): IncludeContext { + return { + basePath: configPath, + visited: new Set([path.normalize(configPath)]), + depth: 0, + fsModule: deps.fs, + json5Module: deps.json5, + logger: deps.logger, + }; +} + export function createConfigIO(overrides: ConfigIoDeps = {}) { const deps = normalizeDeps(overrides); const configPath = resolveConfigPathForDeps(deps); @@ -148,9 +383,14 @@ export function createConfigIO(overrides: ConfigIoDeps = {}) { } const raw = deps.fs.readFileSync(configPath, "utf-8"); const parsed = deps.json5.parse(raw); - warnOnConfigMiskeys(parsed, deps.logger); - if (typeof parsed !== "object" || parsed === null) return {}; - const validated = ClawdbotSchema.safeParse(parsed); + + // Resolve $include directives before validation + const includeCtx = createIncludeContext(configPath, deps); + const resolved = resolveIncludes(parsed, includeCtx); + + warnOnConfigMiskeys(resolved, deps.logger); + if (typeof resolved !== "object" || resolved === null) return {}; + const validated = ClawdbotSchema.safeParse(resolved); if (!validated.success) { deps.logger.error("Invalid config:"); for (const iss of validated.error.issues) { @@ -245,9 +485,31 @@ export function createConfigIO(overrides: ConfigIoDeps = {}) { }; } - const legacyIssues = findLegacyConfigIssues(parsedRes.parsed); + // Resolve $include directives + let resolved: unknown; + try { + const includeCtx = createIncludeContext(configPath, deps); + resolved = resolveIncludes(parsedRes.parsed, includeCtx); + } catch (err) { + const message = + err instanceof ConfigIncludeError + ? err.message + : `Include resolution failed: ${String(err)}`; + return { + path: configPath, + exists: true, + raw, + parsed: parsedRes.parsed, + valid: false, + config: {}, + issues: [{ path: "", message }], + legacyIssues: [], + }; + } - const validated = validateConfigObject(parsedRes.parsed); + const legacyIssues = findLegacyConfigIssues(resolved); + + const validated = validateConfigObject(resolved); if (!validated.ok) { return { path: configPath,