feat(config): add $include directive for modular configs

Adds support for splitting clawdbot.json into multiple files using the
$include directive. This enables:

- Single file includes: { "$include": "./agents.json5" }
- Multiple file merging: { "$include": ["./a.json5", "./b.json5"] }
- Nested includes (up to 10 levels deep)
- Sibling key merging with includes

Features:
- Relative paths resolved from including file
- Absolute paths supported
- Circular include detection
- Clear error messages with resolved paths

Use case: Per-client agent configs for isolated sandboxed environments
(e.g., legal case management with strict data separation).
This commit is contained in:
sheeek
2026-01-11 15:01:02 +01:00
committed by Peter Steinberger
parent 6b2634512c
commit 15d286b617
3 changed files with 696 additions and 5 deletions

View File

@@ -82,6 +82,131 @@ To prevent the bot from responding to WhatsApp @-mentions in groups (only respon
}
```
## Config Includes (`$include`)
Split your config into multiple files using the `$include` directive. This is useful for:
- Organizing large configs (e.g., per-client agent definitions)
- Sharing common settings across environments
- Keeping sensitive configs separate
### Basic usage
```json5
// ~/.clawdbot/clawdbot.json
{
gateway: { port: 18789 },
// Include a single file (replaces the key's value)
agents: { "$include": "./agents.json5" },
// Include multiple files (deep-merged in order)
broadcast: {
"$include": [
"./clients/mueller.json5",
"./clients/schmidt.json5"
]
}
}
```
```json5
// ~/.clawdbot/agents.json5
{
defaults: { sandbox: { mode: "all", scope: "session" } },
list: [
{ id: "main", workspace: "~/clawd" }
]
}
```
### Merge behavior
- **Single file**: Replaces the object containing `$include`
- **Array of files**: Deep-merges files in order (later files override earlier ones)
- **With sibling keys**: Sibling keys are merged after includes (override included values)
```json5
// Sibling keys override included values
{
"$include": "./base.json5", // { a: 1, b: 2 }
b: 99 // Result: { a: 1, b: 99 }
}
```
### Nested includes
Included files can themselves contain `$include` directives (up to 10 levels deep):
```json5
// clients/mueller.json5
{
agents: { "$include": "./mueller/agents.json5" },
broadcast: { "$include": "./mueller/broadcast.json5" }
}
```
### Path resolution
- **Relative paths**: Resolved relative to the including file
- **Absolute paths**: Used as-is
- **Parent directories**: `../` references work as expected
```json5
{ "$include": "./sub/config.json5" } // relative
{ "$include": "/etc/clawdbot/base.json5" } // absolute
{ "$include": "../shared/common.json5" } // parent dir
```
### Error handling
- **Missing file**: Clear error with resolved path
- **Parse error**: Shows which included file failed
- **Circular includes**: Detected and reported with include chain
### Example: Multi-client legal setup
```json5
// ~/.clawdbot/clawdbot.json
{
gateway: { port: 18789, auth: { token: "secret" } },
// Common agent defaults
agents: {
defaults: {
sandbox: { mode: "all", scope: "session" }
},
// Merge agent lists from all clients
list: { "$include": [
"./clients/mueller/agents.json5",
"./clients/schmidt/agents.json5"
]}
},
// Merge broadcast configs
broadcast: { "$include": [
"./clients/mueller/broadcast.json5",
"./clients/schmidt/broadcast.json5"
]},
whatsapp: { groupPolicy: "allowlist" }
}
```
```json5
// ~/.clawdbot/clients/mueller/agents.json5
[
{ id: "mueller-transcribe", workspace: "~/clients/mueller/transcribe" },
{ id: "mueller-docs", workspace: "~/clients/mueller/docs" }
]
```
```json5
// ~/.clawdbot/clients/mueller/broadcast.json5
{
"120363403215116621@g.us": ["mueller-transcribe", "mueller-docs"]
}
```
## Common options
### Env vars + `.env`

View File

@@ -0,0 +1,304 @@
import { describe, expect, it } from "vitest";
import {
CircularIncludeError,
ConfigIncludeError,
resolveIncludes,
} from "./io.js";
function createMockContext(
files: Record<string, unknown>,
basePath = "/config/clawdbot.json",
) {
const fsModule = {
readFileSync: (filePath: string) => {
if (filePath in files) {
return JSON.stringify(files[filePath]);
}
const err = new Error(`ENOENT: no such file: ${filePath}`);
(err as NodeJS.ErrnoException).code = "ENOENT";
throw err;
},
} as typeof import("node:fs");
const json5Module = {
parse: JSON.parse,
} as typeof import("json5");
return {
basePath,
visited: new Set([basePath]),
depth: 0,
fsModule,
json5Module,
logger: { error: () => {}, warn: () => {} },
};
}
describe("resolveIncludes", () => {
it("passes through primitives unchanged", () => {
const ctx = createMockContext({});
expect(resolveIncludes("hello", ctx)).toBe("hello");
expect(resolveIncludes(42, ctx)).toBe(42);
expect(resolveIncludes(true, ctx)).toBe(true);
expect(resolveIncludes(null, ctx)).toBe(null);
});
it("passes through arrays with recursion", () => {
const ctx = createMockContext({});
expect(resolveIncludes([1, 2, { a: 1 }], ctx)).toEqual([1, 2, { a: 1 }]);
});
it("passes through objects without $include", () => {
const ctx = createMockContext({});
const obj = { foo: "bar", nested: { x: 1 } };
expect(resolveIncludes(obj, ctx)).toEqual(obj);
});
it("resolves single file $include", () => {
const ctx = createMockContext({
"/config/agents.json": { list: [{ id: "main" }] },
});
const obj = { agents: { $include: "./agents.json" } };
expect(resolveIncludes(obj, ctx)).toEqual({
agents: { list: [{ id: "main" }] },
});
});
it("resolves absolute path $include", () => {
const ctx = createMockContext({
"/etc/clawdbot/agents.json": { list: [{ id: "main" }] },
});
const obj = { agents: { $include: "/etc/clawdbot/agents.json" } };
expect(resolveIncludes(obj, ctx)).toEqual({
agents: { list: [{ id: "main" }] },
});
});
it("resolves array $include with deep merge", () => {
const ctx = createMockContext({
"/config/a.json": { "group-a": ["agent1"] },
"/config/b.json": { "group-b": ["agent2"] },
});
const obj = { broadcast: { $include: ["./a.json", "./b.json"] } };
expect(resolveIncludes(obj, ctx)).toEqual({
broadcast: {
"group-a": ["agent1"],
"group-b": ["agent2"],
},
});
});
it("deep merges overlapping keys in array $include", () => {
const ctx = createMockContext({
"/config/a.json": { agents: { defaults: { workspace: "~/a" } } },
"/config/b.json": { agents: { list: [{ id: "main" }] } },
});
const obj = { $include: ["./a.json", "./b.json"] };
expect(resolveIncludes(obj, ctx)).toEqual({
agents: {
defaults: { workspace: "~/a" },
list: [{ id: "main" }],
},
});
});
it("merges $include with sibling keys", () => {
const ctx = createMockContext({
"/config/base.json": { a: 1, b: 2 },
});
const obj = { $include: "./base.json", c: 3 };
expect(resolveIncludes(obj, ctx)).toEqual({ a: 1, b: 2, c: 3 });
});
it("sibling keys override included values", () => {
const ctx = createMockContext({
"/config/base.json": { a: 1, b: 2 },
});
const obj = { $include: "./base.json", b: 99 };
expect(resolveIncludes(obj, ctx)).toEqual({ a: 1, b: 99 });
});
it("resolves nested includes", () => {
const ctx = createMockContext({
"/config/level1.json": { nested: { $include: "./level2.json" } },
"/config/level2.json": { deep: "value" },
});
const obj = { $include: "./level1.json" };
expect(resolveIncludes(obj, ctx)).toEqual({
nested: { deep: "value" },
});
});
it("throws ConfigIncludeError for missing file", () => {
const ctx = createMockContext({});
const obj = { $include: "./missing.json" };
expect(() => resolveIncludes(obj, ctx)).toThrow(ConfigIncludeError);
expect(() => resolveIncludes(obj, ctx)).toThrow(/Failed to read include file/);
});
it("throws ConfigIncludeError for invalid JSON", () => {
const fsModule = {
readFileSync: () => "{ invalid json }",
} as typeof import("node:fs");
const json5Module = {
parse: JSON.parse,
} as typeof import("json5");
const ctx = {
basePath: "/config/clawdbot.json",
visited: new Set(["/config/clawdbot.json"]),
depth: 0,
fsModule,
json5Module,
logger: { error: () => {}, warn: () => {} },
};
const obj = { $include: "./bad.json" };
expect(() => resolveIncludes(obj, ctx)).toThrow(ConfigIncludeError);
expect(() => resolveIncludes(obj, ctx)).toThrow(/Failed to parse include file/);
});
it("throws CircularIncludeError for circular includes", () => {
// Create a mock that simulates circular includes
const fsModule = {
readFileSync: (filePath: string) => {
if (filePath === "/config/a.json") {
return JSON.stringify({ $include: "./b.json" });
}
if (filePath === "/config/b.json") {
return JSON.stringify({ $include: "./a.json" });
}
throw new Error(`Unknown file: ${filePath}`);
},
} as typeof import("node:fs");
const json5Module = { parse: JSON.parse } as typeof import("json5");
const ctx = {
basePath: "/config/clawdbot.json",
visited: new Set(["/config/clawdbot.json"]),
depth: 0,
fsModule,
json5Module,
logger: { error: () => {}, warn: () => {} },
};
const obj = { $include: "./a.json" };
expect(() => resolveIncludes(obj, ctx)).toThrow(CircularIncludeError);
expect(() => resolveIncludes(obj, ctx)).toThrow(/Circular include detected/);
});
it("throws ConfigIncludeError for invalid $include value type", () => {
const ctx = createMockContext({});
const obj = { $include: 123 };
expect(() => resolveIncludes(obj, ctx)).toThrow(ConfigIncludeError);
expect(() => resolveIncludes(obj, ctx)).toThrow(/expected string or array/);
});
it("throws ConfigIncludeError for invalid array item type", () => {
const ctx = createMockContext({
"/config/valid.json": { valid: true },
});
const obj = { $include: ["./valid.json", 123] };
expect(() => resolveIncludes(obj, ctx)).toThrow(ConfigIncludeError);
expect(() => resolveIncludes(obj, ctx)).toThrow(/expected string, got number/);
});
it("respects max depth limit", () => {
// Create deeply nested includes
const files: Record<string, unknown> = {};
for (let i = 0; i < 15; i++) {
files[`/config/level${i}.json`] = { $include: `./level${i + 1}.json` };
}
files["/config/level15.json"] = { done: true };
const ctx = createMockContext(files);
const obj = { $include: "./level0.json" };
expect(() => resolveIncludes(obj, ctx)).toThrow(ConfigIncludeError);
expect(() => resolveIncludes(obj, ctx)).toThrow(/Maximum include depth/);
});
it("handles relative paths correctly", () => {
const ctx = createMockContext(
{
"/config/clients/mueller/agents.json": { id: "mueller" },
},
"/config/clawdbot.json",
);
const obj = { agent: { $include: "./clients/mueller/agents.json" } };
expect(resolveIncludes(obj, ctx)).toEqual({
agent: { id: "mueller" },
});
});
it("resolves parent directory references", () => {
const ctx = createMockContext(
{
"/shared/common.json": { shared: true },
},
"/config/sub/clawdbot.json",
);
const obj = { $include: "../../shared/common.json" };
expect(resolveIncludes(obj, ctx)).toEqual({ shared: true });
});
});
describe("real-world config patterns", () => {
it("supports per-client agent includes", () => {
const ctx = createMockContext({
"/config/clients/mueller.json": {
agents: [
{ id: "mueller-screenshot", workspace: "~/clients/mueller/screenshot" },
{ id: "mueller-transcribe", workspace: "~/clients/mueller/transcribe" },
],
broadcast: { "group-mueller": ["mueller-screenshot", "mueller-transcribe"] },
},
"/config/clients/schmidt.json": {
agents: [
{ id: "schmidt-screenshot", workspace: "~/clients/schmidt/screenshot" },
],
broadcast: { "group-schmidt": ["schmidt-screenshot"] },
},
});
const obj = {
gateway: { port: 18789 },
$include: ["./clients/mueller.json", "./clients/schmidt.json"],
};
expect(resolveIncludes(obj, ctx)).toEqual({
gateway: { port: 18789 },
agents: [
{ id: "mueller-screenshot", workspace: "~/clients/mueller/screenshot" },
{ id: "mueller-transcribe", workspace: "~/clients/mueller/transcribe" },
{ id: "schmidt-screenshot", workspace: "~/clients/schmidt/screenshot" },
],
broadcast: {
"group-mueller": ["mueller-screenshot", "mueller-transcribe"],
"group-schmidt": ["schmidt-screenshot"],
},
});
});
it("supports modular config structure", () => {
const ctx = createMockContext({
"/config/gateway.json": { gateway: { port: 18789, bind: "loopback" } },
"/config/providers/whatsapp.json": {
whatsapp: { dmPolicy: "pairing", allowFrom: ["+49123"] },
},
"/config/agents/defaults.json": {
agents: { defaults: { sandbox: { mode: "all" } } },
},
});
const obj = {
$include: [
"./gateway.json",
"./providers/whatsapp.json",
"./agents/defaults.json",
],
};
expect(resolveIncludes(obj, ctx)).toEqual({
gateway: { port: 18789, bind: "loopback" },
whatsapp: { dmPolicy: "pairing", allowFrom: ["+49123"] },
agents: { defaults: { sandbox: { mode: "all" } } },
});
});
});

View File

@@ -4,6 +4,7 @@ import os from "node:os";
import path from "node:path";
import JSON5 from "json5";
import {
loadShellEnvFallback,
resolveShellEnvFallbackTimeoutMs,
@@ -128,6 +129,240 @@ export function parseConfigJson5(
}
}
// ============================================================================
// Config Includes ($include directive)
// ============================================================================
const INCLUDE_KEY = "$include";
const MAX_INCLUDE_DEPTH = 10;
export class ConfigIncludeError extends Error {
constructor(
message: string,
public readonly includePath: string,
public readonly cause?: Error,
) {
super(message);
this.name = "ConfigIncludeError";
}
}
export class CircularIncludeError extends ConfigIncludeError {
constructor(
public readonly chain: string[],
) {
super(
`Circular include detected: ${chain.join(" -> ")}`,
chain[chain.length - 1],
);
this.name = "CircularIncludeError";
}
}
type IncludeContext = {
basePath: string;
visited: Set<string>;
depth: number;
fsModule: typeof fs;
json5Module: typeof JSON5;
logger: Pick<typeof console, "error" | "warn">;
};
function isPlainObject(value: unknown): value is Record<string, unknown> {
return (
typeof value === "object" &&
value !== null &&
!Array.isArray(value) &&
Object.prototype.toString.call(value) === "[object Object]"
);
}
function deepMerge(target: unknown, source: unknown): unknown {
if (Array.isArray(target) && Array.isArray(source)) {
return [...target, ...source];
}
if (isPlainObject(target) && isPlainObject(source)) {
const result: Record<string, unknown> = { ...target };
for (const key of Object.keys(source)) {
if (key in result) {
result[key] = deepMerge(result[key], source[key]);
} else {
result[key] = source[key];
}
}
return result;
}
return source;
}
function resolveIncludePath(includePath: string, basePath: string): string {
if (path.isAbsolute(includePath)) {
return includePath;
}
const baseDir = path.dirname(basePath);
return path.resolve(baseDir, includePath);
}
function loadIncludeFile(
includePath: string,
ctx: IncludeContext,
): unknown {
const resolvedPath = resolveIncludePath(includePath, ctx.basePath);
const normalizedPath = path.normalize(resolvedPath);
// Check for circular includes
if (ctx.visited.has(normalizedPath)) {
throw new CircularIncludeError([...ctx.visited, normalizedPath]);
}
// Check depth limit
if (ctx.depth >= MAX_INCLUDE_DEPTH) {
throw new ConfigIncludeError(
`Maximum include depth (${MAX_INCLUDE_DEPTH}) exceeded at: ${includePath}`,
includePath,
);
}
// Read and parse the file
let raw: string;
try {
raw = ctx.fsModule.readFileSync(normalizedPath, "utf-8");
} catch (err) {
throw new ConfigIncludeError(
`Failed to read include file: ${includePath} (resolved: ${normalizedPath})`,
includePath,
err instanceof Error ? err : undefined,
);
}
let parsed: unknown;
try {
parsed = ctx.json5Module.parse(raw);
} catch (err) {
throw new ConfigIncludeError(
`Failed to parse include file: ${includePath} (resolved: ${normalizedPath})`,
includePath,
err instanceof Error ? err : undefined,
);
}
// Recursively resolve includes in the loaded file
const newCtx: IncludeContext = {
...ctx,
basePath: normalizedPath,
visited: new Set([...ctx.visited, normalizedPath]),
depth: ctx.depth + 1,
};
return resolveIncludes(parsed, newCtx);
}
function resolveIncludeDirective(
includeValue: unknown,
ctx: IncludeContext,
): unknown {
if (typeof includeValue === "string") {
// Single file include
return loadIncludeFile(includeValue, ctx);
}
if (Array.isArray(includeValue)) {
// Multiple files - deep merge them
let result: unknown = {};
for (const item of includeValue) {
if (typeof item !== "string") {
throw new ConfigIncludeError(
`Invalid $include array item: expected string, got ${typeof item}`,
String(item),
);
}
const loaded = loadIncludeFile(item, ctx);
result = deepMerge(result, loaded);
}
return result;
}
throw new ConfigIncludeError(
`Invalid $include value: expected string or array of strings, got ${typeof includeValue}`,
String(includeValue),
);
}
/**
* Recursively resolves $include directives in the config object.
*
* Supports:
* - `{ "$include": "./path/to/file.json5" }` - replaces object with file contents
* - `{ "$include": ["./a.json5", "./b.json5"] }` - deep merges multiple files
* - Nested includes up to MAX_INCLUDE_DEPTH levels
*
* @example
* ```json5
* // clawdbot.json
* {
* gateway: { port: 18789 },
* agents: { "$include": "./agents.json5" },
* broadcast: { "$include": ["./clients/a.json5", "./clients/b.json5"] }
* }
* ```
*/
export function resolveIncludes(
obj: unknown,
ctx: IncludeContext,
): unknown {
if (Array.isArray(obj)) {
return obj.map((item) => resolveIncludes(item, ctx));
}
if (isPlainObject(obj)) {
// Check if this object is an include directive
if (INCLUDE_KEY in obj) {
const includeValue = obj[INCLUDE_KEY];
const otherKeys = Object.keys(obj).filter((k) => k !== INCLUDE_KEY);
if (otherKeys.length > 0) {
// Has other keys besides $include - merge include result with them
const included = resolveIncludeDirective(includeValue, ctx);
const rest: Record<string, unknown> = {};
for (const key of otherKeys) {
rest[key] = resolveIncludes(obj[key], ctx);
}
return deepMerge(included, rest);
}
// Pure include directive
return resolveIncludeDirective(includeValue, ctx);
}
// Regular object - recurse into properties
const result: Record<string, unknown> = {};
for (const [key, value] of Object.entries(obj)) {
result[key] = resolveIncludes(value, ctx);
}
return result;
}
// Primitives pass through unchanged
return obj;
}
/**
* Creates an include context for resolving $include directives.
*/
function createIncludeContext(
configPath: string,
deps: Required<ConfigIoDeps>,
): IncludeContext {
return {
basePath: configPath,
visited: new Set([path.normalize(configPath)]),
depth: 0,
fsModule: deps.fs,
json5Module: deps.json5,
logger: deps.logger,
};
}
export function createConfigIO(overrides: ConfigIoDeps = {}) {
const deps = normalizeDeps(overrides);
const configPath = resolveConfigPathForDeps(deps);
@@ -148,9 +383,14 @@ export function createConfigIO(overrides: ConfigIoDeps = {}) {
}
const raw = deps.fs.readFileSync(configPath, "utf-8");
const parsed = deps.json5.parse(raw);
warnOnConfigMiskeys(parsed, deps.logger);
if (typeof parsed !== "object" || parsed === null) return {};
const validated = ClawdbotSchema.safeParse(parsed);
// Resolve $include directives before validation
const includeCtx = createIncludeContext(configPath, deps);
const resolved = resolveIncludes(parsed, includeCtx);
warnOnConfigMiskeys(resolved, deps.logger);
if (typeof resolved !== "object" || resolved === null) return {};
const validated = ClawdbotSchema.safeParse(resolved);
if (!validated.success) {
deps.logger.error("Invalid config:");
for (const iss of validated.error.issues) {
@@ -245,9 +485,31 @@ export function createConfigIO(overrides: ConfigIoDeps = {}) {
};
}
const legacyIssues = findLegacyConfigIssues(parsedRes.parsed);
// Resolve $include directives
let resolved: unknown;
try {
const includeCtx = createIncludeContext(configPath, deps);
resolved = resolveIncludes(parsedRes.parsed, includeCtx);
} catch (err) {
const message =
err instanceof ConfigIncludeError
? err.message
: `Include resolution failed: ${String(err)}`;
return {
path: configPath,
exists: true,
raw,
parsed: parsedRes.parsed,
valid: false,
config: {},
issues: [{ path: "", message }],
legacyIssues: [],
};
}
const validated = validateConfigObject(parsedRes.parsed);
const legacyIssues = findLegacyConfigIssues(resolved);
const validated = validateConfigObject(resolved);
if (!validated.ok) {
return {
path: configPath,