Merge pull request #731 from pasogott/feat/config-includes
feat(config): add $include directive for modular configs
This commit is contained in:
@@ -7,6 +7,7 @@
|
|||||||
- Plugins: voice-call plugin now real (Twilio/log), adds start/status RPC/CLI/tool + tests.
|
- Plugins: voice-call plugin now real (Twilio/log), adds start/status RPC/CLI/tool + tests.
|
||||||
- Docs: add plugins doc + cross-links from tools/skills/gateway config.
|
- Docs: add plugins doc + cross-links from tools/skills/gateway config.
|
||||||
- Tests: add Docker plugin loader smoke test.
|
- Tests: add Docker plugin loader smoke test.
|
||||||
|
- Config: add `$include` directive for modular config files. (#731) — thanks @pasogott.
|
||||||
- Build: set pnpm minimum release age to 2880 minutes (2 days). (#718) — thanks @dan-dr.
|
- Build: set pnpm minimum release age to 2880 minutes (2 days). (#718) — thanks @dan-dr.
|
||||||
- macOS: prompt to install the global `clawdbot` CLI when missing in local mode; install via `clawd.bot/install-cli.sh` (no onboarding) and use external launchd/CLI instead of the embedded gateway runtime.
|
- macOS: prompt to install the global `clawdbot` CLI when missing in local mode; install via `clawd.bot/install-cli.sh` (no onboarding) and use external launchd/CLI instead of the embedded gateway runtime.
|
||||||
- Docs: add gog calendar event color IDs from `gog calendar colors`. (#715) — thanks @mjrussell.
|
- Docs: add gog calendar event color IDs from `gog calendar colors`. (#715) — thanks @mjrussell.
|
||||||
|
|||||||
@@ -82,6 +82,132 @@ To prevent the bot from responding to WhatsApp @-mentions in groups (only respon
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Config Includes (`$include`)
|
||||||
|
|
||||||
|
Split your config into multiple files using the `$include` directive. This is useful for:
|
||||||
|
- Organizing large configs (e.g., per-client agent definitions)
|
||||||
|
- Sharing common settings across environments
|
||||||
|
- Keeping sensitive configs separate
|
||||||
|
|
||||||
|
### Basic usage
|
||||||
|
|
||||||
|
```json5
|
||||||
|
// ~/.clawdbot/clawdbot.json
|
||||||
|
{
|
||||||
|
gateway: { port: 18789 },
|
||||||
|
|
||||||
|
// Include a single file (replaces the key's value)
|
||||||
|
agents: { "$include": "./agents.json5" },
|
||||||
|
|
||||||
|
// Include multiple files (deep-merged in order)
|
||||||
|
broadcast: {
|
||||||
|
"$include": [
|
||||||
|
"./clients/mueller.json5",
|
||||||
|
"./clients/schmidt.json5"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
```json5
|
||||||
|
// ~/.clawdbot/agents.json5
|
||||||
|
{
|
||||||
|
defaults: { sandbox: { mode: "all", scope: "session" } },
|
||||||
|
list: [
|
||||||
|
{ id: "main", workspace: "~/clawd" }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Merge behavior
|
||||||
|
|
||||||
|
- **Single file**: Replaces the object containing `$include`
|
||||||
|
- **Array of files**: Deep-merges files in order (later files override earlier ones)
|
||||||
|
- **With sibling keys**: Sibling keys are merged after includes (override included values)
|
||||||
|
- **Sibling keys + arrays/primitives**: Not supported (included content must be an object)
|
||||||
|
|
||||||
|
```json5
|
||||||
|
// Sibling keys override included values
|
||||||
|
{
|
||||||
|
"$include": "./base.json5", // { a: 1, b: 2 }
|
||||||
|
b: 99 // Result: { a: 1, b: 99 }
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Nested includes
|
||||||
|
|
||||||
|
Included files can themselves contain `$include` directives (up to 10 levels deep):
|
||||||
|
|
||||||
|
```json5
|
||||||
|
// clients/mueller.json5
|
||||||
|
{
|
||||||
|
agents: { "$include": "./mueller/agents.json5" },
|
||||||
|
broadcast: { "$include": "./mueller/broadcast.json5" }
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Path resolution
|
||||||
|
|
||||||
|
- **Relative paths**: Resolved relative to the including file
|
||||||
|
- **Absolute paths**: Used as-is
|
||||||
|
- **Parent directories**: `../` references work as expected
|
||||||
|
|
||||||
|
```json5
|
||||||
|
{ "$include": "./sub/config.json5" } // relative
|
||||||
|
{ "$include": "/etc/clawdbot/base.json5" } // absolute
|
||||||
|
{ "$include": "../shared/common.json5" } // parent dir
|
||||||
|
```
|
||||||
|
|
||||||
|
### Error handling
|
||||||
|
|
||||||
|
- **Missing file**: Clear error with resolved path
|
||||||
|
- **Parse error**: Shows which included file failed
|
||||||
|
- **Circular includes**: Detected and reported with include chain
|
||||||
|
|
||||||
|
### Example: Multi-client legal setup
|
||||||
|
|
||||||
|
```json5
|
||||||
|
// ~/.clawdbot/clawdbot.json
|
||||||
|
{
|
||||||
|
gateway: { port: 18789, auth: { token: "secret" } },
|
||||||
|
|
||||||
|
// Common agent defaults
|
||||||
|
agents: {
|
||||||
|
defaults: {
|
||||||
|
sandbox: { mode: "all", scope: "session" }
|
||||||
|
},
|
||||||
|
// Merge agent lists from all clients
|
||||||
|
list: { "$include": [
|
||||||
|
"./clients/mueller/agents.json5",
|
||||||
|
"./clients/schmidt/agents.json5"
|
||||||
|
]}
|
||||||
|
},
|
||||||
|
|
||||||
|
// Merge broadcast configs
|
||||||
|
broadcast: { "$include": [
|
||||||
|
"./clients/mueller/broadcast.json5",
|
||||||
|
"./clients/schmidt/broadcast.json5"
|
||||||
|
]},
|
||||||
|
|
||||||
|
whatsapp: { groupPolicy: "allowlist" }
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
```json5
|
||||||
|
// ~/.clawdbot/clients/mueller/agents.json5
|
||||||
|
[
|
||||||
|
{ id: "mueller-transcribe", workspace: "~/clients/mueller/transcribe" },
|
||||||
|
{ id: "mueller-docs", workspace: "~/clients/mueller/docs" }
|
||||||
|
]
|
||||||
|
```
|
||||||
|
|
||||||
|
```json5
|
||||||
|
// ~/.clawdbot/clients/mueller/broadcast.json5
|
||||||
|
{
|
||||||
|
"120363403215116621@g.us": ["mueller-transcribe", "mueller-docs"]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
## Common options
|
## Common options
|
||||||
|
|
||||||
### Env vars + `.env`
|
### Env vars + `.env`
|
||||||
|
|||||||
280
src/config/includes.test.ts
Normal file
280
src/config/includes.test.ts
Normal file
@@ -0,0 +1,280 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
|
||||||
|
import {
|
||||||
|
CircularIncludeError,
|
||||||
|
ConfigIncludeError,
|
||||||
|
type IncludeResolver,
|
||||||
|
resolveConfigIncludes,
|
||||||
|
} from "./includes.js";
|
||||||
|
|
||||||
|
function createMockResolver(files: Record<string, unknown>): IncludeResolver {
|
||||||
|
return {
|
||||||
|
readFile: (filePath: string) => {
|
||||||
|
if (filePath in files) {
|
||||||
|
return JSON.stringify(files[filePath]);
|
||||||
|
}
|
||||||
|
throw new Error(`ENOENT: no such file: ${filePath}`);
|
||||||
|
},
|
||||||
|
parseJson: JSON.parse,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolve(
|
||||||
|
obj: unknown,
|
||||||
|
files: Record<string, unknown> = {},
|
||||||
|
basePath = "/config/clawdbot.json",
|
||||||
|
) {
|
||||||
|
return resolveConfigIncludes(obj, basePath, createMockResolver(files));
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("resolveConfigIncludes", () => {
|
||||||
|
it("passes through primitives unchanged", () => {
|
||||||
|
expect(resolve("hello")).toBe("hello");
|
||||||
|
expect(resolve(42)).toBe(42);
|
||||||
|
expect(resolve(true)).toBe(true);
|
||||||
|
expect(resolve(null)).toBe(null);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("passes through arrays with recursion", () => {
|
||||||
|
expect(resolve([1, 2, { a: 1 }])).toEqual([1, 2, { a: 1 }]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("passes through objects without $include", () => {
|
||||||
|
const obj = { foo: "bar", nested: { x: 1 } };
|
||||||
|
expect(resolve(obj)).toEqual(obj);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("resolves single file $include", () => {
|
||||||
|
const files = { "/config/agents.json": { list: [{ id: "main" }] } };
|
||||||
|
const obj = { agents: { $include: "./agents.json" } };
|
||||||
|
expect(resolve(obj, files)).toEqual({
|
||||||
|
agents: { list: [{ id: "main" }] },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("resolves absolute path $include", () => {
|
||||||
|
const files = { "/etc/clawdbot/agents.json": { list: [{ id: "main" }] } };
|
||||||
|
const obj = { agents: { $include: "/etc/clawdbot/agents.json" } };
|
||||||
|
expect(resolve(obj, files)).toEqual({
|
||||||
|
agents: { list: [{ id: "main" }] },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("resolves array $include with deep merge", () => {
|
||||||
|
const files = {
|
||||||
|
"/config/a.json": { "group-a": ["agent1"] },
|
||||||
|
"/config/b.json": { "group-b": ["agent2"] },
|
||||||
|
};
|
||||||
|
const obj = { broadcast: { $include: ["./a.json", "./b.json"] } };
|
||||||
|
expect(resolve(obj, files)).toEqual({
|
||||||
|
broadcast: {
|
||||||
|
"group-a": ["agent1"],
|
||||||
|
"group-b": ["agent2"],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("deep merges overlapping keys in array $include", () => {
|
||||||
|
const files = {
|
||||||
|
"/config/a.json": { agents: { defaults: { workspace: "~/a" } } },
|
||||||
|
"/config/b.json": { agents: { list: [{ id: "main" }] } },
|
||||||
|
};
|
||||||
|
const obj = { $include: ["./a.json", "./b.json"] };
|
||||||
|
expect(resolve(obj, files)).toEqual({
|
||||||
|
agents: {
|
||||||
|
defaults: { workspace: "~/a" },
|
||||||
|
list: [{ id: "main" }],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("merges $include with sibling keys", () => {
|
||||||
|
const files = { "/config/base.json": { a: 1, b: 2 } };
|
||||||
|
const obj = { $include: "./base.json", c: 3 };
|
||||||
|
expect(resolve(obj, files)).toEqual({ a: 1, b: 2, c: 3 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("sibling keys override included values", () => {
|
||||||
|
const files = { "/config/base.json": { a: 1, b: 2 } };
|
||||||
|
const obj = { $include: "./base.json", b: 99 };
|
||||||
|
expect(resolve(obj, files)).toEqual({ a: 1, b: 99 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("throws when sibling keys are used with non-object includes", () => {
|
||||||
|
const files = { "/config/list.json": ["a", "b"] };
|
||||||
|
const obj = { $include: "./list.json", extra: true };
|
||||||
|
expect(() => resolve(obj, files)).toThrow(ConfigIncludeError);
|
||||||
|
expect(() => resolve(obj, files)).toThrow(
|
||||||
|
/Sibling keys require included content to be an object/,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("resolves nested includes", () => {
|
||||||
|
const files = {
|
||||||
|
"/config/level1.json": { nested: { $include: "./level2.json" } },
|
||||||
|
"/config/level2.json": { deep: "value" },
|
||||||
|
};
|
||||||
|
const obj = { $include: "./level1.json" };
|
||||||
|
expect(resolve(obj, files)).toEqual({
|
||||||
|
nested: { deep: "value" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("throws ConfigIncludeError for missing file", () => {
|
||||||
|
const obj = { $include: "./missing.json" };
|
||||||
|
expect(() => resolve(obj)).toThrow(ConfigIncludeError);
|
||||||
|
expect(() => resolve(obj)).toThrow(/Failed to read include file/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("throws ConfigIncludeError for invalid JSON", () => {
|
||||||
|
const resolver: IncludeResolver = {
|
||||||
|
readFile: () => "{ invalid json }",
|
||||||
|
parseJson: JSON.parse,
|
||||||
|
};
|
||||||
|
const obj = { $include: "./bad.json" };
|
||||||
|
expect(() =>
|
||||||
|
resolveConfigIncludes(obj, "/config/clawdbot.json", resolver),
|
||||||
|
).toThrow(ConfigIncludeError);
|
||||||
|
expect(() =>
|
||||||
|
resolveConfigIncludes(obj, "/config/clawdbot.json", resolver),
|
||||||
|
).toThrow(/Failed to parse include file/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("throws CircularIncludeError for circular includes", () => {
|
||||||
|
const resolver: IncludeResolver = {
|
||||||
|
readFile: (filePath: string) => {
|
||||||
|
if (filePath === "/config/a.json") {
|
||||||
|
return JSON.stringify({ $include: "./b.json" });
|
||||||
|
}
|
||||||
|
if (filePath === "/config/b.json") {
|
||||||
|
return JSON.stringify({ $include: "./a.json" });
|
||||||
|
}
|
||||||
|
throw new Error(`Unknown file: ${filePath}`);
|
||||||
|
},
|
||||||
|
parseJson: JSON.parse,
|
||||||
|
};
|
||||||
|
const obj = { $include: "./a.json" };
|
||||||
|
expect(() =>
|
||||||
|
resolveConfigIncludes(obj, "/config/clawdbot.json", resolver),
|
||||||
|
).toThrow(CircularIncludeError);
|
||||||
|
expect(() =>
|
||||||
|
resolveConfigIncludes(obj, "/config/clawdbot.json", resolver),
|
||||||
|
).toThrow(/Circular include detected/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("throws ConfigIncludeError for invalid $include value type", () => {
|
||||||
|
const obj = { $include: 123 };
|
||||||
|
expect(() => resolve(obj)).toThrow(ConfigIncludeError);
|
||||||
|
expect(() => resolve(obj)).toThrow(/expected string or array/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("throws ConfigIncludeError for invalid array item type", () => {
|
||||||
|
const files = { "/config/valid.json": { valid: true } };
|
||||||
|
const obj = { $include: ["./valid.json", 123] };
|
||||||
|
expect(() => resolve(obj, files)).toThrow(ConfigIncludeError);
|
||||||
|
expect(() => resolve(obj, files)).toThrow(/expected string, got number/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("respects max depth limit", () => {
|
||||||
|
const files: Record<string, unknown> = {};
|
||||||
|
for (let i = 0; i < 15; i++) {
|
||||||
|
files[`/config/level${i}.json`] = { $include: `./level${i + 1}.json` };
|
||||||
|
}
|
||||||
|
files["/config/level15.json"] = { done: true };
|
||||||
|
|
||||||
|
const obj = { $include: "./level0.json" };
|
||||||
|
expect(() => resolve(obj, files)).toThrow(ConfigIncludeError);
|
||||||
|
expect(() => resolve(obj, files)).toThrow(/Maximum include depth/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("handles relative paths correctly", () => {
|
||||||
|
const files = { "/config/clients/mueller/agents.json": { id: "mueller" } };
|
||||||
|
const obj = { agent: { $include: "./clients/mueller/agents.json" } };
|
||||||
|
expect(resolve(obj, files)).toEqual({
|
||||||
|
agent: { id: "mueller" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("resolves parent directory references", () => {
|
||||||
|
const files = { "/shared/common.json": { shared: true } };
|
||||||
|
const obj = { $include: "../../shared/common.json" };
|
||||||
|
expect(resolve(obj, files, "/config/sub/clawdbot.json")).toEqual({
|
||||||
|
shared: true,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("real-world config patterns", () => {
|
||||||
|
it("supports per-client agent includes", () => {
|
||||||
|
const files = {
|
||||||
|
"/config/clients/mueller.json": {
|
||||||
|
agents: [
|
||||||
|
{
|
||||||
|
id: "mueller-screenshot",
|
||||||
|
workspace: "~/clients/mueller/screenshot",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "mueller-transcribe",
|
||||||
|
workspace: "~/clients/mueller/transcribe",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
broadcast: {
|
||||||
|
"group-mueller": ["mueller-screenshot", "mueller-transcribe"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"/config/clients/schmidt.json": {
|
||||||
|
agents: [
|
||||||
|
{
|
||||||
|
id: "schmidt-screenshot",
|
||||||
|
workspace: "~/clients/schmidt/screenshot",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
broadcast: { "group-schmidt": ["schmidt-screenshot"] },
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const obj = {
|
||||||
|
gateway: { port: 18789 },
|
||||||
|
$include: ["./clients/mueller.json", "./clients/schmidt.json"],
|
||||||
|
};
|
||||||
|
|
||||||
|
expect(resolve(obj, files)).toEqual({
|
||||||
|
gateway: { port: 18789 },
|
||||||
|
agents: [
|
||||||
|
{ id: "mueller-screenshot", workspace: "~/clients/mueller/screenshot" },
|
||||||
|
{ id: "mueller-transcribe", workspace: "~/clients/mueller/transcribe" },
|
||||||
|
{ id: "schmidt-screenshot", workspace: "~/clients/schmidt/screenshot" },
|
||||||
|
],
|
||||||
|
broadcast: {
|
||||||
|
"group-mueller": ["mueller-screenshot", "mueller-transcribe"],
|
||||||
|
"group-schmidt": ["schmidt-screenshot"],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("supports modular config structure", () => {
|
||||||
|
const files = {
|
||||||
|
"/config/gateway.json": { gateway: { port: 18789, bind: "loopback" } },
|
||||||
|
"/config/providers/whatsapp.json": {
|
||||||
|
whatsapp: { dmPolicy: "pairing", allowFrom: ["+49123"] },
|
||||||
|
},
|
||||||
|
"/config/agents/defaults.json": {
|
||||||
|
agents: { defaults: { sandbox: { mode: "all" } } },
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const obj = {
|
||||||
|
$include: [
|
||||||
|
"./gateway.json",
|
||||||
|
"./providers/whatsapp.json",
|
||||||
|
"./agents/defaults.json",
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
expect(resolve(obj, files)).toEqual({
|
||||||
|
gateway: { port: 18789, bind: "loopback" },
|
||||||
|
whatsapp: { dmPolicy: "pairing", allowFrom: ["+49123"] },
|
||||||
|
agents: { defaults: { sandbox: { mode: "all" } } },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
258
src/config/includes.ts
Normal file
258
src/config/includes.ts
Normal file
@@ -0,0 +1,258 @@
|
|||||||
|
/**
|
||||||
|
* Config includes: $include directive for modular configs
|
||||||
|
*
|
||||||
|
* @example
|
||||||
|
* ```json5
|
||||||
|
* {
|
||||||
|
* "$include": "./base.json5", // single file
|
||||||
|
* "$include": ["./a.json5", "./b.json5"] // merge multiple
|
||||||
|
* }
|
||||||
|
* ```
|
||||||
|
*/
|
||||||
|
|
||||||
|
import fs from "node:fs";
|
||||||
|
import path from "node:path";
|
||||||
|
|
||||||
|
import JSON5 from "json5";
|
||||||
|
|
||||||
|
export const INCLUDE_KEY = "$include";
|
||||||
|
export const MAX_INCLUDE_DEPTH = 10;
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Types
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
export type IncludeResolver = {
|
||||||
|
readFile: (path: string) => string;
|
||||||
|
parseJson: (raw: string) => unknown;
|
||||||
|
};
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Errors
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
export class ConfigIncludeError extends Error {
|
||||||
|
constructor(
|
||||||
|
message: string,
|
||||||
|
public readonly includePath: string,
|
||||||
|
public readonly cause?: Error,
|
||||||
|
) {
|
||||||
|
super(message);
|
||||||
|
this.name = "ConfigIncludeError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class CircularIncludeError extends ConfigIncludeError {
|
||||||
|
constructor(public readonly chain: string[]) {
|
||||||
|
super(
|
||||||
|
`Circular include detected: ${chain.join(" -> ")}`,
|
||||||
|
chain[chain.length - 1],
|
||||||
|
);
|
||||||
|
this.name = "CircularIncludeError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Utilities
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
function isPlainObject(value: unknown): value is Record<string, unknown> {
|
||||||
|
return (
|
||||||
|
typeof value === "object" &&
|
||||||
|
value !== null &&
|
||||||
|
!Array.isArray(value) &&
|
||||||
|
Object.prototype.toString.call(value) === "[object Object]"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Deep merge: arrays concatenate, objects merge recursively, primitives: source wins */
|
||||||
|
export function deepMerge(target: unknown, source: unknown): unknown {
|
||||||
|
if (Array.isArray(target) && Array.isArray(source)) {
|
||||||
|
return [...target, ...source];
|
||||||
|
}
|
||||||
|
if (isPlainObject(target) && isPlainObject(source)) {
|
||||||
|
const result: Record<string, unknown> = { ...target };
|
||||||
|
for (const key of Object.keys(source)) {
|
||||||
|
result[key] =
|
||||||
|
key in result ? deepMerge(result[key], source[key]) : source[key];
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
return source;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Include Resolver Class
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
class IncludeProcessor {
|
||||||
|
private visited = new Set<string>();
|
||||||
|
private depth = 0;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private basePath: string,
|
||||||
|
private resolver: IncludeResolver,
|
||||||
|
) {
|
||||||
|
this.visited.add(path.normalize(basePath));
|
||||||
|
}
|
||||||
|
|
||||||
|
process(obj: unknown): unknown {
|
||||||
|
if (Array.isArray(obj)) {
|
||||||
|
return obj.map((item) => this.process(item));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!isPlainObject(obj)) {
|
||||||
|
return obj;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!(INCLUDE_KEY in obj)) {
|
||||||
|
return this.processObject(obj);
|
||||||
|
}
|
||||||
|
|
||||||
|
return this.processInclude(obj);
|
||||||
|
}
|
||||||
|
|
||||||
|
private processObject(obj: Record<string, unknown>): Record<string, unknown> {
|
||||||
|
const result: Record<string, unknown> = {};
|
||||||
|
for (const [key, value] of Object.entries(obj)) {
|
||||||
|
result[key] = this.process(value);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private processInclude(obj: Record<string, unknown>): unknown {
|
||||||
|
const includeValue = obj[INCLUDE_KEY];
|
||||||
|
const otherKeys = Object.keys(obj).filter((k) => k !== INCLUDE_KEY);
|
||||||
|
const included = this.resolveInclude(includeValue);
|
||||||
|
|
||||||
|
if (otherKeys.length === 0) {
|
||||||
|
return included;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!isPlainObject(included)) {
|
||||||
|
throw new ConfigIncludeError(
|
||||||
|
"Sibling keys require included content to be an object",
|
||||||
|
typeof includeValue === "string" ? includeValue : INCLUDE_KEY,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Merge included content with sibling keys
|
||||||
|
const rest: Record<string, unknown> = {};
|
||||||
|
for (const key of otherKeys) {
|
||||||
|
rest[key] = this.process(obj[key]);
|
||||||
|
}
|
||||||
|
return deepMerge(included, rest);
|
||||||
|
}
|
||||||
|
|
||||||
|
private resolveInclude(value: unknown): unknown {
|
||||||
|
if (typeof value === "string") {
|
||||||
|
return this.loadFile(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (Array.isArray(value)) {
|
||||||
|
return value.reduce<unknown>((merged, item) => {
|
||||||
|
if (typeof item !== "string") {
|
||||||
|
throw new ConfigIncludeError(
|
||||||
|
`Invalid $include array item: expected string, got ${typeof item}`,
|
||||||
|
String(item),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return deepMerge(merged, this.loadFile(item));
|
||||||
|
}, {});
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new ConfigIncludeError(
|
||||||
|
`Invalid $include value: expected string or array of strings, got ${typeof value}`,
|
||||||
|
String(value),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private loadFile(includePath: string): unknown {
|
||||||
|
const resolvedPath = this.resolvePath(includePath);
|
||||||
|
|
||||||
|
this.checkCircular(resolvedPath);
|
||||||
|
this.checkDepth(includePath);
|
||||||
|
|
||||||
|
const raw = this.readFile(includePath, resolvedPath);
|
||||||
|
const parsed = this.parseFile(includePath, resolvedPath, raw);
|
||||||
|
|
||||||
|
return this.processNested(resolvedPath, parsed);
|
||||||
|
}
|
||||||
|
|
||||||
|
private resolvePath(includePath: string): string {
|
||||||
|
const resolved = path.isAbsolute(includePath)
|
||||||
|
? includePath
|
||||||
|
: path.resolve(path.dirname(this.basePath), includePath);
|
||||||
|
return path.normalize(resolved);
|
||||||
|
}
|
||||||
|
|
||||||
|
private checkCircular(resolvedPath: string): void {
|
||||||
|
if (this.visited.has(resolvedPath)) {
|
||||||
|
throw new CircularIncludeError([...this.visited, resolvedPath]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private checkDepth(includePath: string): void {
|
||||||
|
if (this.depth >= MAX_INCLUDE_DEPTH) {
|
||||||
|
throw new ConfigIncludeError(
|
||||||
|
`Maximum include depth (${MAX_INCLUDE_DEPTH}) exceeded at: ${includePath}`,
|
||||||
|
includePath,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private readFile(includePath: string, resolvedPath: string): string {
|
||||||
|
try {
|
||||||
|
return this.resolver.readFile(resolvedPath);
|
||||||
|
} catch (err) {
|
||||||
|
throw new ConfigIncludeError(
|
||||||
|
`Failed to read include file: ${includePath} (resolved: ${resolvedPath})`,
|
||||||
|
includePath,
|
||||||
|
err instanceof Error ? err : undefined,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private parseFile(
|
||||||
|
includePath: string,
|
||||||
|
resolvedPath: string,
|
||||||
|
raw: string,
|
||||||
|
): unknown {
|
||||||
|
try {
|
||||||
|
return this.resolver.parseJson(raw);
|
||||||
|
} catch (err) {
|
||||||
|
throw new ConfigIncludeError(
|
||||||
|
`Failed to parse include file: ${includePath} (resolved: ${resolvedPath})`,
|
||||||
|
includePath,
|
||||||
|
err instanceof Error ? err : undefined,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private processNested(resolvedPath: string, parsed: unknown): unknown {
|
||||||
|
const nested = new IncludeProcessor(resolvedPath, this.resolver);
|
||||||
|
nested.visited = new Set([...this.visited, resolvedPath]);
|
||||||
|
nested.depth = this.depth + 1;
|
||||||
|
return nested.process(parsed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Public API
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
const defaultResolver: IncludeResolver = {
|
||||||
|
readFile: (p) => fs.readFileSync(p, "utf-8"),
|
||||||
|
parseJson: (raw) => JSON5.parse(raw),
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resolves all $include directives in a parsed config object.
|
||||||
|
*/
|
||||||
|
export function resolveConfigIncludes(
|
||||||
|
obj: unknown,
|
||||||
|
configPath: string,
|
||||||
|
resolver: IncludeResolver = defaultResolver,
|
||||||
|
): unknown {
|
||||||
|
return new IncludeProcessor(configPath, resolver).process(obj);
|
||||||
|
}
|
||||||
@@ -4,6 +4,7 @@ import os from "node:os";
|
|||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
|
|
||||||
import JSON5 from "json5";
|
import JSON5 from "json5";
|
||||||
|
|
||||||
import {
|
import {
|
||||||
loadShellEnvFallback,
|
loadShellEnvFallback,
|
||||||
resolveShellEnvFallbackTimeoutMs,
|
resolveShellEnvFallbackTimeoutMs,
|
||||||
@@ -21,6 +22,7 @@ import {
|
|||||||
applySessionDefaults,
|
applySessionDefaults,
|
||||||
applyTalkApiKey,
|
applyTalkApiKey,
|
||||||
} from "./defaults.js";
|
} from "./defaults.js";
|
||||||
|
import { ConfigIncludeError, resolveConfigIncludes } from "./includes.js";
|
||||||
import { findLegacyConfigIssues } from "./legacy.js";
|
import { findLegacyConfigIssues } from "./legacy.js";
|
||||||
import { resolveConfigPath, resolveStateDir } from "./paths.js";
|
import { resolveConfigPath, resolveStateDir } from "./paths.js";
|
||||||
import { applyConfigOverrides } from "./runtime-overrides.js";
|
import { applyConfigOverrides } from "./runtime-overrides.js";
|
||||||
@@ -32,6 +34,12 @@ import type {
|
|||||||
import { validateConfigObject } from "./validation.js";
|
import { validateConfigObject } from "./validation.js";
|
||||||
import { ClawdbotSchema } from "./zod-schema.js";
|
import { ClawdbotSchema } from "./zod-schema.js";
|
||||||
|
|
||||||
|
// Re-export for backwards compatibility
|
||||||
|
export {
|
||||||
|
CircularIncludeError,
|
||||||
|
ConfigIncludeError,
|
||||||
|
} from "./includes.js";
|
||||||
|
|
||||||
const SHELL_ENV_EXPECTED_KEYS = [
|
const SHELL_ENV_EXPECTED_KEYS = [
|
||||||
"OPENAI_API_KEY",
|
"OPENAI_API_KEY",
|
||||||
"ANTHROPIC_API_KEY",
|
"ANTHROPIC_API_KEY",
|
||||||
@@ -148,9 +156,16 @@ export function createConfigIO(overrides: ConfigIoDeps = {}) {
|
|||||||
}
|
}
|
||||||
const raw = deps.fs.readFileSync(configPath, "utf-8");
|
const raw = deps.fs.readFileSync(configPath, "utf-8");
|
||||||
const parsed = deps.json5.parse(raw);
|
const parsed = deps.json5.parse(raw);
|
||||||
warnOnConfigMiskeys(parsed, deps.logger);
|
|
||||||
if (typeof parsed !== "object" || parsed === null) return {};
|
// Resolve $include directives before validation
|
||||||
const validated = ClawdbotSchema.safeParse(parsed);
|
const resolved = resolveConfigIncludes(parsed, configPath, {
|
||||||
|
readFile: (p) => deps.fs.readFileSync(p, "utf-8"),
|
||||||
|
parseJson: (raw) => deps.json5.parse(raw),
|
||||||
|
});
|
||||||
|
|
||||||
|
warnOnConfigMiskeys(resolved, deps.logger);
|
||||||
|
if (typeof resolved !== "object" || resolved === null) return {};
|
||||||
|
const validated = ClawdbotSchema.safeParse(resolved);
|
||||||
if (!validated.success) {
|
if (!validated.success) {
|
||||||
deps.logger.error("Invalid config:");
|
deps.logger.error("Invalid config:");
|
||||||
for (const iss of validated.error.issues) {
|
for (const iss of validated.error.issues) {
|
||||||
@@ -245,9 +260,33 @@ export function createConfigIO(overrides: ConfigIoDeps = {}) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const legacyIssues = findLegacyConfigIssues(parsedRes.parsed);
|
// Resolve $include directives
|
||||||
|
let resolved: unknown;
|
||||||
|
try {
|
||||||
|
resolved = resolveConfigIncludes(parsedRes.parsed, configPath, {
|
||||||
|
readFile: (p) => deps.fs.readFileSync(p, "utf-8"),
|
||||||
|
parseJson: (raw) => deps.json5.parse(raw),
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
const message =
|
||||||
|
err instanceof ConfigIncludeError
|
||||||
|
? err.message
|
||||||
|
: `Include resolution failed: ${String(err)}`;
|
||||||
|
return {
|
||||||
|
path: configPath,
|
||||||
|
exists: true,
|
||||||
|
raw,
|
||||||
|
parsed: parsedRes.parsed,
|
||||||
|
valid: false,
|
||||||
|
config: {},
|
||||||
|
issues: [{ path: "", message }],
|
||||||
|
legacyIssues: [],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
const validated = validateConfigObject(parsedRes.parsed);
|
const legacyIssues = findLegacyConfigIssues(resolved);
|
||||||
|
|
||||||
|
const validated = validateConfigObject(resolved);
|
||||||
if (!validated.ok) {
|
if (!validated.ok) {
|
||||||
return {
|
return {
|
||||||
path: configPath,
|
path: configPath,
|
||||||
|
|||||||
@@ -16,13 +16,13 @@ describe("postinstall patcher", () => {
|
|||||||
fs.mkdirSync(target);
|
fs.mkdirSync(target);
|
||||||
|
|
||||||
const filePath = path.join(target, "main.js");
|
const filePath = path.join(target, "main.js");
|
||||||
const original = [
|
const original = `${[
|
||||||
"var QRCode = require('./../vendor/QRCode'),",
|
"var QRCode = require('./../vendor/QRCode'),",
|
||||||
" QRErrorCorrectLevel = require('./../vendor/QRCode/QRErrorCorrectLevel'),",
|
" QRErrorCorrectLevel = require('./../vendor/QRCode/QRErrorCorrectLevel'),",
|
||||||
' black = "\\033[40m \\033[0m",',
|
' black = "\\033[40m \\033[0m",',
|
||||||
' white = "\\033[47m \\033[0m",',
|
' white = "\\033[47m \\033[0m",',
|
||||||
" toCell = function (isBlack) {",
|
" toCell = function (isBlack) {",
|
||||||
].join("\n") + "\n";
|
].join("\n")}\n`;
|
||||||
fs.writeFileSync(filePath, original, "utf-8");
|
fs.writeFileSync(filePath, original, "utf-8");
|
||||||
|
|
||||||
const patchText = `diff --git a/lib/main.js b/lib/main.js
|
const patchText = `diff --git a/lib/main.js b/lib/main.js
|
||||||
@@ -43,13 +43,13 @@ index 0000000..1111111 100644
|
|||||||
|
|
||||||
const updated = fs.readFileSync(filePath, "utf-8");
|
const updated = fs.readFileSync(filePath, "utf-8");
|
||||||
expect(updated).toBe(
|
expect(updated).toBe(
|
||||||
[
|
`${[
|
||||||
"var QRCode = require('./../vendor/QRCode/index.js'),",
|
"var QRCode = require('./../vendor/QRCode/index.js'),",
|
||||||
" QRErrorCorrectLevel = require('./../vendor/QRCode/QRErrorCorrectLevel.js'),",
|
" QRErrorCorrectLevel = require('./../vendor/QRCode/QRErrorCorrectLevel.js'),",
|
||||||
' black = "\\033[40m \\033[0m",',
|
' black = "\\033[40m \\033[0m",',
|
||||||
' white = "\\033[47m \\033[0m",',
|
' white = "\\033[47m \\033[0m",',
|
||||||
" toCell = function (isBlack) {",
|
" toCell = function (isBlack) {",
|
||||||
].join("\n") + "\n",
|
].join("\n")}\n`,
|
||||||
);
|
);
|
||||||
|
|
||||||
fs.rmSync(dir, { recursive: true, force: true });
|
fs.rmSync(dir, { recursive: true, force: true });
|
||||||
@@ -60,7 +60,7 @@ index 0000000..1111111 100644
|
|||||||
const filePath = path.join(dir, "file.txt");
|
const filePath = path.join(dir, "file.txt");
|
||||||
fs.writeFileSync(
|
fs.writeFileSync(
|
||||||
filePath,
|
filePath,
|
||||||
["alpha", "beta", "gamma", "delta", "epsilon"].join("\n") + "\n",
|
`${["alpha", "beta", "gamma", "delta", "epsilon"].join("\n")}\n`,
|
||||||
"utf-8",
|
"utf-8",
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user