refactor(config): extract includes logic to separate module
- Move $include resolution to src/config/includes.ts - Simplify io.ts by importing from includes module - Cleaner API: resolveConfigIncludes(obj, configPath, resolver?) - Re-export errors from io.ts for backwards compatibility - Rename test file to match module name
This commit is contained in:
committed by
Peter Steinberger
parent
15d286b617
commit
e6400b0b0f
@@ -3,13 +3,10 @@ import { describe, expect, it } from "vitest";
|
|||||||
import {
|
import {
|
||||||
CircularIncludeError,
|
CircularIncludeError,
|
||||||
ConfigIncludeError,
|
ConfigIncludeError,
|
||||||
resolveIncludes,
|
resolveConfigIncludes,
|
||||||
} from "./io.js";
|
} from "./includes.js";
|
||||||
|
|
||||||
function createMockContext(
|
function createMockResolver(files: Record<string, unknown>) {
|
||||||
files: Record<string, unknown>,
|
|
||||||
basePath = "/config/clawdbot.json",
|
|
||||||
) {
|
|
||||||
const fsModule = {
|
const fsModule = {
|
||||||
readFileSync: (filePath: string) => {
|
readFileSync: (filePath: string) => {
|
||||||
if (filePath in files) {
|
if (filePath in files) {
|
||||||
@@ -25,63 +22,57 @@ function createMockContext(
|
|||||||
parse: JSON.parse,
|
parse: JSON.parse,
|
||||||
} as typeof import("json5");
|
} as typeof import("json5");
|
||||||
|
|
||||||
return {
|
return { fsModule, json5Module };
|
||||||
basePath,
|
|
||||||
visited: new Set([basePath]),
|
|
||||||
depth: 0,
|
|
||||||
fsModule,
|
|
||||||
json5Module,
|
|
||||||
logger: { error: () => {}, warn: () => {} },
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("resolveIncludes", () => {
|
function resolve(
|
||||||
|
obj: unknown,
|
||||||
|
files: Record<string, unknown> = {},
|
||||||
|
basePath = "/config/clawdbot.json",
|
||||||
|
) {
|
||||||
|
return resolveConfigIncludes(obj, basePath, createMockResolver(files));
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("resolveConfigIncludes", () => {
|
||||||
it("passes through primitives unchanged", () => {
|
it("passes through primitives unchanged", () => {
|
||||||
const ctx = createMockContext({});
|
expect(resolve("hello")).toBe("hello");
|
||||||
expect(resolveIncludes("hello", ctx)).toBe("hello");
|
expect(resolve(42)).toBe(42);
|
||||||
expect(resolveIncludes(42, ctx)).toBe(42);
|
expect(resolve(true)).toBe(true);
|
||||||
expect(resolveIncludes(true, ctx)).toBe(true);
|
expect(resolve(null)).toBe(null);
|
||||||
expect(resolveIncludes(null, ctx)).toBe(null);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it("passes through arrays with recursion", () => {
|
it("passes through arrays with recursion", () => {
|
||||||
const ctx = createMockContext({});
|
expect(resolve([1, 2, { a: 1 }])).toEqual([1, 2, { a: 1 }]);
|
||||||
expect(resolveIncludes([1, 2, { a: 1 }], ctx)).toEqual([1, 2, { a: 1 }]);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it("passes through objects without $include", () => {
|
it("passes through objects without $include", () => {
|
||||||
const ctx = createMockContext({});
|
|
||||||
const obj = { foo: "bar", nested: { x: 1 } };
|
const obj = { foo: "bar", nested: { x: 1 } };
|
||||||
expect(resolveIncludes(obj, ctx)).toEqual(obj);
|
expect(resolve(obj)).toEqual(obj);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("resolves single file $include", () => {
|
it("resolves single file $include", () => {
|
||||||
const ctx = createMockContext({
|
const files = { "/config/agents.json": { list: [{ id: "main" }] } };
|
||||||
"/config/agents.json": { list: [{ id: "main" }] },
|
|
||||||
});
|
|
||||||
const obj = { agents: { $include: "./agents.json" } };
|
const obj = { agents: { $include: "./agents.json" } };
|
||||||
expect(resolveIncludes(obj, ctx)).toEqual({
|
expect(resolve(obj, files)).toEqual({
|
||||||
agents: { list: [{ id: "main" }] },
|
agents: { list: [{ id: "main" }] },
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("resolves absolute path $include", () => {
|
it("resolves absolute path $include", () => {
|
||||||
const ctx = createMockContext({
|
const files = { "/etc/clawdbot/agents.json": { list: [{ id: "main" }] } };
|
||||||
"/etc/clawdbot/agents.json": { list: [{ id: "main" }] },
|
|
||||||
});
|
|
||||||
const obj = { agents: { $include: "/etc/clawdbot/agents.json" } };
|
const obj = { agents: { $include: "/etc/clawdbot/agents.json" } };
|
||||||
expect(resolveIncludes(obj, ctx)).toEqual({
|
expect(resolve(obj, files)).toEqual({
|
||||||
agents: { list: [{ id: "main" }] },
|
agents: { list: [{ id: "main" }] },
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("resolves array $include with deep merge", () => {
|
it("resolves array $include with deep merge", () => {
|
||||||
const ctx = createMockContext({
|
const files = {
|
||||||
"/config/a.json": { "group-a": ["agent1"] },
|
"/config/a.json": { "group-a": ["agent1"] },
|
||||||
"/config/b.json": { "group-b": ["agent2"] },
|
"/config/b.json": { "group-b": ["agent2"] },
|
||||||
});
|
};
|
||||||
const obj = { broadcast: { $include: ["./a.json", "./b.json"] } };
|
const obj = { broadcast: { $include: ["./a.json", "./b.json"] } };
|
||||||
expect(resolveIncludes(obj, ctx)).toEqual({
|
expect(resolve(obj, files)).toEqual({
|
||||||
broadcast: {
|
broadcast: {
|
||||||
"group-a": ["agent1"],
|
"group-a": ["agent1"],
|
||||||
"group-b": ["agent2"],
|
"group-b": ["agent2"],
|
||||||
@@ -90,12 +81,12 @@ describe("resolveIncludes", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("deep merges overlapping keys in array $include", () => {
|
it("deep merges overlapping keys in array $include", () => {
|
||||||
const ctx = createMockContext({
|
const files = {
|
||||||
"/config/a.json": { agents: { defaults: { workspace: "~/a" } } },
|
"/config/a.json": { agents: { defaults: { workspace: "~/a" } } },
|
||||||
"/config/b.json": { agents: { list: [{ id: "main" }] } },
|
"/config/b.json": { agents: { list: [{ id: "main" }] } },
|
||||||
});
|
};
|
||||||
const obj = { $include: ["./a.json", "./b.json"] };
|
const obj = { $include: ["./a.json", "./b.json"] };
|
||||||
expect(resolveIncludes(obj, ctx)).toEqual({
|
expect(resolve(obj, files)).toEqual({
|
||||||
agents: {
|
agents: {
|
||||||
defaults: { workspace: "~/a" },
|
defaults: { workspace: "~/a" },
|
||||||
list: [{ id: "main" }],
|
list: [{ id: "main" }],
|
||||||
@@ -104,144 +95,111 @@ describe("resolveIncludes", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("merges $include with sibling keys", () => {
|
it("merges $include with sibling keys", () => {
|
||||||
const ctx = createMockContext({
|
const files = { "/config/base.json": { a: 1, b: 2 } };
|
||||||
"/config/base.json": { a: 1, b: 2 },
|
|
||||||
});
|
|
||||||
const obj = { $include: "./base.json", c: 3 };
|
const obj = { $include: "./base.json", c: 3 };
|
||||||
expect(resolveIncludes(obj, ctx)).toEqual({ a: 1, b: 2, c: 3 });
|
expect(resolve(obj, files)).toEqual({ a: 1, b: 2, c: 3 });
|
||||||
});
|
});
|
||||||
|
|
||||||
it("sibling keys override included values", () => {
|
it("sibling keys override included values", () => {
|
||||||
const ctx = createMockContext({
|
const files = { "/config/base.json": { a: 1, b: 2 } };
|
||||||
"/config/base.json": { a: 1, b: 2 },
|
|
||||||
});
|
|
||||||
const obj = { $include: "./base.json", b: 99 };
|
const obj = { $include: "./base.json", b: 99 };
|
||||||
expect(resolveIncludes(obj, ctx)).toEqual({ a: 1, b: 99 });
|
expect(resolve(obj, files)).toEqual({ a: 1, b: 99 });
|
||||||
});
|
});
|
||||||
|
|
||||||
it("resolves nested includes", () => {
|
it("resolves nested includes", () => {
|
||||||
const ctx = createMockContext({
|
const files = {
|
||||||
"/config/level1.json": { nested: { $include: "./level2.json" } },
|
"/config/level1.json": { nested: { $include: "./level2.json" } },
|
||||||
"/config/level2.json": { deep: "value" },
|
"/config/level2.json": { deep: "value" },
|
||||||
});
|
};
|
||||||
const obj = { $include: "./level1.json" };
|
const obj = { $include: "./level1.json" };
|
||||||
expect(resolveIncludes(obj, ctx)).toEqual({
|
expect(resolve(obj, files)).toEqual({
|
||||||
nested: { deep: "value" },
|
nested: { deep: "value" },
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("throws ConfigIncludeError for missing file", () => {
|
it("throws ConfigIncludeError for missing file", () => {
|
||||||
const ctx = createMockContext({});
|
|
||||||
const obj = { $include: "./missing.json" };
|
const obj = { $include: "./missing.json" };
|
||||||
expect(() => resolveIncludes(obj, ctx)).toThrow(ConfigIncludeError);
|
expect(() => resolve(obj)).toThrow(ConfigIncludeError);
|
||||||
expect(() => resolveIncludes(obj, ctx)).toThrow(/Failed to read include file/);
|
expect(() => resolve(obj)).toThrow(/Failed to read include file/);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("throws ConfigIncludeError for invalid JSON", () => {
|
it("throws ConfigIncludeError for invalid JSON", () => {
|
||||||
const fsModule = {
|
const resolver = {
|
||||||
readFileSync: () => "{ invalid json }",
|
fsModule: { readFileSync: () => "{ invalid json }" } as typeof import("node:fs"),
|
||||||
} as typeof import("node:fs");
|
json5Module: { parse: JSON.parse } as typeof import("json5"),
|
||||||
const json5Module = {
|
|
||||||
parse: JSON.parse,
|
|
||||||
} as typeof import("json5");
|
|
||||||
const ctx = {
|
|
||||||
basePath: "/config/clawdbot.json",
|
|
||||||
visited: new Set(["/config/clawdbot.json"]),
|
|
||||||
depth: 0,
|
|
||||||
fsModule,
|
|
||||||
json5Module,
|
|
||||||
logger: { error: () => {}, warn: () => {} },
|
|
||||||
};
|
};
|
||||||
const obj = { $include: "./bad.json" };
|
const obj = { $include: "./bad.json" };
|
||||||
expect(() => resolveIncludes(obj, ctx)).toThrow(ConfigIncludeError);
|
expect(() => resolveConfigIncludes(obj, "/config/clawdbot.json", resolver))
|
||||||
expect(() => resolveIncludes(obj, ctx)).toThrow(/Failed to parse include file/);
|
.toThrow(ConfigIncludeError);
|
||||||
|
expect(() => resolveConfigIncludes(obj, "/config/clawdbot.json", resolver))
|
||||||
|
.toThrow(/Failed to parse include file/);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("throws CircularIncludeError for circular includes", () => {
|
it("throws CircularIncludeError for circular includes", () => {
|
||||||
// Create a mock that simulates circular includes
|
const resolver = {
|
||||||
const fsModule = {
|
fsModule: {
|
||||||
readFileSync: (filePath: string) => {
|
readFileSync: (filePath: string) => {
|
||||||
if (filePath === "/config/a.json") {
|
if (filePath === "/config/a.json") {
|
||||||
return JSON.stringify({ $include: "./b.json" });
|
return JSON.stringify({ $include: "./b.json" });
|
||||||
}
|
}
|
||||||
if (filePath === "/config/b.json") {
|
if (filePath === "/config/b.json") {
|
||||||
return JSON.stringify({ $include: "./a.json" });
|
return JSON.stringify({ $include: "./a.json" });
|
||||||
}
|
}
|
||||||
throw new Error(`Unknown file: ${filePath}`);
|
throw new Error(`Unknown file: ${filePath}`);
|
||||||
},
|
},
|
||||||
} as typeof import("node:fs");
|
} as typeof import("node:fs"),
|
||||||
const json5Module = { parse: JSON.parse } as typeof import("json5");
|
json5Module: { parse: JSON.parse } as typeof import("json5"),
|
||||||
const ctx = {
|
|
||||||
basePath: "/config/clawdbot.json",
|
|
||||||
visited: new Set(["/config/clawdbot.json"]),
|
|
||||||
depth: 0,
|
|
||||||
fsModule,
|
|
||||||
json5Module,
|
|
||||||
logger: { error: () => {}, warn: () => {} },
|
|
||||||
};
|
};
|
||||||
const obj = { $include: "./a.json" };
|
const obj = { $include: "./a.json" };
|
||||||
expect(() => resolveIncludes(obj, ctx)).toThrow(CircularIncludeError);
|
expect(() => resolveConfigIncludes(obj, "/config/clawdbot.json", resolver))
|
||||||
expect(() => resolveIncludes(obj, ctx)).toThrow(/Circular include detected/);
|
.toThrow(CircularIncludeError);
|
||||||
|
expect(() => resolveConfigIncludes(obj, "/config/clawdbot.json", resolver))
|
||||||
|
.toThrow(/Circular include detected/);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("throws ConfigIncludeError for invalid $include value type", () => {
|
it("throws ConfigIncludeError for invalid $include value type", () => {
|
||||||
const ctx = createMockContext({});
|
|
||||||
const obj = { $include: 123 };
|
const obj = { $include: 123 };
|
||||||
expect(() => resolveIncludes(obj, ctx)).toThrow(ConfigIncludeError);
|
expect(() => resolve(obj)).toThrow(ConfigIncludeError);
|
||||||
expect(() => resolveIncludes(obj, ctx)).toThrow(/expected string or array/);
|
expect(() => resolve(obj)).toThrow(/expected string or array/);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("throws ConfigIncludeError for invalid array item type", () => {
|
it("throws ConfigIncludeError for invalid array item type", () => {
|
||||||
const ctx = createMockContext({
|
const files = { "/config/valid.json": { valid: true } };
|
||||||
"/config/valid.json": { valid: true },
|
|
||||||
});
|
|
||||||
const obj = { $include: ["./valid.json", 123] };
|
const obj = { $include: ["./valid.json", 123] };
|
||||||
expect(() => resolveIncludes(obj, ctx)).toThrow(ConfigIncludeError);
|
expect(() => resolve(obj, files)).toThrow(ConfigIncludeError);
|
||||||
expect(() => resolveIncludes(obj, ctx)).toThrow(/expected string, got number/);
|
expect(() => resolve(obj, files)).toThrow(/expected string, got number/);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("respects max depth limit", () => {
|
it("respects max depth limit", () => {
|
||||||
// Create deeply nested includes
|
|
||||||
const files: Record<string, unknown> = {};
|
const files: Record<string, unknown> = {};
|
||||||
for (let i = 0; i < 15; i++) {
|
for (let i = 0; i < 15; i++) {
|
||||||
files[`/config/level${i}.json`] = { $include: `./level${i + 1}.json` };
|
files[`/config/level${i}.json`] = { $include: `./level${i + 1}.json` };
|
||||||
}
|
}
|
||||||
files["/config/level15.json"] = { done: true };
|
files["/config/level15.json"] = { done: true };
|
||||||
|
|
||||||
const ctx = createMockContext(files);
|
|
||||||
const obj = { $include: "./level0.json" };
|
const obj = { $include: "./level0.json" };
|
||||||
expect(() => resolveIncludes(obj, ctx)).toThrow(ConfigIncludeError);
|
expect(() => resolve(obj, files)).toThrow(ConfigIncludeError);
|
||||||
expect(() => resolveIncludes(obj, ctx)).toThrow(/Maximum include depth/);
|
expect(() => resolve(obj, files)).toThrow(/Maximum include depth/);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("handles relative paths correctly", () => {
|
it("handles relative paths correctly", () => {
|
||||||
const ctx = createMockContext(
|
const files = { "/config/clients/mueller/agents.json": { id: "mueller" } };
|
||||||
{
|
|
||||||
"/config/clients/mueller/agents.json": { id: "mueller" },
|
|
||||||
},
|
|
||||||
"/config/clawdbot.json",
|
|
||||||
);
|
|
||||||
const obj = { agent: { $include: "./clients/mueller/agents.json" } };
|
const obj = { agent: { $include: "./clients/mueller/agents.json" } };
|
||||||
expect(resolveIncludes(obj, ctx)).toEqual({
|
expect(resolve(obj, files)).toEqual({
|
||||||
agent: { id: "mueller" },
|
agent: { id: "mueller" },
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("resolves parent directory references", () => {
|
it("resolves parent directory references", () => {
|
||||||
const ctx = createMockContext(
|
const files = { "/shared/common.json": { shared: true } };
|
||||||
{
|
|
||||||
"/shared/common.json": { shared: true },
|
|
||||||
},
|
|
||||||
"/config/sub/clawdbot.json",
|
|
||||||
);
|
|
||||||
const obj = { $include: "../../shared/common.json" };
|
const obj = { $include: "../../shared/common.json" };
|
||||||
expect(resolveIncludes(obj, ctx)).toEqual({ shared: true });
|
expect(resolve(obj, files, "/config/sub/clawdbot.json")).toEqual({ shared: true });
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("real-world config patterns", () => {
|
describe("real-world config patterns", () => {
|
||||||
it("supports per-client agent includes", () => {
|
it("supports per-client agent includes", () => {
|
||||||
const ctx = createMockContext({
|
const files = {
|
||||||
"/config/clients/mueller.json": {
|
"/config/clients/mueller.json": {
|
||||||
agents: [
|
agents: [
|
||||||
{ id: "mueller-screenshot", workspace: "~/clients/mueller/screenshot" },
|
{ id: "mueller-screenshot", workspace: "~/clients/mueller/screenshot" },
|
||||||
@@ -255,14 +213,14 @@ describe("real-world config patterns", () => {
|
|||||||
],
|
],
|
||||||
broadcast: { "group-schmidt": ["schmidt-screenshot"] },
|
broadcast: { "group-schmidt": ["schmidt-screenshot"] },
|
||||||
},
|
},
|
||||||
});
|
};
|
||||||
|
|
||||||
const obj = {
|
const obj = {
|
||||||
gateway: { port: 18789 },
|
gateway: { port: 18789 },
|
||||||
$include: ["./clients/mueller.json", "./clients/schmidt.json"],
|
$include: ["./clients/mueller.json", "./clients/schmidt.json"],
|
||||||
};
|
};
|
||||||
|
|
||||||
expect(resolveIncludes(obj, ctx)).toEqual({
|
expect(resolve(obj, files)).toEqual({
|
||||||
gateway: { port: 18789 },
|
gateway: { port: 18789 },
|
||||||
agents: [
|
agents: [
|
||||||
{ id: "mueller-screenshot", workspace: "~/clients/mueller/screenshot" },
|
{ id: "mueller-screenshot", workspace: "~/clients/mueller/screenshot" },
|
||||||
@@ -277,7 +235,7 @@ describe("real-world config patterns", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("supports modular config structure", () => {
|
it("supports modular config structure", () => {
|
||||||
const ctx = createMockContext({
|
const files = {
|
||||||
"/config/gateway.json": { gateway: { port: 18789, bind: "loopback" } },
|
"/config/gateway.json": { gateway: { port: 18789, bind: "loopback" } },
|
||||||
"/config/providers/whatsapp.json": {
|
"/config/providers/whatsapp.json": {
|
||||||
whatsapp: { dmPolicy: "pairing", allowFrom: ["+49123"] },
|
whatsapp: { dmPolicy: "pairing", allowFrom: ["+49123"] },
|
||||||
@@ -285,7 +243,7 @@ describe("real-world config patterns", () => {
|
|||||||
"/config/agents/defaults.json": {
|
"/config/agents/defaults.json": {
|
||||||
agents: { defaults: { sandbox: { mode: "all" } } },
|
agents: { defaults: { sandbox: { mode: "all" } } },
|
||||||
},
|
},
|
||||||
});
|
};
|
||||||
|
|
||||||
const obj = {
|
const obj = {
|
||||||
$include: [
|
$include: [
|
||||||
@@ -295,7 +253,7 @@ describe("real-world config patterns", () => {
|
|||||||
],
|
],
|
||||||
};
|
};
|
||||||
|
|
||||||
expect(resolveIncludes(obj, ctx)).toEqual({
|
expect(resolve(obj, files)).toEqual({
|
||||||
gateway: { port: 18789, bind: "loopback" },
|
gateway: { port: 18789, bind: "loopback" },
|
||||||
whatsapp: { dmPolicy: "pairing", allowFrom: ["+49123"] },
|
whatsapp: { dmPolicy: "pairing", allowFrom: ["+49123"] },
|
||||||
agents: { defaults: { sandbox: { mode: "all" } } },
|
agents: { defaults: { sandbox: { mode: "all" } } },
|
||||||
246
src/config/includes.ts
Normal file
246
src/config/includes.ts
Normal file
@@ -0,0 +1,246 @@
|
|||||||
|
/**
|
||||||
|
* Config includes: $include directive for modular configs
|
||||||
|
*
|
||||||
|
* Supports:
|
||||||
|
* - `{ "$include": "./path/to/file.json5" }` - single file include
|
||||||
|
* - `{ "$include": ["./a.json5", "./b.json5"] }` - deep merge multiple files
|
||||||
|
* - Nested includes up to MAX_INCLUDE_DEPTH levels
|
||||||
|
* - Circular include detection
|
||||||
|
*/
|
||||||
|
|
||||||
|
import fs from "node:fs";
|
||||||
|
import path from "node:path";
|
||||||
|
|
||||||
|
import JSON5 from "json5";
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Constants
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
export const INCLUDE_KEY = "$include";
|
||||||
|
export const MAX_INCLUDE_DEPTH = 10;
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Types
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
export type IncludeResolver = {
|
||||||
|
fsModule: typeof fs;
|
||||||
|
json5Module: typeof JSON5;
|
||||||
|
};
|
||||||
|
|
||||||
|
type IncludeContext = {
|
||||||
|
basePath: string;
|
||||||
|
visited: Set<string>;
|
||||||
|
depth: number;
|
||||||
|
resolver: IncludeResolver;
|
||||||
|
};
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Errors
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
export class ConfigIncludeError extends Error {
|
||||||
|
constructor(
|
||||||
|
message: string,
|
||||||
|
public readonly includePath: string,
|
||||||
|
public readonly cause?: Error,
|
||||||
|
) {
|
||||||
|
super(message);
|
||||||
|
this.name = "ConfigIncludeError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class CircularIncludeError extends ConfigIncludeError {
|
||||||
|
constructor(public readonly chain: string[]) {
|
||||||
|
super(
|
||||||
|
`Circular include detected: ${chain.join(" -> ")}`,
|
||||||
|
chain[chain.length - 1],
|
||||||
|
);
|
||||||
|
this.name = "CircularIncludeError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Utilities
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
function isPlainObject(value: unknown): value is Record<string, unknown> {
|
||||||
|
return (
|
||||||
|
typeof value === "object" &&
|
||||||
|
value !== null &&
|
||||||
|
!Array.isArray(value) &&
|
||||||
|
Object.prototype.toString.call(value) === "[object Object]"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Deep merge two values.
|
||||||
|
* - Arrays: concatenate
|
||||||
|
* - Objects: recursive merge
|
||||||
|
* - Primitives: source wins
|
||||||
|
*/
|
||||||
|
export function deepMerge(target: unknown, source: unknown): unknown {
|
||||||
|
if (Array.isArray(target) && Array.isArray(source)) {
|
||||||
|
return [...target, ...source];
|
||||||
|
}
|
||||||
|
if (isPlainObject(target) && isPlainObject(source)) {
|
||||||
|
const result: Record<string, unknown> = { ...target };
|
||||||
|
for (const key of Object.keys(source)) {
|
||||||
|
result[key] =
|
||||||
|
key in result ? deepMerge(result[key], source[key]) : source[key];
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
return source;
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveIncludePath(includePath: string, basePath: string): string {
|
||||||
|
if (path.isAbsolute(includePath)) {
|
||||||
|
return includePath;
|
||||||
|
}
|
||||||
|
return path.resolve(path.dirname(basePath), includePath);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Core Logic
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
function loadIncludeFile(includePath: string, ctx: IncludeContext): unknown {
|
||||||
|
const resolvedPath = resolveIncludePath(includePath, ctx.basePath);
|
||||||
|
const normalizedPath = path.normalize(resolvedPath);
|
||||||
|
|
||||||
|
if (ctx.visited.has(normalizedPath)) {
|
||||||
|
throw new CircularIncludeError([...ctx.visited, normalizedPath]);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ctx.depth >= MAX_INCLUDE_DEPTH) {
|
||||||
|
throw new ConfigIncludeError(
|
||||||
|
`Maximum include depth (${MAX_INCLUDE_DEPTH}) exceeded at: ${includePath}`,
|
||||||
|
includePath,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let raw: string;
|
||||||
|
try {
|
||||||
|
raw = ctx.resolver.fsModule.readFileSync(normalizedPath, "utf-8");
|
||||||
|
} catch (err) {
|
||||||
|
throw new ConfigIncludeError(
|
||||||
|
`Failed to read include file: ${includePath} (resolved: ${normalizedPath})`,
|
||||||
|
includePath,
|
||||||
|
err instanceof Error ? err : undefined,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let parsed: unknown;
|
||||||
|
try {
|
||||||
|
parsed = ctx.resolver.json5Module.parse(raw);
|
||||||
|
} catch (err) {
|
||||||
|
throw new ConfigIncludeError(
|
||||||
|
`Failed to parse include file: ${includePath} (resolved: ${normalizedPath})`,
|
||||||
|
includePath,
|
||||||
|
err instanceof Error ? err : undefined,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const newCtx: IncludeContext = {
|
||||||
|
...ctx,
|
||||||
|
basePath: normalizedPath,
|
||||||
|
visited: new Set([...ctx.visited, normalizedPath]),
|
||||||
|
depth: ctx.depth + 1,
|
||||||
|
};
|
||||||
|
|
||||||
|
return resolveIncludesInternal(parsed, newCtx);
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveIncludeDirective(
|
||||||
|
includeValue: unknown,
|
||||||
|
ctx: IncludeContext,
|
||||||
|
): unknown {
|
||||||
|
if (typeof includeValue === "string") {
|
||||||
|
return loadIncludeFile(includeValue, ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (Array.isArray(includeValue)) {
|
||||||
|
let result: unknown = {};
|
||||||
|
for (const item of includeValue) {
|
||||||
|
if (typeof item !== "string") {
|
||||||
|
throw new ConfigIncludeError(
|
||||||
|
`Invalid $include array item: expected string, got ${typeof item}`,
|
||||||
|
String(item),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
result = deepMerge(result, loadIncludeFile(item, ctx));
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new ConfigIncludeError(
|
||||||
|
`Invalid $include value: expected string or array of strings, got ${typeof includeValue}`,
|
||||||
|
String(includeValue),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveIncludesInternal(obj: unknown, ctx: IncludeContext): unknown {
|
||||||
|
if (Array.isArray(obj)) {
|
||||||
|
return obj.map((item) => resolveIncludesInternal(item, ctx));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isPlainObject(obj)) {
|
||||||
|
if (INCLUDE_KEY in obj) {
|
||||||
|
const includeValue = obj[INCLUDE_KEY];
|
||||||
|
const otherKeys = Object.keys(obj).filter((k) => k !== INCLUDE_KEY);
|
||||||
|
|
||||||
|
if (otherKeys.length > 0) {
|
||||||
|
const included = resolveIncludeDirective(includeValue, ctx);
|
||||||
|
const rest: Record<string, unknown> = {};
|
||||||
|
for (const key of otherKeys) {
|
||||||
|
rest[key] = resolveIncludesInternal(obj[key], ctx);
|
||||||
|
}
|
||||||
|
return deepMerge(included, rest);
|
||||||
|
}
|
||||||
|
|
||||||
|
return resolveIncludeDirective(includeValue, ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
const result: Record<string, unknown> = {};
|
||||||
|
for (const [key, value] of Object.entries(obj)) {
|
||||||
|
result[key] = resolveIncludesInternal(value, ctx);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
return obj;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Public API
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resolves all $include directives in a parsed config object.
|
||||||
|
*
|
||||||
|
* @param obj - Parsed config object (from JSON5.parse)
|
||||||
|
* @param configPath - Path to the main config file (for relative path resolution)
|
||||||
|
* @param resolver - Optional custom fs/json5 modules (for testing)
|
||||||
|
* @returns Config object with all includes resolved
|
||||||
|
*
|
||||||
|
* @example
|
||||||
|
* ```typescript
|
||||||
|
* const parsed = JSON5.parse(raw);
|
||||||
|
* const resolved = resolveConfigIncludes(parsed, "/path/to/config.json5");
|
||||||
|
* ```
|
||||||
|
*/
|
||||||
|
export function resolveConfigIncludes(
|
||||||
|
obj: unknown,
|
||||||
|
configPath: string,
|
||||||
|
resolver: IncludeResolver = { fsModule: fs, json5Module: JSON5 },
|
||||||
|
): unknown {
|
||||||
|
const ctx: IncludeContext = {
|
||||||
|
basePath: configPath,
|
||||||
|
visited: new Set([path.normalize(configPath)]),
|
||||||
|
depth: 0,
|
||||||
|
resolver,
|
||||||
|
};
|
||||||
|
return resolveIncludesInternal(obj, ctx);
|
||||||
|
}
|
||||||
256
src/config/io.ts
256
src/config/io.ts
@@ -22,6 +22,10 @@ import {
|
|||||||
applySessionDefaults,
|
applySessionDefaults,
|
||||||
applyTalkApiKey,
|
applyTalkApiKey,
|
||||||
} from "./defaults.js";
|
} from "./defaults.js";
|
||||||
|
import {
|
||||||
|
ConfigIncludeError,
|
||||||
|
resolveConfigIncludes,
|
||||||
|
} from "./includes.js";
|
||||||
import { findLegacyConfigIssues } from "./legacy.js";
|
import { findLegacyConfigIssues } from "./legacy.js";
|
||||||
import { resolveConfigPath, resolveStateDir } from "./paths.js";
|
import { resolveConfigPath, resolveStateDir } from "./paths.js";
|
||||||
import { applyConfigOverrides } from "./runtime-overrides.js";
|
import { applyConfigOverrides } from "./runtime-overrides.js";
|
||||||
@@ -33,6 +37,12 @@ import type {
|
|||||||
import { validateConfigObject } from "./validation.js";
|
import { validateConfigObject } from "./validation.js";
|
||||||
import { ClawdbotSchema } from "./zod-schema.js";
|
import { ClawdbotSchema } from "./zod-schema.js";
|
||||||
|
|
||||||
|
// Re-export for backwards compatibility
|
||||||
|
export {
|
||||||
|
CircularIncludeError,
|
||||||
|
ConfigIncludeError,
|
||||||
|
} from "./includes.js";
|
||||||
|
|
||||||
const SHELL_ENV_EXPECTED_KEYS = [
|
const SHELL_ENV_EXPECTED_KEYS = [
|
||||||
"OPENAI_API_KEY",
|
"OPENAI_API_KEY",
|
||||||
"ANTHROPIC_API_KEY",
|
"ANTHROPIC_API_KEY",
|
||||||
@@ -129,240 +139,6 @@ export function parseConfigJson5(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ============================================================================
|
|
||||||
// Config Includes ($include directive)
|
|
||||||
// ============================================================================
|
|
||||||
|
|
||||||
const INCLUDE_KEY = "$include";
|
|
||||||
const MAX_INCLUDE_DEPTH = 10;
|
|
||||||
|
|
||||||
export class ConfigIncludeError extends Error {
|
|
||||||
constructor(
|
|
||||||
message: string,
|
|
||||||
public readonly includePath: string,
|
|
||||||
public readonly cause?: Error,
|
|
||||||
) {
|
|
||||||
super(message);
|
|
||||||
this.name = "ConfigIncludeError";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export class CircularIncludeError extends ConfigIncludeError {
|
|
||||||
constructor(
|
|
||||||
public readonly chain: string[],
|
|
||||||
) {
|
|
||||||
super(
|
|
||||||
`Circular include detected: ${chain.join(" -> ")}`,
|
|
||||||
chain[chain.length - 1],
|
|
||||||
);
|
|
||||||
this.name = "CircularIncludeError";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type IncludeContext = {
|
|
||||||
basePath: string;
|
|
||||||
visited: Set<string>;
|
|
||||||
depth: number;
|
|
||||||
fsModule: typeof fs;
|
|
||||||
json5Module: typeof JSON5;
|
|
||||||
logger: Pick<typeof console, "error" | "warn">;
|
|
||||||
};
|
|
||||||
|
|
||||||
function isPlainObject(value: unknown): value is Record<string, unknown> {
|
|
||||||
return (
|
|
||||||
typeof value === "object" &&
|
|
||||||
value !== null &&
|
|
||||||
!Array.isArray(value) &&
|
|
||||||
Object.prototype.toString.call(value) === "[object Object]"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function deepMerge(target: unknown, source: unknown): unknown {
|
|
||||||
if (Array.isArray(target) && Array.isArray(source)) {
|
|
||||||
return [...target, ...source];
|
|
||||||
}
|
|
||||||
if (isPlainObject(target) && isPlainObject(source)) {
|
|
||||||
const result: Record<string, unknown> = { ...target };
|
|
||||||
for (const key of Object.keys(source)) {
|
|
||||||
if (key in result) {
|
|
||||||
result[key] = deepMerge(result[key], source[key]);
|
|
||||||
} else {
|
|
||||||
result[key] = source[key];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
return source;
|
|
||||||
}
|
|
||||||
|
|
||||||
function resolveIncludePath(includePath: string, basePath: string): string {
|
|
||||||
if (path.isAbsolute(includePath)) {
|
|
||||||
return includePath;
|
|
||||||
}
|
|
||||||
const baseDir = path.dirname(basePath);
|
|
||||||
return path.resolve(baseDir, includePath);
|
|
||||||
}
|
|
||||||
|
|
||||||
function loadIncludeFile(
|
|
||||||
includePath: string,
|
|
||||||
ctx: IncludeContext,
|
|
||||||
): unknown {
|
|
||||||
const resolvedPath = resolveIncludePath(includePath, ctx.basePath);
|
|
||||||
const normalizedPath = path.normalize(resolvedPath);
|
|
||||||
|
|
||||||
// Check for circular includes
|
|
||||||
if (ctx.visited.has(normalizedPath)) {
|
|
||||||
throw new CircularIncludeError([...ctx.visited, normalizedPath]);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check depth limit
|
|
||||||
if (ctx.depth >= MAX_INCLUDE_DEPTH) {
|
|
||||||
throw new ConfigIncludeError(
|
|
||||||
`Maximum include depth (${MAX_INCLUDE_DEPTH}) exceeded at: ${includePath}`,
|
|
||||||
includePath,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Read and parse the file
|
|
||||||
let raw: string;
|
|
||||||
try {
|
|
||||||
raw = ctx.fsModule.readFileSync(normalizedPath, "utf-8");
|
|
||||||
} catch (err) {
|
|
||||||
throw new ConfigIncludeError(
|
|
||||||
`Failed to read include file: ${includePath} (resolved: ${normalizedPath})`,
|
|
||||||
includePath,
|
|
||||||
err instanceof Error ? err : undefined,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
let parsed: unknown;
|
|
||||||
try {
|
|
||||||
parsed = ctx.json5Module.parse(raw);
|
|
||||||
} catch (err) {
|
|
||||||
throw new ConfigIncludeError(
|
|
||||||
`Failed to parse include file: ${includePath} (resolved: ${normalizedPath})`,
|
|
||||||
includePath,
|
|
||||||
err instanceof Error ? err : undefined,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Recursively resolve includes in the loaded file
|
|
||||||
const newCtx: IncludeContext = {
|
|
||||||
...ctx,
|
|
||||||
basePath: normalizedPath,
|
|
||||||
visited: new Set([...ctx.visited, normalizedPath]),
|
|
||||||
depth: ctx.depth + 1,
|
|
||||||
};
|
|
||||||
|
|
||||||
return resolveIncludes(parsed, newCtx);
|
|
||||||
}
|
|
||||||
|
|
||||||
function resolveIncludeDirective(
|
|
||||||
includeValue: unknown,
|
|
||||||
ctx: IncludeContext,
|
|
||||||
): unknown {
|
|
||||||
if (typeof includeValue === "string") {
|
|
||||||
// Single file include
|
|
||||||
return loadIncludeFile(includeValue, ctx);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (Array.isArray(includeValue)) {
|
|
||||||
// Multiple files - deep merge them
|
|
||||||
let result: unknown = {};
|
|
||||||
for (const item of includeValue) {
|
|
||||||
if (typeof item !== "string") {
|
|
||||||
throw new ConfigIncludeError(
|
|
||||||
`Invalid $include array item: expected string, got ${typeof item}`,
|
|
||||||
String(item),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
const loaded = loadIncludeFile(item, ctx);
|
|
||||||
result = deepMerge(result, loaded);
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new ConfigIncludeError(
|
|
||||||
`Invalid $include value: expected string or array of strings, got ${typeof includeValue}`,
|
|
||||||
String(includeValue),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Recursively resolves $include directives in the config object.
|
|
||||||
*
|
|
||||||
* Supports:
|
|
||||||
* - `{ "$include": "./path/to/file.json5" }` - replaces object with file contents
|
|
||||||
* - `{ "$include": ["./a.json5", "./b.json5"] }` - deep merges multiple files
|
|
||||||
* - Nested includes up to MAX_INCLUDE_DEPTH levels
|
|
||||||
*
|
|
||||||
* @example
|
|
||||||
* ```json5
|
|
||||||
* // clawdbot.json
|
|
||||||
* {
|
|
||||||
* gateway: { port: 18789 },
|
|
||||||
* agents: { "$include": "./agents.json5" },
|
|
||||||
* broadcast: { "$include": ["./clients/a.json5", "./clients/b.json5"] }
|
|
||||||
* }
|
|
||||||
* ```
|
|
||||||
*/
|
|
||||||
export function resolveIncludes(
|
|
||||||
obj: unknown,
|
|
||||||
ctx: IncludeContext,
|
|
||||||
): unknown {
|
|
||||||
if (Array.isArray(obj)) {
|
|
||||||
return obj.map((item) => resolveIncludes(item, ctx));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (isPlainObject(obj)) {
|
|
||||||
// Check if this object is an include directive
|
|
||||||
if (INCLUDE_KEY in obj) {
|
|
||||||
const includeValue = obj[INCLUDE_KEY];
|
|
||||||
const otherKeys = Object.keys(obj).filter((k) => k !== INCLUDE_KEY);
|
|
||||||
|
|
||||||
if (otherKeys.length > 0) {
|
|
||||||
// Has other keys besides $include - merge include result with them
|
|
||||||
const included = resolveIncludeDirective(includeValue, ctx);
|
|
||||||
const rest: Record<string, unknown> = {};
|
|
||||||
for (const key of otherKeys) {
|
|
||||||
rest[key] = resolveIncludes(obj[key], ctx);
|
|
||||||
}
|
|
||||||
return deepMerge(included, rest);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Pure include directive
|
|
||||||
return resolveIncludeDirective(includeValue, ctx);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Regular object - recurse into properties
|
|
||||||
const result: Record<string, unknown> = {};
|
|
||||||
for (const [key, value] of Object.entries(obj)) {
|
|
||||||
result[key] = resolveIncludes(value, ctx);
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Primitives pass through unchanged
|
|
||||||
return obj;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates an include context for resolving $include directives.
|
|
||||||
*/
|
|
||||||
function createIncludeContext(
|
|
||||||
configPath: string,
|
|
||||||
deps: Required<ConfigIoDeps>,
|
|
||||||
): IncludeContext {
|
|
||||||
return {
|
|
||||||
basePath: configPath,
|
|
||||||
visited: new Set([path.normalize(configPath)]),
|
|
||||||
depth: 0,
|
|
||||||
fsModule: deps.fs,
|
|
||||||
json5Module: deps.json5,
|
|
||||||
logger: deps.logger,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
export function createConfigIO(overrides: ConfigIoDeps = {}) {
|
export function createConfigIO(overrides: ConfigIoDeps = {}) {
|
||||||
const deps = normalizeDeps(overrides);
|
const deps = normalizeDeps(overrides);
|
||||||
const configPath = resolveConfigPathForDeps(deps);
|
const configPath = resolveConfigPathForDeps(deps);
|
||||||
@@ -385,8 +161,10 @@ export function createConfigIO(overrides: ConfigIoDeps = {}) {
|
|||||||
const parsed = deps.json5.parse(raw);
|
const parsed = deps.json5.parse(raw);
|
||||||
|
|
||||||
// Resolve $include directives before validation
|
// Resolve $include directives before validation
|
||||||
const includeCtx = createIncludeContext(configPath, deps);
|
const resolved = resolveConfigIncludes(parsed, configPath, {
|
||||||
const resolved = resolveIncludes(parsed, includeCtx);
|
fsModule: deps.fs,
|
||||||
|
json5Module: deps.json5,
|
||||||
|
});
|
||||||
|
|
||||||
warnOnConfigMiskeys(resolved, deps.logger);
|
warnOnConfigMiskeys(resolved, deps.logger);
|
||||||
if (typeof resolved !== "object" || resolved === null) return {};
|
if (typeof resolved !== "object" || resolved === null) return {};
|
||||||
@@ -488,8 +266,10 @@ export function createConfigIO(overrides: ConfigIoDeps = {}) {
|
|||||||
// Resolve $include directives
|
// Resolve $include directives
|
||||||
let resolved: unknown;
|
let resolved: unknown;
|
||||||
try {
|
try {
|
||||||
const includeCtx = createIncludeContext(configPath, deps);
|
resolved = resolveConfigIncludes(parsedRes.parsed, configPath, {
|
||||||
resolved = resolveIncludes(parsedRes.parsed, includeCtx);
|
fsModule: deps.fs,
|
||||||
|
json5Module: deps.json5,
|
||||||
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const message =
|
const message =
|
||||||
err instanceof ConfigIncludeError
|
err instanceof ConfigIncludeError
|
||||||
|
|||||||
Reference in New Issue
Block a user