let5see/clawdbot
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

feat(config): add env var substitution in config values

Browse Source 
Support ${VAR_NAME} syntax in any config string value, substituted at
config load time. Useful for referencing API keys and secrets from
environment variables without hardcoding them in the config file.

- Only uppercase env vars matched: [A-Z_][A-Z0-9_]*
- Missing/empty env vars throw MissingEnvVarError with path context
- Escape with $${VAR} to output literal ${VAR}
- Works with $include (included files also get substitution)

Closes #1009
This commit is contained in:
Sebastian
2026-01-16 16:32:07 -05:00
parent d887027e95
commit a36735b913
5 changed files with 469 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
docs/environment.md
View File

@@ -53,6 +53,24 @@ Env var equivalents:
- `CLAWDBOT_LOAD_SHELL_ENV=1`
- `CLAWDBOT_SHELL_ENV_TIMEOUT_MS=15000`
## Env var substitution in config
You can reference env vars directly in config string values using `${VAR_NAME}` syntax:
```json5
{
models: {
providers: {
"vercel-gateway": {
apiKey: "${VERCEL_GATEWAY_API_KEY}"
}
}
}
}
```
See [Configuration: Env var substitution](/gateway/configuration#env-var-substitution-in-config) for full details.
## Related
- [Gateway configuration](/gateway/configuration)

42
docs/gateway/configuration.md
View File

@@ -283,6 +283,48 @@ Env var equivalent:
- `CLAWDBOT_LOAD_SHELL_ENV=1`
- `CLAWDBOT_SHELL_ENV_TIMEOUT_MS=15000`
### Env var substitution in config
You can reference environment variables directly in any config string value using
`${VAR_NAME}` syntax. Variables are substituted at config load time, before validation.
```json5
{
models: {
providers: {
"vercel-gateway": {
apiKey: "${VERCEL_GATEWAY_API_KEY}"
}
}
},
gateway: {
auth: {
token: "${CLAWDBOT_GATEWAY_TOKEN}"
}
}
}
```
**Rules:**
- Only uppercase env var names are matched: `[A-Z_][A-Z0-9_]*`
- Missing or empty env vars throw an error at config load
- Escape with `$${VAR}` to output a literal `${VAR}`
- Works with `$include` (included files also get substitution)
**Inline substitution:**
```json5
{
models: {
providers: {
custom: {
baseUrl: "${CUSTOM_API_BASE}/v1" // → "https://api.example.com/v1"
}
}
}
}
```
### Auth storage (OAuth + API keys)
Clawdbot stores **per-agent** auth profiles (OAuth + API keys) in:

275
src/config/env-substitution.test.ts Normal file
View File

@@ -0,0 +1,275 @@
import { describe, expect, it } from "vitest";
import { MissingEnvVarError, resolveConfigEnvVars } from "./env-substitution.js";
describe("resolveConfigEnvVars", () => {
describe("basic substitution", () => {
it("substitutes a single env var", () => {
const result = resolveConfigEnvVars({ key: "${FOO}" }, { FOO: "bar" });
expect(result).toEqual({ key: "bar" });
});
it("substitutes multiple different env vars in same string", () => {
const result = resolveConfigEnvVars({ key: "${A}/${B}" }, { A: "x", B: "y" });
expect(result).toEqual({ key: "x/y" });
});
it("substitutes inline with prefix and suffix", () => {
const result = resolveConfigEnvVars({ key: "prefix-${FOO}-suffix" }, { FOO: "bar" });
expect(result).toEqual({ key: "prefix-bar-suffix" });
});
it("substitutes same var multiple times", () => {
const result = resolveConfigEnvVars({ key: "${FOO}:${FOO}" }, { FOO: "bar" });
expect(result).toEqual({ key: "bar:bar" });
});
});
describe("nested structures", () => {
it("substitutes in nested objects", () => {
const result = resolveConfigEnvVars(
{
outer: {
inner: {
key: "${API_KEY}",
},
},
},
{ API_KEY: "secret123" },
);
expect(result).toEqual({
outer: {
inner: {
key: "secret123",
},
},
});
});
it("substitutes in arrays", () => {
const result = resolveConfigEnvVars(
{ items: ["${A}", "${B}", "${C}"] },
{ A: "1", B: "2", C: "3" },
);
expect(result).toEqual({ items: ["1", "2", "3"] });
});
it("substitutes in deeply nested arrays and objects", () => {
const result = resolveConfigEnvVars(
{
providers: [
{ name: "openai", apiKey: "${OPENAI_KEY}" },
{ name: "anthropic", apiKey: "${ANTHROPIC_KEY}" },
],
},
{ OPENAI_KEY: "sk-xxx", ANTHROPIC_KEY: "sk-yyy" },
);
expect(result).toEqual({
providers: [
{ name: "openai", apiKey: "sk-xxx" },
{ name: "anthropic", apiKey: "sk-yyy" },
],
});
});
});
describe("missing env var handling", () => {
it("throws MissingEnvVarError for missing env var", () => {
expect(() => resolveConfigEnvVars({ key: "${MISSING}" }, {})).toThrow(MissingEnvVarError);
});
it("includes var name in error", () => {
try {
resolveConfigEnvVars({ key: "${MISSING_VAR}" }, {});
throw new Error("Expected to throw");
} catch (err) {
expect(err).toBeInstanceOf(MissingEnvVarError);
const error = err as MissingEnvVarError;
expect(error.varName).toBe("MISSING_VAR");
}
});
it("includes config path in error", () => {
try {
resolveConfigEnvVars({ outer: { inner: { key: "${MISSING}" } } }, {});
throw new Error("Expected to throw");
} catch (err) {
expect(err).toBeInstanceOf(MissingEnvVarError);
const error = err as MissingEnvVarError;
expect(error.configPath).toBe("outer.inner.key");
}
});
it("includes array index in config path", () => {
try {
resolveConfigEnvVars({ items: ["ok", "${MISSING}"] }, { OK: "val" });
throw new Error("Expected to throw");
} catch (err) {
expect(err).toBeInstanceOf(MissingEnvVarError);
const error = err as MissingEnvVarError;
expect(error.configPath).toBe("items[1]");
}
});
it("treats empty string env var as missing", () => {
expect(() => resolveConfigEnvVars({ key: "${EMPTY}" }, { EMPTY: "" })).toThrow(
MissingEnvVarError,
);
});
});
describe("escape syntax", () => {
it("outputs literal ${VAR} when escaped with $$", () => {
const result = resolveConfigEnvVars({ key: "$${VAR}" }, { VAR: "value" });
expect(result).toEqual({ key: "${VAR}" });
});
it("handles mix of escaped and unescaped", () => {
const result = resolveConfigEnvVars({ key: "${REAL}/$${LITERAL}" }, { REAL: "resolved" });
expect(result).toEqual({ key: "resolved/${LITERAL}" });
});
it("handles multiple escaped vars", () => {
const result = resolveConfigEnvVars({ key: "$${A}:$${B}" }, {});
expect(result).toEqual({ key: "${A}:${B}" });
});
});
describe("non-matching patterns unchanged", () => {
it("leaves $VAR (no braces) unchanged", () => {
const result = resolveConfigEnvVars({ key: "$VAR" }, { VAR: "value" });
expect(result).toEqual({ key: "$VAR" });
});
it("leaves ${lowercase} unchanged (uppercase only)", () => {
const result = resolveConfigEnvVars({ key: "${lowercase}" }, { lowercase: "value" });
expect(result).toEqual({ key: "${lowercase}" });
});
it("leaves ${MixedCase} unchanged", () => {
const result = resolveConfigEnvVars({ key: "${MixedCase}" }, { MixedCase: "value" });
expect(result).toEqual({ key: "${MixedCase}" });
});
it("leaves ${123INVALID} unchanged (must start with letter or underscore)", () => {
const result = resolveConfigEnvVars({ key: "${123INVALID}" }, {});
expect(result).toEqual({ key: "${123INVALID}" });
});
it("substitutes ${_UNDERSCORE_START} (valid)", () => {
const result = resolveConfigEnvVars(
{ key: "${_UNDERSCORE_START}" },
{ _UNDERSCORE_START: "valid" },
);
expect(result).toEqual({ key: "valid" });
});
it("substitutes ${VAR_WITH_NUMBERS_123} (valid)", () => {
const result = resolveConfigEnvVars(
{ key: "${VAR_WITH_NUMBERS_123}" },
{ VAR_WITH_NUMBERS_123: "valid" },
);
expect(result).toEqual({ key: "valid" });
});
});
describe("passthrough behavior", () => {
it("passes through primitives unchanged", () => {
expect(resolveConfigEnvVars("hello", {})).toBe("hello");
expect(resolveConfigEnvVars(42, {})).toBe(42);
expect(resolveConfigEnvVars(true, {})).toBe(true);
expect(resolveConfigEnvVars(null, {})).toBe(null);
});
it("passes through empty object", () => {
expect(resolveConfigEnvVars({}, {})).toEqual({});
});
it("passes through empty array", () => {
expect(resolveConfigEnvVars([], {})).toEqual([]);
});
it("passes through non-string values in objects", () => {
const result = resolveConfigEnvVars({ num: 42, bool: true, nil: null, arr: [1, 2] }, {});
expect(result).toEqual({ num: 42, bool: true, nil: null, arr: [1, 2] });
});
});
describe("real-world config patterns", () => {
it("substitutes API keys in provider config", () => {
const config = {
models: {
providers: {
"vercel-gateway": {
apiKey: "${VERCEL_GATEWAY_API_KEY}",
},
openai: {
apiKey: "${OPENAI_API_KEY}",
},
},
},
};
const env = {
VERCEL_GATEWAY_API_KEY: "vg_key_123",
OPENAI_API_KEY: "sk-xxx",
};
const result = resolveConfigEnvVars(config, env);
expect(result).toEqual({
models: {
providers: {
"vercel-gateway": {
apiKey: "vg_key_123",
},
openai: {
apiKey: "sk-xxx",
},
},
},
});
});
it("substitutes gateway auth token", () => {
const config = {
gateway: {
auth: {
token: "${CLAWDBOT_GATEWAY_TOKEN}",
},
},
};
const result = resolveConfigEnvVars(config, {
CLAWDBOT_GATEWAY_TOKEN: "secret-token",
});
expect(result).toEqual({
gateway: {
auth: {
token: "secret-token",
},
},
});
});
it("substitutes base URL with env var", () => {
const config = {
models: {
providers: {
custom: {
baseUrl: "${CUSTOM_API_BASE}/v1",
},
},
},
};
const result = resolveConfigEnvVars(config, {
CUSTOM_API_BASE: "https://api.example.com",
});
expect(result).toEqual({
models: {
providers: {
custom: {
baseUrl: "https://api.example.com/v1",
},
},
},
});
});
});
});

103
src/config/env-substitution.ts Normal file
View File

@@ -0,0 +1,103 @@
/**
* Environment variable substitution for config values.
*
* Supports `${VAR_NAME}` syntax in string values, substituted at config load time.
* - Only uppercase env vars are matched: `[A-Z_][A-Z0-9_]*`
* - Escape with `$${}` to output literal `${}`
* - Missing env vars throw `MissingEnvVarError` with context
*
* @example
* ```json5
* {
* models: {
* providers: {
* "vercel-gateway": {
* apiKey: "${VERCEL_GATEWAY_API_KEY}"
* }
* }
* }
* }
* ```
*/
// Pattern for valid uppercase env var names: starts with letter or underscore,
// followed by letters, numbers, or underscores (all uppercase)
const ENV_VAR_PATTERN = /\$\{([A-Z_][A-Z0-9_]*)\}/g;
// Pattern for escaped env vars: $${...} -> ${...}
const ESCAPED_PATTERN = /\$\$\{([A-Z_][A-Z0-9_]*)\}/g;
export class MissingEnvVarError extends Error {
constructor(
public readonly varName: string,
public readonly configPath: string,
) {
super(`Missing env var "${varName}" referenced at config path: ${configPath}`);
this.name = "MissingEnvVarError";
}
}
function isPlainObject(value: unknown): value is Record<string, unknown> {
return (
typeof value === "object" &&
value !== null &&
!Array.isArray(value) &&
Object.prototype.toString.call(value) === "[object Object]"
);
}
function substituteString(value: string, env: NodeJS.ProcessEnv, configPath: string): string {
// First pass: substitute real env vars
const substituted = value.replace(ENV_VAR_PATTERN, (match, varName: string) => {
// Check if this is actually an escaped var (preceded by $)
// We handle this in the second pass, so check the original string
const idx = value.indexOf(match);
if (idx > 0 && value[idx - 1] === "$") {
// This will be handled by ESCAPED_PATTERN, return as-is for now
return match;
}
const envValue = env[varName];
if (envValue === undefined || envValue === "") {
throw new MissingEnvVarError(varName, configPath);
}
return envValue;
});
// Second pass: convert escaped $${VAR} to literal ${VAR}
return substituted.replace(ESCAPED_PATTERN, (_, varName: string) => `\${${varName}}`);
}
function substituteAny(value: unknown, env: NodeJS.ProcessEnv, path: string): unknown {
if (typeof value === "string") {
return substituteString(value, env, path);
}
if (Array.isArray(value)) {
return value.map((item, index) => substituteAny(item, env, `${path}[${index}]`));
}
if (isPlainObject(value)) {
const result: Record<string, unknown> = {};
for (const [key, val] of Object.entries(value)) {
const childPath = path ? `${path}.${key}` : key;
result[key] = substituteAny(val, env, childPath);
}
return result;
}
// Primitives (number, boolean, null) pass through unchanged
return value;
}
/**
* Resolves `${VAR_NAME}` environment variable references in config values.
*
* @param obj - The parsed config object (after JSON5 parse and $include resolution)
* @param env - Environment variables to use for substitution (defaults to process.env)
* @returns The config object with env vars substituted
* @throws {MissingEnvVarError} If a referenced env var is not set or empty
*/
export function resolveConfigEnvVars(obj: unknown, env: NodeJS.ProcessEnv = process.env): unknown {
return substituteAny(obj, env, "");
}

35
src/config/io.ts
View File

@@ -19,6 +19,7 @@ import {
applySessionDefaults,
applyTalkApiKey,
} from "./defaults.js";
import { MissingEnvVarError, resolveConfigEnvVars } from "./env-substitution.js";
import { ConfigIncludeError, resolveConfigIncludes } from "./includes.js";
import { applyLegacyMigrations, findLegacyConfigIssues } from "./legacy.js";
import { normalizeConfigPaths } from "./normalize-paths.js";
@@ -30,6 +31,7 @@ import { ClawdbotSchema } from "./zod-schema.js";
// Re-export for backwards compatibility
export { CircularIncludeError, ConfigIncludeError } from "./includes.js";
export { MissingEnvVarError } from "./env-substitution.js";
const SHELL_ENV_EXPECTED_KEYS = [
"OPENAI_API_KEY",
@@ -219,8 +221,11 @@ export function createConfigIO(overrides: ConfigIoDeps = {}) {
parseJson: (raw) => deps.json5.parse(raw),
});
const migrated = applyLegacyMigrations(resolved);
const resolvedConfig = migrated.next ?? resolved;
// Substitute ${VAR} env var references
const substituted = resolveConfigEnvVars(resolved, deps.env);
const migrated = applyLegacyMigrations(substituted);
const resolvedConfig = migrated.next ?? substituted;
warnOnConfigMiskeys(resolvedConfig, deps.logger);
if (typeof resolvedConfig !== "object" || resolvedConfig === null) return {};
const validated = ClawdbotSchema.safeParse(resolvedConfig);
@@ -346,8 +351,30 @@ export function createConfigIO(overrides: ConfigIoDeps = {}) {
};
}
const migrated = applyLegacyMigrations(resolved);
const resolvedConfigRaw = migrated.next ?? resolved;
// Substitute ${VAR} env var references
let substituted: unknown;
try {
substituted = resolveConfigEnvVars(resolved, deps.env);
} catch (err) {
const message =
err instanceof MissingEnvVarError
? err.message
: `Env var substitution failed: ${String(err)}`;
return {
path: configPath,
exists: true,
raw,
parsed: parsedRes.parsed,
valid: false,
config: {},
hash,
issues: [{ path: "", message }],
legacyIssues: [],
};
}
const migrated = applyLegacyMigrations(substituted);
const resolvedConfigRaw = migrated.next ?? substituted;
const legacyIssues = findLegacyConfigIssues(resolvedConfigRaw);
const validated = validateConfigObject(resolvedConfigRaw);