diff --git a/docs/environment.md b/docs/environment.md index 60d2921ff..4fb49b6b2 100644 --- a/docs/environment.md +++ b/docs/environment.md @@ -53,6 +53,24 @@ Env var equivalents: - `CLAWDBOT_LOAD_SHELL_ENV=1` - `CLAWDBOT_SHELL_ENV_TIMEOUT_MS=15000` +## Env var substitution in config + +You can reference env vars directly in config string values using `${VAR_NAME}` syntax: + +```json5 +{ + models: { + providers: { + "vercel-gateway": { + apiKey: "${VERCEL_GATEWAY_API_KEY}" + } + } + } +} +``` + +See [Configuration: Env var substitution](/gateway/configuration#env-var-substitution-in-config) for full details. + ## Related - [Gateway configuration](/gateway/configuration) diff --git a/docs/gateway/configuration.md b/docs/gateway/configuration.md index 5b9a900fd..3793cc3d0 100644 --- a/docs/gateway/configuration.md +++ b/docs/gateway/configuration.md @@ -283,6 +283,48 @@ Env var equivalent: - `CLAWDBOT_LOAD_SHELL_ENV=1` - `CLAWDBOT_SHELL_ENV_TIMEOUT_MS=15000` +### Env var substitution in config + +You can reference environment variables directly in any config string value using +`${VAR_NAME}` syntax. Variables are substituted at config load time, before validation. + +```json5 +{ + models: { + providers: { + "vercel-gateway": { + apiKey: "${VERCEL_GATEWAY_API_KEY}" + } + } + }, + gateway: { + auth: { + token: "${CLAWDBOT_GATEWAY_TOKEN}" + } + } +} +``` + +**Rules:** +- Only uppercase env var names are matched: `[A-Z_][A-Z0-9_]*` +- Missing or empty env vars throw an error at config load +- Escape with `$${VAR}` to output a literal `${VAR}` +- Works with `$include` (included files also get substitution) + +**Inline substitution:** + +```json5 +{ + models: { + providers: { + custom: { + baseUrl: "${CUSTOM_API_BASE}/v1" // → "https://api.example.com/v1" + } + } + } +} +``` + ### Auth storage (OAuth + API keys) Clawdbot stores **per-agent** auth profiles (OAuth + API keys) in: diff --git a/src/config/env-substitution.test.ts b/src/config/env-substitution.test.ts new file mode 100644 index 000000000..5027c237a --- /dev/null +++ b/src/config/env-substitution.test.ts @@ -0,0 +1,275 @@ +import { describe, expect, it } from "vitest"; + +import { MissingEnvVarError, resolveConfigEnvVars } from "./env-substitution.js"; + +describe("resolveConfigEnvVars", () => { + describe("basic substitution", () => { + it("substitutes a single env var", () => { + const result = resolveConfigEnvVars({ key: "${FOO}" }, { FOO: "bar" }); + expect(result).toEqual({ key: "bar" }); + }); + + it("substitutes multiple different env vars in same string", () => { + const result = resolveConfigEnvVars({ key: "${A}/${B}" }, { A: "x", B: "y" }); + expect(result).toEqual({ key: "x/y" }); + }); + + it("substitutes inline with prefix and suffix", () => { + const result = resolveConfigEnvVars({ key: "prefix-${FOO}-suffix" }, { FOO: "bar" }); + expect(result).toEqual({ key: "prefix-bar-suffix" }); + }); + + it("substitutes same var multiple times", () => { + const result = resolveConfigEnvVars({ key: "${FOO}:${FOO}" }, { FOO: "bar" }); + expect(result).toEqual({ key: "bar:bar" }); + }); + }); + + describe("nested structures", () => { + it("substitutes in nested objects", () => { + const result = resolveConfigEnvVars( + { + outer: { + inner: { + key: "${API_KEY}", + }, + }, + }, + { API_KEY: "secret123" }, + ); + expect(result).toEqual({ + outer: { + inner: { + key: "secret123", + }, + }, + }); + }); + + it("substitutes in arrays", () => { + const result = resolveConfigEnvVars( + { items: ["${A}", "${B}", "${C}"] }, + { A: "1", B: "2", C: "3" }, + ); + expect(result).toEqual({ items: ["1", "2", "3"] }); + }); + + it("substitutes in deeply nested arrays and objects", () => { + const result = resolveConfigEnvVars( + { + providers: [ + { name: "openai", apiKey: "${OPENAI_KEY}" }, + { name: "anthropic", apiKey: "${ANTHROPIC_KEY}" }, + ], + }, + { OPENAI_KEY: "sk-xxx", ANTHROPIC_KEY: "sk-yyy" }, + ); + expect(result).toEqual({ + providers: [ + { name: "openai", apiKey: "sk-xxx" }, + { name: "anthropic", apiKey: "sk-yyy" }, + ], + }); + }); + }); + + describe("missing env var handling", () => { + it("throws MissingEnvVarError for missing env var", () => { + expect(() => resolveConfigEnvVars({ key: "${MISSING}" }, {})).toThrow(MissingEnvVarError); + }); + + it("includes var name in error", () => { + try { + resolveConfigEnvVars({ key: "${MISSING_VAR}" }, {}); + throw new Error("Expected to throw"); + } catch (err) { + expect(err).toBeInstanceOf(MissingEnvVarError); + const error = err as MissingEnvVarError; + expect(error.varName).toBe("MISSING_VAR"); + } + }); + + it("includes config path in error", () => { + try { + resolveConfigEnvVars({ outer: { inner: { key: "${MISSING}" } } }, {}); + throw new Error("Expected to throw"); + } catch (err) { + expect(err).toBeInstanceOf(MissingEnvVarError); + const error = err as MissingEnvVarError; + expect(error.configPath).toBe("outer.inner.key"); + } + }); + + it("includes array index in config path", () => { + try { + resolveConfigEnvVars({ items: ["ok", "${MISSING}"] }, { OK: "val" }); + throw new Error("Expected to throw"); + } catch (err) { + expect(err).toBeInstanceOf(MissingEnvVarError); + const error = err as MissingEnvVarError; + expect(error.configPath).toBe("items[1]"); + } + }); + + it("treats empty string env var as missing", () => { + expect(() => resolveConfigEnvVars({ key: "${EMPTY}" }, { EMPTY: "" })).toThrow( + MissingEnvVarError, + ); + }); + }); + + describe("escape syntax", () => { + it("outputs literal ${VAR} when escaped with $$", () => { + const result = resolveConfigEnvVars({ key: "$${VAR}" }, { VAR: "value" }); + expect(result).toEqual({ key: "${VAR}" }); + }); + + it("handles mix of escaped and unescaped", () => { + const result = resolveConfigEnvVars({ key: "${REAL}/$${LITERAL}" }, { REAL: "resolved" }); + expect(result).toEqual({ key: "resolved/${LITERAL}" }); + }); + + it("handles multiple escaped vars", () => { + const result = resolveConfigEnvVars({ key: "$${A}:$${B}" }, {}); + expect(result).toEqual({ key: "${A}:${B}" }); + }); + }); + + describe("non-matching patterns unchanged", () => { + it("leaves $VAR (no braces) unchanged", () => { + const result = resolveConfigEnvVars({ key: "$VAR" }, { VAR: "value" }); + expect(result).toEqual({ key: "$VAR" }); + }); + + it("leaves ${lowercase} unchanged (uppercase only)", () => { + const result = resolveConfigEnvVars({ key: "${lowercase}" }, { lowercase: "value" }); + expect(result).toEqual({ key: "${lowercase}" }); + }); + + it("leaves ${MixedCase} unchanged", () => { + const result = resolveConfigEnvVars({ key: "${MixedCase}" }, { MixedCase: "value" }); + expect(result).toEqual({ key: "${MixedCase}" }); + }); + + it("leaves ${123INVALID} unchanged (must start with letter or underscore)", () => { + const result = resolveConfigEnvVars({ key: "${123INVALID}" }, {}); + expect(result).toEqual({ key: "${123INVALID}" }); + }); + + it("substitutes ${_UNDERSCORE_START} (valid)", () => { + const result = resolveConfigEnvVars( + { key: "${_UNDERSCORE_START}" }, + { _UNDERSCORE_START: "valid" }, + ); + expect(result).toEqual({ key: "valid" }); + }); + + it("substitutes ${VAR_WITH_NUMBERS_123} (valid)", () => { + const result = resolveConfigEnvVars( + { key: "${VAR_WITH_NUMBERS_123}" }, + { VAR_WITH_NUMBERS_123: "valid" }, + ); + expect(result).toEqual({ key: "valid" }); + }); + }); + + describe("passthrough behavior", () => { + it("passes through primitives unchanged", () => { + expect(resolveConfigEnvVars("hello", {})).toBe("hello"); + expect(resolveConfigEnvVars(42, {})).toBe(42); + expect(resolveConfigEnvVars(true, {})).toBe(true); + expect(resolveConfigEnvVars(null, {})).toBe(null); + }); + + it("passes through empty object", () => { + expect(resolveConfigEnvVars({}, {})).toEqual({}); + }); + + it("passes through empty array", () => { + expect(resolveConfigEnvVars([], {})).toEqual([]); + }); + + it("passes through non-string values in objects", () => { + const result = resolveConfigEnvVars({ num: 42, bool: true, nil: null, arr: [1, 2] }, {}); + expect(result).toEqual({ num: 42, bool: true, nil: null, arr: [1, 2] }); + }); + }); + + describe("real-world config patterns", () => { + it("substitutes API keys in provider config", () => { + const config = { + models: { + providers: { + "vercel-gateway": { + apiKey: "${VERCEL_GATEWAY_API_KEY}", + }, + openai: { + apiKey: "${OPENAI_API_KEY}", + }, + }, + }, + }; + const env = { + VERCEL_GATEWAY_API_KEY: "vg_key_123", + OPENAI_API_KEY: "sk-xxx", + }; + const result = resolveConfigEnvVars(config, env); + expect(result).toEqual({ + models: { + providers: { + "vercel-gateway": { + apiKey: "vg_key_123", + }, + openai: { + apiKey: "sk-xxx", + }, + }, + }, + }); + }); + + it("substitutes gateway auth token", () => { + const config = { + gateway: { + auth: { + token: "${CLAWDBOT_GATEWAY_TOKEN}", + }, + }, + }; + const result = resolveConfigEnvVars(config, { + CLAWDBOT_GATEWAY_TOKEN: "secret-token", + }); + expect(result).toEqual({ + gateway: { + auth: { + token: "secret-token", + }, + }, + }); + }); + + it("substitutes base URL with env var", () => { + const config = { + models: { + providers: { + custom: { + baseUrl: "${CUSTOM_API_BASE}/v1", + }, + }, + }, + }; + const result = resolveConfigEnvVars(config, { + CUSTOM_API_BASE: "https://api.example.com", + }); + expect(result).toEqual({ + models: { + providers: { + custom: { + baseUrl: "https://api.example.com/v1", + }, + }, + }, + }); + }); + }); +}); diff --git a/src/config/env-substitution.ts b/src/config/env-substitution.ts new file mode 100644 index 000000000..3332fa3b8 --- /dev/null +++ b/src/config/env-substitution.ts @@ -0,0 +1,103 @@ +/** + * Environment variable substitution for config values. + * + * Supports `${VAR_NAME}` syntax in string values, substituted at config load time. + * - Only uppercase env vars are matched: `[A-Z_][A-Z0-9_]*` + * - Escape with `$${}` to output literal `${}` + * - Missing env vars throw `MissingEnvVarError` with context + * + * @example + * ```json5 + * { + * models: { + * providers: { + * "vercel-gateway": { + * apiKey: "${VERCEL_GATEWAY_API_KEY}" + * } + * } + * } + * } + * ``` + */ + +// Pattern for valid uppercase env var names: starts with letter or underscore, +// followed by letters, numbers, or underscores (all uppercase) +const ENV_VAR_PATTERN = /\$\{([A-Z_][A-Z0-9_]*)\}/g; + +// Pattern for escaped env vars: $${...} -> ${...} +const ESCAPED_PATTERN = /\$\$\{([A-Z_][A-Z0-9_]*)\}/g; + +export class MissingEnvVarError extends Error { + constructor( + public readonly varName: string, + public readonly configPath: string, + ) { + super(`Missing env var "${varName}" referenced at config path: ${configPath}`); + this.name = "MissingEnvVarError"; + } +} + +function isPlainObject(value: unknown): value is Record { + return ( + typeof value === "object" && + value !== null && + !Array.isArray(value) && + Object.prototype.toString.call(value) === "[object Object]" + ); +} + +function substituteString(value: string, env: NodeJS.ProcessEnv, configPath: string): string { + // First pass: substitute real env vars + const substituted = value.replace(ENV_VAR_PATTERN, (match, varName: string) => { + // Check if this is actually an escaped var (preceded by $) + // We handle this in the second pass, so check the original string + const idx = value.indexOf(match); + if (idx > 0 && value[idx - 1] === "$") { + // This will be handled by ESCAPED_PATTERN, return as-is for now + return match; + } + + const envValue = env[varName]; + if (envValue === undefined || envValue === "") { + throw new MissingEnvVarError(varName, configPath); + } + return envValue; + }); + + // Second pass: convert escaped $${VAR} to literal ${VAR} + return substituted.replace(ESCAPED_PATTERN, (_, varName: string) => `\${${varName}}`); +} + +function substituteAny(value: unknown, env: NodeJS.ProcessEnv, path: string): unknown { + if (typeof value === "string") { + return substituteString(value, env, path); + } + + if (Array.isArray(value)) { + return value.map((item, index) => substituteAny(item, env, `${path}[${index}]`)); + } + + if (isPlainObject(value)) { + const result: Record = {}; + for (const [key, val] of Object.entries(value)) { + const childPath = path ? `${path}.${key}` : key; + result[key] = substituteAny(val, env, childPath); + } + return result; + } + + // Primitives (number, boolean, null) pass through unchanged + return value; +} + +/** + * Resolves `${VAR_NAME}` environment variable references in config values. + * + * @param obj - The parsed config object (after JSON5 parse and $include resolution) + * @param env - Environment variables to use for substitution (defaults to process.env) + * @returns The config object with env vars substituted + * @throws {MissingEnvVarError} If a referenced env var is not set or empty + */ +export function resolveConfigEnvVars(obj: unknown, env: NodeJS.ProcessEnv = process.env): unknown { + return substituteAny(obj, env, ""); +} diff --git a/src/config/io.ts b/src/config/io.ts index b7d1a6f92..20cd5c774 100644 --- a/src/config/io.ts +++ b/src/config/io.ts @@ -19,6 +19,7 @@ import { applySessionDefaults, applyTalkApiKey, } from "./defaults.js"; +import { MissingEnvVarError, resolveConfigEnvVars } from "./env-substitution.js"; import { ConfigIncludeError, resolveConfigIncludes } from "./includes.js"; import { applyLegacyMigrations, findLegacyConfigIssues } from "./legacy.js"; import { normalizeConfigPaths } from "./normalize-paths.js"; @@ -30,6 +31,7 @@ import { ClawdbotSchema } from "./zod-schema.js"; // Re-export for backwards compatibility export { CircularIncludeError, ConfigIncludeError } from "./includes.js"; +export { MissingEnvVarError } from "./env-substitution.js"; const SHELL_ENV_EXPECTED_KEYS = [ "OPENAI_API_KEY", @@ -219,8 +221,11 @@ export function createConfigIO(overrides: ConfigIoDeps = {}) { parseJson: (raw) => deps.json5.parse(raw), }); - const migrated = applyLegacyMigrations(resolved); - const resolvedConfig = migrated.next ?? resolved; + // Substitute ${VAR} env var references + const substituted = resolveConfigEnvVars(resolved, deps.env); + + const migrated = applyLegacyMigrations(substituted); + const resolvedConfig = migrated.next ?? substituted; warnOnConfigMiskeys(resolvedConfig, deps.logger); if (typeof resolvedConfig !== "object" || resolvedConfig === null) return {}; const validated = ClawdbotSchema.safeParse(resolvedConfig); @@ -346,8 +351,30 @@ export function createConfigIO(overrides: ConfigIoDeps = {}) { }; } - const migrated = applyLegacyMigrations(resolved); - const resolvedConfigRaw = migrated.next ?? resolved; + // Substitute ${VAR} env var references + let substituted: unknown; + try { + substituted = resolveConfigEnvVars(resolved, deps.env); + } catch (err) { + const message = + err instanceof MissingEnvVarError + ? err.message + : `Env var substitution failed: ${String(err)}`; + return { + path: configPath, + exists: true, + raw, + parsed: parsedRes.parsed, + valid: false, + config: {}, + hash, + issues: [{ path: "", message }], + legacyIssues: [], + }; + } + + const migrated = applyLegacyMigrations(substituted); + const resolvedConfigRaw = migrated.next ?? substituted; const legacyIssues = findLegacyConfigIssues(resolvedConfigRaw); const validated = validateConfigObject(resolvedConfigRaw);