Merge pull request #819 from mukhtharcm/feature/memory-search-custom-endpoint

feat(memory): support custom OpenAI-compatible embedding endpoints

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Changes
 - Models/Moonshot: add Kimi K2 turbo + thinking variants to the preset + docs. (#818 — thanks @mickahouan)
+- Memory: allow custom OpenAI-compatible embedding endpoints for memory search (remote baseUrl/apiKey/headers). (#819 — thanks @mukhtharcm)
 
 ### Fixes
 - Onboarding/Configure: refuse to proceed with invalid configs; run `clawdbot doctor` first to avoid wiping custom fields. (#764 — thanks @mukhtharcm)

docs/concepts/memory.md

@@ -79,10 +79,32 @@ Defaults:
 - Uses remote embeddings (OpenAI) unless configured for local.
 - Local mode uses node-llama-cpp and may require `pnpm approve-builds`.
 
-Remote embeddings **require** an OpenAI API key (`OPENAI_API_KEY` or
-`models.providers.openai.apiKey`). Codex OAuth only covers chat/completions and
-does **not** satisfy embeddings for memory search. If you don't want to set an
-API key, use `memorySearch.provider = "local"` or set
+Remote embeddings **require** an API key for the embedding provider. By default
+this is OpenAI (`OPENAI_API_KEY` or `models.providers.openai.apiKey`). Codex
+OAuth only covers chat/completions and does **not** satisfy embeddings for
+memory search. When using a custom OpenAI-compatible endpoint, set
+`memorySearch.remote.apiKey` (and optional `memorySearch.remote.headers`).
+
+If you want to use a **custom OpenAI-compatible endpoint** (like Gemini, OpenRouter, or a proxy),
+you can use the `remote` configuration:
+
+```json5
+agents: {
+  defaults: {
+    memorySearch: {
+      provider: "openai",
+      model: "text-embedding-3-small",
+      remote: {
+        baseUrl: "https://generativelanguage.googleapis.com/v1beta/openai/",
+        apiKey: "YOUR_GEMINI_API_KEY",
+        headers: { "X-Custom-Header": "value" }
+      }
+    }
+  }
+}
+```
+
+If you don't want to set an API key, use `memorySearch.provider = "local"` or set
 `memorySearch.fallback = "none"`.
 
 Config example:

docs/gateway/configuration-examples.md

@@ -241,6 +241,14 @@ Save to `~/.clawdbot/clawdbot.json` and you can DM the bot from that number.
         prompt: "HEARTBEAT",
         ackMaxChars: 300
       },
+      memorySearch: {
+        provider: "openai",
+        model: "text-embedding-004",
+        remote: {
+          baseUrl: "https://generativelanguage.googleapis.com/v1beta/openai/",
+          apiKey: "${GEMINI_API_KEY}"
+        }
+      },
       sandbox: {
         mode: "non-main",
         perSession: true,

src/agents/memory-search.test.ts

@@ -53,4 +53,37 @@ describe("memory search config", () => {
     expect(resolved?.query.maxResults).toBe(8);
     expect(resolved?.query.minScore).toBe(0.2);
   });
+
+  it("merges remote defaults with agent overrides", () => {
+    const cfg = {
+      agents: {
+        defaults: {
+          memorySearch: {
+            remote: {
+              baseUrl: "https://default.example/v1",
+              apiKey: "default-key",
+              headers: { "X-Default": "on" },
+            },
+          },
+        },
+        list: [
+          {
+            id: "main",
+            default: true,
+            memorySearch: {
+              remote: {
+                baseUrl: "https://agent.example/v1",
+              },
+            },
+          },
+        ],
+      },
+    };
+    const resolved = resolveMemorySearchConfig(cfg, "main");
+    expect(resolved?.remote).toEqual({
+      baseUrl: "https://agent.example/v1",
+      apiKey: "default-key",
+      headers: { "X-Default": "on" },
+    });
+  });
 });

src/agents/memory-search.ts

@@ -9,6 +9,11 @@ import { resolveAgentConfig } from "./agent-scope.js";
 export type ResolvedMemorySearchConfig = {
   enabled: boolean;
   provider: "openai" | "local";
+  remote?: {
+    baseUrl?: string;
+    apiKey?: string;
+    headers?: Record<string, string>;
+  };
   fallback: "openai" | "none";
   model: string;
   local: {
@@ -60,6 +65,14 @@ function mergeConfig(
 ): ResolvedMemorySearchConfig {
   const enabled = overrides?.enabled ?? defaults?.enabled ?? true;
   const provider = overrides?.provider ?? defaults?.provider ?? "openai";
+  const hasRemote = Boolean(defaults?.remote || overrides?.remote);
+  const remote = hasRemote
+    ? {
+        baseUrl: overrides?.remote?.baseUrl ?? defaults?.remote?.baseUrl,
+        apiKey: overrides?.remote?.apiKey ?? defaults?.remote?.apiKey,
+        headers: overrides?.remote?.headers ?? defaults?.remote?.headers,
+      }
+    : undefined;
   const fallback = overrides?.fallback ?? defaults?.fallback ?? "openai";
   const model = overrides?.model ?? defaults?.model ?? DEFAULT_MODEL;
   const local = {
@@ -112,6 +125,7 @@ function mergeConfig(
   return {
     enabled,
     provider,
+    remote,
     fallback,
     model,
     local,

src/config/schema.ts

@@ -118,6 +118,9 @@ const FIELD_LABELS: Record<string, string> = {
   "agents.defaults.memorySearch": "Memory Search",
   "agents.defaults.memorySearch.enabled": "Enable Memory Search",
   "agents.defaults.memorySearch.provider": "Memory Search Provider",
+  "agents.defaults.memorySearch.remote.baseUrl": "Remote Embedding Base URL",
+  "agents.defaults.memorySearch.remote.apiKey": "Remote Embedding API Key",
+  "agents.defaults.memorySearch.remote.headers": "Remote Embedding Headers",
   "agents.defaults.memorySearch.model": "Memory Search Model",
   "agents.defaults.memorySearch.fallback": "Memory Search Fallback",
   "agents.defaults.memorySearch.local.modelPath": "Local Embedding Model Path",
@@ -236,6 +239,12 @@ const FIELD_HELP: Record<string, string> = {
     "Vector search over MEMORY.md and memory/*.md (per-agent overrides supported).",
   "agents.defaults.memorySearch.provider":
     'Embedding provider ("openai" or "local").',
+  "agents.defaults.memorySearch.remote.baseUrl":
+    "Custom OpenAI-compatible base URL (e.g. for Gemini/OpenRouter proxies).",
+  "agents.defaults.memorySearch.remote.apiKey":
+    "Custom API key for the remote embedding provider.",
+  "agents.defaults.memorySearch.remote.headers":
+    "Extra headers for remote embeddings (merged; remote overrides OpenAI headers).",
   "agents.defaults.memorySearch.local.modelPath":
     "Local GGUF model path or hf: URI (node-llama-cpp).",
   "agents.defaults.memorySearch.fallback":

src/config/types.ts

@@ -1011,6 +1011,11 @@ export type MemorySearchConfig = {
   enabled?: boolean;
   /** Embedding provider mode. */
   provider?: "openai" | "local";
+  remote?: {
+    baseUrl?: string;
+    apiKey?: string;
+    headers?: Record<string, string>;
+  };
   /** Fallback behavior when local embeddings fail. */
   fallback?: "openai" | "none";
   /** Embedding model id (remote) or alias (local). */

src/config/zod-schema.ts

@@ -886,6 +886,13 @@ const MemorySearchSchema = z
   .object({
     enabled: z.boolean().optional(),
     provider: z.union([z.literal("openai"), z.literal("local")]).optional(),
+    remote: z
+      .object({
+        baseUrl: z.string().optional(),
+        apiKey: z.string().optional(),
+        headers: z.record(z.string(), z.string()).optional(),
+      })
+      .optional(),
     fallback: z.union([z.literal("openai"), z.literal("none")]).optional(),
     model: z.string().optional(),
     local: z

src/memory/embeddings.test.ts

@@ -0,0 +1,110 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+vi.mock("../agents/model-auth.js", () => ({
+  resolveApiKeyForProvider: vi.fn(),
+}));
+
+const createFetchMock = () =>
+  vi.fn(async () => ({
+    ok: true,
+    status: 200,
+    json: async () => ({ data: [{ embedding: [1, 2, 3] }] }),
+  })) as unknown as typeof fetch;
+
+describe("embedding provider remote overrides", () => {
+  afterEach(() => {
+    vi.resetAllMocks();
+    vi.unstubAllGlobals();
+  });
+
+  it("uses remote baseUrl/apiKey and merges headers", async () => {
+    const fetchMock = createFetchMock();
+    vi.stubGlobal("fetch", fetchMock);
+
+    const { createEmbeddingProvider } = await import("./embeddings.js");
+    const authModule = await import("../agents/model-auth.js");
+    vi.mocked(authModule.resolveApiKeyForProvider).mockResolvedValue({
+      apiKey: "provider-key",
+    });
+
+    const cfg = {
+      models: {
+        providers: {
+          openai: {
+            baseUrl: "https://provider.example/v1",
+            headers: {
+              "X-Provider": "p",
+              "X-Shared": "provider",
+            },
+          },
+        },
+      },
+    };
+
+    const result = await createEmbeddingProvider({
+      config: cfg as never,
+      provider: "openai",
+      remote: {
+        baseUrl: "https://remote.example/v1",
+        apiKey: "  remote-key  ",
+        headers: {
+          "X-Shared": "remote",
+          "X-Remote": "r",
+        },
+      },
+      model: "text-embedding-3-small",
+      fallback: "openai",
+    });
+
+    await result.provider.embedQuery("hello");
+
+    expect(authModule.resolveApiKeyForProvider).not.toHaveBeenCalled();
+    const [url, init] = fetchMock.mock.calls[0] ?? [];
+    expect(url).toBe("https://remote.example/v1/embeddings");
+    const headers = (init?.headers ?? {}) as Record<string, string>;
+    expect(headers.Authorization).toBe("Bearer remote-key");
+    expect(headers["Content-Type"]).toBe("application/json");
+    expect(headers["X-Provider"]).toBe("p");
+    expect(headers["X-Shared"]).toBe("remote");
+    expect(headers["X-Remote"]).toBe("r");
+  });
+
+  it("falls back to resolved api key when remote apiKey is blank", async () => {
+    const fetchMock = createFetchMock();
+    vi.stubGlobal("fetch", fetchMock);
+
+    const { createEmbeddingProvider } = await import("./embeddings.js");
+    const authModule = await import("../agents/model-auth.js");
+    vi.mocked(authModule.resolveApiKeyForProvider).mockResolvedValue({
+      apiKey: "provider-key",
+    });
+
+    const cfg = {
+      models: {
+        providers: {
+          openai: {
+            baseUrl: "https://provider.example/v1",
+          },
+        },
+      },
+    };
+
+    const result = await createEmbeddingProvider({
+      config: cfg as never,
+      provider: "openai",
+      remote: {
+        baseUrl: "https://remote.example/v1",
+        apiKey: "   ",
+      },
+      model: "text-embedding-3-small",
+      fallback: "openai",
+    });
+
+    await result.provider.embedQuery("hello");
+
+    expect(authModule.resolveApiKeyForProvider).toHaveBeenCalledTimes(1);
+    const headers =
+      (fetchMock.mock.calls[0]?.[1]?.headers as Record<string, string>) ?? {};
+    expect(headers.Authorization).toBe("Bearer provider-key");
+  });
+});

src/memory/embeddings.ts

@@ -20,6 +20,11 @@ export type EmbeddingProviderOptions = {
   config: ClawdbotConfig;
   agentDir?: string;
   provider: "openai" | "local";
+  remote?: {
+    baseUrl?: string;
+    apiKey?: string;
+    headers?: Record<string, string>;
+  };
   model: string;
   fallback: "openai" | "none";
   local?: {
@@ -42,16 +47,27 @@ function normalizeOpenAiModel(model: string): string {
 async function createOpenAiEmbeddingProvider(
   options: EmbeddingProviderOptions,
 ): Promise<EmbeddingProvider> {
-  const { apiKey } = await resolveApiKeyForProvider({
-    provider: "openai",
-    cfg: options.config,
-    agentDir: options.agentDir,
-  });
+  const remote = options.remote;
+  const remoteApiKey = remote?.apiKey?.trim();
+  const remoteBaseUrl = remote?.baseUrl?.trim();
+
+  const { apiKey } = remoteApiKey
+    ? { apiKey: remoteApiKey }
+    : await resolveApiKeyForProvider({
+        provider: "openai",
+        cfg: options.config,
+        agentDir: options.agentDir,
+      });
 
   const providerConfig = options.config.models?.providers?.openai;
-  const baseUrl = providerConfig?.baseUrl?.trim() || DEFAULT_OPENAI_BASE_URL;
+  const baseUrl =
+    remoteBaseUrl || providerConfig?.baseUrl?.trim() || DEFAULT_OPENAI_BASE_URL;
   const url = `${baseUrl.replace(/\/$/, "")}/embeddings`;
-  const headerOverrides = providerConfig?.headers ?? {};
+  const headerOverrides = Object.assign(
+    {},
+    providerConfig?.headers,
+    remote?.headers,
+  );
   const headers: Record<string, string> = {
     "Content-Type": "application/json",
     Authorization: `Bearer ${apiKey}`,

src/memory/index.ts

@@ -88,6 +88,7 @@ export class MemoryIndexManager {
       config: cfg,
       agentDir: resolveAgentDir(cfg, agentId),
       provider: settings.provider,
+      remote: settings.remote,
       model: settings.model,
       fallback: settings.fallback,
       local: settings.local,