fix: validate web_search freshness (#1688) (thanks @JonUleis)
This commit is contained in:
@@ -10,6 +10,7 @@ Docs: https://docs.clawd.bot
|
||||
|
||||
### Changes
|
||||
- TTS: add Edge TTS provider fallback, defaulting to keyless Edge with MP3 retry on format failures. (#1668) Thanks @steipete. https://docs.clawd.bot/tts
|
||||
- Web search: add Brave freshness filter parameter for time-scoped results. (#1688) Thanks @JonUleis. https://docs.clawd.bot/tools/web
|
||||
- Docs: expand FAQ (migration, scheduling, concurrency, model recommendations, OpenAI subscription auth, Pi sizing, hackable install, docs SSL workaround).
|
||||
- Docs: add verbose installer troubleshooting guidance.
|
||||
- Docs: update Fly.io guide notes.
|
||||
|
||||
@@ -174,6 +174,7 @@ Search the web using your configured provider.
|
||||
- `country` (optional): 2-letter country code for region-specific results (e.g., "DE", "US", "ALL"). If omitted, Brave chooses its default region.
|
||||
- `search_lang` (optional): ISO language code for search results (e.g., "de", "en", "fr")
|
||||
- `ui_lang` (optional): ISO language code for UI elements
|
||||
- `freshness` (optional, Brave only): filter by discovery time (`pd`, `pw`, `pm`, `py`, or `YYYY-MM-DDtoYYYY-MM-DD`)
|
||||
|
||||
**Examples:**
|
||||
|
||||
@@ -193,6 +194,12 @@ await web_search({
|
||||
search_lang: "fr",
|
||||
ui_lang: "fr"
|
||||
});
|
||||
|
||||
// Recent results (past week)
|
||||
await web_search({
|
||||
query: "TMBG interview",
|
||||
freshness: "pw"
|
||||
});
|
||||
```
|
||||
|
||||
## web_fetch
|
||||
|
||||
@@ -2,7 +2,8 @@ import { describe, expect, it } from "vitest";
|
||||
|
||||
import { __testing } from "./web-search.js";
|
||||
|
||||
const { inferPerplexityBaseUrlFromApiKey, resolvePerplexityBaseUrl } = __testing;
|
||||
const { inferPerplexityBaseUrlFromApiKey, resolvePerplexityBaseUrl, normalizeFreshness } =
|
||||
__testing;
|
||||
|
||||
describe("web_search perplexity baseUrl defaults", () => {
|
||||
it("detects a Perplexity key prefix", () => {
|
||||
@@ -51,3 +52,20 @@ describe("web_search perplexity baseUrl defaults", () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("web_search freshness normalization", () => {
|
||||
it("accepts Brave shortcut values", () => {
|
||||
expect(normalizeFreshness("pd")).toBe("pd");
|
||||
expect(normalizeFreshness("PW")).toBe("pw");
|
||||
});
|
||||
|
||||
it("accepts valid date ranges", () => {
|
||||
expect(normalizeFreshness("2024-01-01to2024-01-31")).toBe("2024-01-01to2024-01-31");
|
||||
});
|
||||
|
||||
it("rejects invalid date ranges", () => {
|
||||
expect(normalizeFreshness("2024-13-01to2024-01-31")).toBeUndefined();
|
||||
expect(normalizeFreshness("2024-02-30to2024-03-01")).toBeUndefined();
|
||||
expect(normalizeFreshness("2024-03-10to2024-03-01")).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -29,6 +29,8 @@ const PERPLEXITY_KEY_PREFIXES = ["pplx-"];
|
||||
const OPENROUTER_KEY_PREFIXES = ["sk-or-"];
|
||||
|
||||
const SEARCH_CACHE = new Map<string, CacheEntry<Record<string, unknown>>>();
|
||||
const BRAVE_FRESHNESS_SHORTCUTS = new Set(["pd", "pw", "pm", "py"]);
|
||||
const BRAVE_FRESHNESS_RANGE = /^(\d{4}-\d{2}-\d{2})to(\d{4}-\d{2}-\d{2})$/;
|
||||
|
||||
const WebSearchSchema = Type.Object({
|
||||
query: Type.String({ description: "Search query string." }),
|
||||
@@ -58,7 +60,7 @@ const WebSearchSchema = Type.Object({
|
||||
freshness: Type.Optional(
|
||||
Type.String({
|
||||
description:
|
||||
"Filter results by discovery time. Values: 'pd' (past 24h), 'pw' (past week), 'pm' (past month), 'py' (past year), or date range 'YYYY-MM-DDtoYYYY-MM-DD'. Brave provider only.",
|
||||
"Filter results by discovery time (Brave only). Values: 'pd' (past 24h), 'pw' (past week), 'pm' (past month), 'py' (past year), or date range 'YYYY-MM-DDtoYYYY-MM-DD'.",
|
||||
}),
|
||||
),
|
||||
});
|
||||
@@ -225,6 +227,35 @@ function resolveSearchCount(value: unknown, fallback: number): number {
|
||||
return clamped;
|
||||
}
|
||||
|
||||
function normalizeFreshness(value: string | undefined): string | undefined {
|
||||
if (!value) return undefined;
|
||||
const trimmed = value.trim();
|
||||
if (!trimmed) return undefined;
|
||||
|
||||
const lower = trimmed.toLowerCase();
|
||||
if (BRAVE_FRESHNESS_SHORTCUTS.has(lower)) return lower;
|
||||
|
||||
const match = trimmed.match(BRAVE_FRESHNESS_RANGE);
|
||||
if (!match) return undefined;
|
||||
|
||||
const [, start, end] = match;
|
||||
if (!isValidIsoDate(start) || !isValidIsoDate(end)) return undefined;
|
||||
if (start > end) return undefined;
|
||||
|
||||
return `${start}to${end}`;
|
||||
}
|
||||
|
||||
function isValidIsoDate(value: string): boolean {
|
||||
if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) return false;
|
||||
const [year, month, day] = value.split("-").map((part) => Number.parseInt(part, 10));
|
||||
if (!Number.isFinite(year) || !Number.isFinite(month) || !Number.isFinite(day)) return false;
|
||||
|
||||
const date = new Date(Date.UTC(year, month - 1, day));
|
||||
return (
|
||||
date.getUTCFullYear() === year && date.getUTCMonth() === month - 1 && date.getUTCDate() === day
|
||||
);
|
||||
}
|
||||
|
||||
function resolveSiteName(url: string | undefined): string | undefined {
|
||||
if (!url) return undefined;
|
||||
try {
|
||||
@@ -290,7 +321,9 @@ async function runWebSearch(params: {
|
||||
perplexityModel?: string;
|
||||
}): Promise<Record<string, unknown>> {
|
||||
const cacheKey = normalizeCacheKey(
|
||||
`${params.provider}:${params.query}:${params.count}:${params.country || "default"}:${params.search_lang || "default"}:${params.ui_lang || "default"}:${params.freshness || "default"}`,
|
||||
params.provider === "brave"
|
||||
? `${params.provider}:${params.query}:${params.count}:${params.country || "default"}:${params.search_lang || "default"}:${params.ui_lang || "default"}:${params.freshness || "default"}`
|
||||
: `${params.provider}:${params.query}:${params.count}:${params.country || "default"}:${params.search_lang || "default"}:${params.ui_lang || "default"}`,
|
||||
);
|
||||
const cached = readCache(SEARCH_CACHE, cacheKey);
|
||||
if (cached) return { ...cached.value, cached: true };
|
||||
@@ -409,7 +442,23 @@ export function createWebSearchTool(options?: {
|
||||
const country = readStringParam(params, "country");
|
||||
const search_lang = readStringParam(params, "search_lang");
|
||||
const ui_lang = readStringParam(params, "ui_lang");
|
||||
const freshness = readStringParam(params, "freshness");
|
||||
const rawFreshness = readStringParam(params, "freshness");
|
||||
if (rawFreshness && provider !== "brave") {
|
||||
return jsonResult({
|
||||
error: "unsupported_freshness",
|
||||
message: "freshness is only supported by the Brave web_search provider.",
|
||||
docs: "https://docs.clawd.bot/tools/web",
|
||||
});
|
||||
}
|
||||
const freshness = rawFreshness ? normalizeFreshness(rawFreshness) : undefined;
|
||||
if (rawFreshness && !freshness) {
|
||||
return jsonResult({
|
||||
error: "invalid_freshness",
|
||||
message:
|
||||
"freshness must be one of pd, pw, pm, py, or a range like YYYY-MM-DDtoYYYY-MM-DD.",
|
||||
docs: "https://docs.clawd.bot/tools/web",
|
||||
});
|
||||
}
|
||||
const result = await runWebSearch({
|
||||
query,
|
||||
count: resolveSearchCount(count, DEFAULT_SEARCH_COUNT),
|
||||
@@ -436,4 +485,5 @@ export function createWebSearchTool(options?: {
|
||||
export const __testing = {
|
||||
inferPerplexityBaseUrlFromApiKey,
|
||||
resolvePerplexityBaseUrl,
|
||||
normalizeFreshness,
|
||||
} as const;
|
||||
|
||||
@@ -88,6 +88,40 @@ describe("web_search country and language parameters", () => {
|
||||
const url = new URL(mockFetch.mock.calls[0][0] as string);
|
||||
expect(url.searchParams.get("ui_lang")).toBe("de");
|
||||
});
|
||||
|
||||
it("should pass freshness parameter to Brave API", async () => {
|
||||
const mockFetch = vi.fn(() =>
|
||||
Promise.resolve({
|
||||
ok: true,
|
||||
json: () => Promise.resolve({ web: { results: [] } }),
|
||||
} as Response),
|
||||
);
|
||||
// @ts-expect-error mock fetch
|
||||
global.fetch = mockFetch;
|
||||
|
||||
const tool = createWebSearchTool({ config: undefined, sandboxed: true });
|
||||
await tool?.execute?.(1, { query: "test", freshness: "pw" });
|
||||
|
||||
const url = new URL(mockFetch.mock.calls[0][0] as string);
|
||||
expect(url.searchParams.get("freshness")).toBe("pw");
|
||||
});
|
||||
|
||||
it("rejects invalid freshness values", async () => {
|
||||
const mockFetch = vi.fn(() =>
|
||||
Promise.resolve({
|
||||
ok: true,
|
||||
json: () => Promise.resolve({ web: { results: [] } }),
|
||||
} as Response),
|
||||
);
|
||||
// @ts-expect-error mock fetch
|
||||
global.fetch = mockFetch;
|
||||
|
||||
const tool = createWebSearchTool({ config: undefined, sandboxed: true });
|
||||
const result = await tool?.execute?.(1, { query: "test", freshness: "yesterday" });
|
||||
|
||||
expect(mockFetch).not.toHaveBeenCalled();
|
||||
expect(result?.details).toMatchObject({ error: "invalid_freshness" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("web_search perplexity baseUrl defaults", () => {
|
||||
@@ -120,6 +154,27 @@ describe("web_search perplexity baseUrl defaults", () => {
|
||||
expect(mockFetch.mock.calls[0]?.[0]).toBe("https://api.perplexity.ai/chat/completions");
|
||||
});
|
||||
|
||||
it("rejects freshness for Perplexity provider", async () => {
|
||||
vi.stubEnv("PERPLEXITY_API_KEY", "pplx-test");
|
||||
const mockFetch = vi.fn(() =>
|
||||
Promise.resolve({
|
||||
ok: true,
|
||||
json: () => Promise.resolve({ choices: [{ message: { content: "ok" } }], citations: [] }),
|
||||
} as Response),
|
||||
);
|
||||
// @ts-expect-error mock fetch
|
||||
global.fetch = mockFetch;
|
||||
|
||||
const tool = createWebSearchTool({
|
||||
config: { tools: { web: { search: { provider: "perplexity" } } } },
|
||||
sandboxed: true,
|
||||
});
|
||||
const result = await tool?.execute?.(1, { query: "test", freshness: "pw" });
|
||||
|
||||
expect(mockFetch).not.toHaveBeenCalled();
|
||||
expect(result?.details).toMatchObject({ error: "unsupported_freshness" });
|
||||
});
|
||||
|
||||
it("defaults to OpenRouter when OPENROUTER_API_KEY is set", async () => {
|
||||
vi.stubEnv("PERPLEXITY_API_KEY", "");
|
||||
vi.stubEnv("OPENROUTER_API_KEY", "sk-or-test");
|
||||
|
||||
Reference in New Issue
Block a user