fix: add provider retry policy
This commit is contained in:
Peter Steinberger
106
src/infra/retry-policy.ts
Normal file
106
src/infra/retry-policy.ts
Normal file
@@ -0,0 +1,106 @@
|
||||
import { RateLimitError } from "@buape/carbon";
|
||||
|
||||
import { formatErrorMessage } from "./errors.js";
|
||||
import { type RetryConfig, resolveRetryConfig, retryAsync } from "./retry.js";
|
||||
|
||||
export type RetryRunner = <T>(
|
||||
fn: () => Promise<T>,
|
||||
label?: string,
|
||||
) => Promise<T>;
|
||||
|
||||
export const DISCORD_RETRY_DEFAULTS = {
|
||||
attempts: 3,
|
||||
minDelayMs: 500,
|
||||
maxDelayMs: 30_000,
|
||||
jitter: 0.1,
|
||||
};
|
||||
|
||||
export const TELEGRAM_RETRY_DEFAULTS = {
|
||||
attempts: 3,
|
||||
minDelayMs: 400,
|
||||
maxDelayMs: 30_000,
|
||||
jitter: 0.1,
|
||||
};
|
||||
|
||||
const TELEGRAM_RETRY_RE =
|
||||
/429|timeout|connect|reset|closed|unavailable|temporarily/i;
|
||||
|
||||
function getTelegramRetryAfterMs(err: unknown): number | undefined {
|
||||
if (!err || typeof err !== "object") return undefined;
|
||||
const candidate =
|
||||
"parameters" in err && err.parameters && typeof err.parameters === "object"
|
||||
? (err.parameters as { retry_after?: unknown }).retry_after
|
||||
: "response" in err &&
|
||||
err.response &&
|
||||
typeof err.response === "object" &&
|
||||
"parameters" in err.response
|
||||
? (
|
||||
err.response as {
|
||||
parameters?: { retry_after?: unknown };
|
||||
}
|
||||
).parameters?.retry_after
|
||||
: "error" in err &&
|
||||
err.error &&
|
||||
typeof err.error === "object" &&
|
||||
"parameters" in err.error
|
||||
? (err.error as { parameters?: { retry_after?: unknown } }).parameters
|
||||
?.retry_after
|
||||
: undefined;
|
||||
return typeof candidate === "number" && Number.isFinite(candidate)
|
||||
? candidate * 1000
|
||||
: undefined;
|
||||
}
|
||||
|
||||
export function createDiscordRetryRunner(params: {
|
||||
retry?: RetryConfig;
|
||||
configRetry?: RetryConfig;
|
||||
verbose?: boolean;
|
||||
}): RetryRunner {
|
||||
const retryConfig = resolveRetryConfig(DISCORD_RETRY_DEFAULTS, {
|
||||
...params.configRetry,
|
||||
...params.retry,
|
||||
});
|
||||
return <T>(fn: () => Promise<T>, label?: string) =>
|
||||
retryAsync(fn, {
|
||||
...retryConfig,
|
||||
label,
|
||||
shouldRetry: (err) => err instanceof RateLimitError,
|
||||
retryAfterMs: (err) =>
|
||||
err instanceof RateLimitError ? err.retryAfter * 1000 : undefined,
|
||||
onRetry: params.verbose
|
||||
? (info) => {
|
||||
const labelText = info.label ?? "request";
|
||||
const maxRetries = Math.max(1, info.maxAttempts - 1);
|
||||
console.warn(
|
||||
`discord ${labelText} rate limited, retry ${info.attempt}/${maxRetries} in ${info.delayMs}ms`,
|
||||
);
|
||||
}
|
||||
: undefined,
|
||||
});
|
||||
}
|
||||
|
||||
export function createTelegramRetryRunner(params: {
|
||||
retry?: RetryConfig;
|
||||
configRetry?: RetryConfig;
|
||||
verbose?: boolean;
|
||||
}): RetryRunner {
|
||||
const retryConfig = resolveRetryConfig(TELEGRAM_RETRY_DEFAULTS, {
|
||||
...params.configRetry,
|
||||
...params.retry,
|
||||
});
|
||||
return <T>(fn: () => Promise<T>, label?: string) =>
|
||||
retryAsync(fn, {
|
||||
...retryConfig,
|
||||
label,
|
||||
shouldRetry: (err) => TELEGRAM_RETRY_RE.test(formatErrorMessage(err)),
|
||||
retryAfterMs: getTelegramRetryAfterMs,
|
||||
onRetry: params.verbose
|
||||
? (info) => {
|
||||
const maxRetries = Math.max(1, info.maxAttempts - 1);
|
||||
console.warn(
|
||||
`telegram send retry ${info.attempt}/${maxRetries} for ${info.label ?? label ?? "request"} in ${info.delayMs}ms: ${formatErrorMessage(info.err)}`,
|
||||
);
|
||||
}
|
||||
: undefined,
|
||||
});
|
||||
}
|
||||
@@ -25,4 +25,80 @@ describe("retryAsync", () => {
|
||||
await expect(retryAsync(fn, 2, 1)).rejects.toThrow("boom");
|
||||
expect(fn).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("stops when shouldRetry returns false", async () => {
|
||||
const fn = vi.fn().mockRejectedValue(new Error("boom"));
|
||||
await expect(
|
||||
retryAsync(fn, { attempts: 3, shouldRetry: () => false }),
|
||||
).rejects.toThrow("boom");
|
||||
expect(fn).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("calls onRetry before retrying", async () => {
|
||||
const fn = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error("boom"))
|
||||
.mockResolvedValueOnce("ok");
|
||||
const onRetry = vi.fn();
|
||||
const res = await retryAsync(fn, {
|
||||
attempts: 2,
|
||||
minDelayMs: 0,
|
||||
maxDelayMs: 0,
|
||||
onRetry,
|
||||
});
|
||||
expect(res).toBe("ok");
|
||||
expect(onRetry).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ attempt: 1, maxAttempts: 2 }),
|
||||
);
|
||||
});
|
||||
|
||||
it("clamps attempts to at least 1", async () => {
|
||||
const fn = vi.fn().mockRejectedValue(new Error("boom"));
|
||||
await expect(
|
||||
retryAsync(fn, { attempts: 0, minDelayMs: 0, maxDelayMs: 0 }),
|
||||
).rejects.toThrow("boom");
|
||||
expect(fn).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("uses retryAfterMs when provided", async () => {
|
||||
vi.useFakeTimers();
|
||||
const fn = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error("boom"))
|
||||
.mockResolvedValueOnce("ok");
|
||||
const delays: number[] = [];
|
||||
const promise = retryAsync(fn, {
|
||||
attempts: 2,
|
||||
minDelayMs: 0,
|
||||
maxDelayMs: 1000,
|
||||
jitter: 0,
|
||||
retryAfterMs: () => 500,
|
||||
onRetry: (info) => delays.push(info.delayMs),
|
||||
});
|
||||
await vi.runAllTimersAsync();
|
||||
await expect(promise).resolves.toBe("ok");
|
||||
expect(delays[0]).toBe(500);
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("clamps retryAfterMs to maxDelayMs", async () => {
|
||||
vi.useFakeTimers();
|
||||
const fn = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error("boom"))
|
||||
.mockResolvedValueOnce("ok");
|
||||
const delays: number[] = [];
|
||||
const promise = retryAsync(fn, {
|
||||
attempts: 2,
|
||||
minDelayMs: 0,
|
||||
maxDelayMs: 100,
|
||||
jitter: 0,
|
||||
retryAfterMs: () => 500,
|
||||
onRetry: (info) => delays.push(info.delayMs),
|
||||
});
|
||||
await vi.runAllTimersAsync();
|
||||
await expect(promise).resolves.toBe("ok");
|
||||
expect(delays[0]).toBe(100);
|
||||
vi.useRealTimers();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,18 +1,137 @@
|
||||
export type RetryConfig = {
|
||||
attempts?: number;
|
||||
minDelayMs?: number;
|
||||
maxDelayMs?: number;
|
||||
jitter?: number;
|
||||
};
|
||||
|
||||
export type RetryInfo = {
|
||||
attempt: number;
|
||||
maxAttempts: number;
|
||||
delayMs: number;
|
||||
err: unknown;
|
||||
label?: string;
|
||||
};
|
||||
|
||||
export type RetryOptions = RetryConfig & {
|
||||
label?: string;
|
||||
shouldRetry?: (err: unknown, attempt: number) => boolean;
|
||||
retryAfterMs?: (err: unknown) => number | undefined;
|
||||
onRetry?: (info: RetryInfo) => void;
|
||||
};
|
||||
|
||||
const DEFAULT_RETRY_CONFIG = {
|
||||
attempts: 3,
|
||||
minDelayMs: 300,
|
||||
maxDelayMs: 30_000,
|
||||
jitter: 0,
|
||||
};
|
||||
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
const asFiniteNumber = (value: unknown): number | undefined =>
|
||||
typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
||||
|
||||
const clampNumber = (
|
||||
value: unknown,
|
||||
fallback: number,
|
||||
min?: number,
|
||||
max?: number,
|
||||
) => {
|
||||
const next = asFiniteNumber(value);
|
||||
if (next === undefined) return fallback;
|
||||
const floor = typeof min === "number" ? min : Number.NEGATIVE_INFINITY;
|
||||
const ceiling = typeof max === "number" ? max : Number.POSITIVE_INFINITY;
|
||||
return Math.min(Math.max(next, floor), ceiling);
|
||||
};
|
||||
|
||||
export function resolveRetryConfig(
|
||||
defaults: Required<RetryConfig> = DEFAULT_RETRY_CONFIG,
|
||||
overrides?: RetryConfig,
|
||||
): Required<RetryConfig> {
|
||||
const attempts = Math.max(
|
||||
1,
|
||||
Math.round(clampNumber(overrides?.attempts, defaults.attempts, 1)),
|
||||
);
|
||||
const minDelayMs = Math.max(
|
||||
0,
|
||||
Math.round(clampNumber(overrides?.minDelayMs, defaults.minDelayMs, 0)),
|
||||
);
|
||||
const maxDelayMs = Math.max(
|
||||
minDelayMs,
|
||||
Math.round(clampNumber(overrides?.maxDelayMs, defaults.maxDelayMs, 0)),
|
||||
);
|
||||
const jitter = clampNumber(overrides?.jitter, defaults.jitter, 0, 1);
|
||||
return { attempts, minDelayMs, maxDelayMs, jitter };
|
||||
}
|
||||
|
||||
function applyJitter(delayMs: number, jitter: number): number {
|
||||
if (jitter <= 0) return delayMs;
|
||||
const offset = (Math.random() * 2 - 1) * jitter;
|
||||
return Math.max(0, Math.round(delayMs * (1 + offset)));
|
||||
}
|
||||
|
||||
export async function retryAsync<T>(
|
||||
fn: () => Promise<T>,
|
||||
attempts = 3,
|
||||
attemptsOrOptions: number | RetryOptions = 3,
|
||||
initialDelayMs = 300,
|
||||
): Promise<T> {
|
||||
if (typeof attemptsOrOptions === "number") {
|
||||
const attempts = Math.max(1, Math.round(attemptsOrOptions));
|
||||
let lastErr: unknown;
|
||||
for (let i = 0; i < attempts; i += 1) {
|
||||
try {
|
||||
return await fn();
|
||||
} catch (err) {
|
||||
lastErr = err;
|
||||
if (i === attempts - 1) break;
|
||||
const delay = initialDelayMs * 2 ** i;
|
||||
await sleep(delay);
|
||||
}
|
||||
}
|
||||
throw lastErr ?? new Error("Retry failed");
|
||||
}
|
||||
|
||||
const options = attemptsOrOptions;
|
||||
|
||||
const resolved = resolveRetryConfig(DEFAULT_RETRY_CONFIG, options);
|
||||
const maxAttempts = resolved.attempts;
|
||||
const minDelayMs = resolved.minDelayMs;
|
||||
const maxDelayMs =
|
||||
Number.isFinite(resolved.maxDelayMs) && resolved.maxDelayMs > 0
|
||||
? resolved.maxDelayMs
|
||||
: Number.POSITIVE_INFINITY;
|
||||
const jitter = resolved.jitter;
|
||||
const shouldRetry = options.shouldRetry ?? (() => true);
|
||||
let lastErr: unknown;
|
||||
for (let i = 0; i < attempts; i += 1) {
|
||||
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
||||
try {
|
||||
return await fn();
|
||||
} catch (err) {
|
||||
lastErr = err;
|
||||
if (i === attempts - 1) break;
|
||||
const delay = initialDelayMs * 2 ** i;
|
||||
await new Promise((r) => setTimeout(r, delay));
|
||||
if (attempt >= maxAttempts || !shouldRetry(err, attempt)) break;
|
||||
|
||||
const retryAfterMs = options.retryAfterMs?.(err);
|
||||
const hasRetryAfter =
|
||||
typeof retryAfterMs === "number" && Number.isFinite(retryAfterMs);
|
||||
const baseDelay = hasRetryAfter
|
||||
? Math.max(retryAfterMs, minDelayMs)
|
||||
: minDelayMs * 2 ** (attempt - 1);
|
||||
let delay = Math.min(baseDelay, maxDelayMs);
|
||||
delay = applyJitter(delay, jitter);
|
||||
delay = Math.min(Math.max(delay, minDelayMs), maxDelayMs);
|
||||
|
||||
options.onRetry?.({
|
||||
attempt,
|
||||
maxAttempts,
|
||||
delayMs: delay,
|
||||
err,
|
||||
label: options.label,
|
||||
});
|
||||
await sleep(delay);
|
||||
}
|
||||
}
|
||||
throw lastErr;
|
||||
|
||||
throw lastErr ?? new Error("Retry failed");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user