fix: wire OTLP logs for diagnostics
This commit is contained in:
Peter Steinberger
@@ -8,9 +8,12 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@opentelemetry/api": "^1.9.0",
|
||||
"@opentelemetry/api-logs": "^0.210.0",
|
||||
"@opentelemetry/exporter-logs-otlp-http": "^0.210.0",
|
||||
"@opentelemetry/exporter-metrics-otlp-http": "^0.210.0",
|
||||
"@opentelemetry/exporter-trace-otlp-http": "^0.210.0",
|
||||
"@opentelemetry/resources": "^2.4.0",
|
||||
"@opentelemetry/sdk-logs": "^0.210.0",
|
||||
"@opentelemetry/sdk-metrics": "^2.4.0",
|
||||
"@opentelemetry/sdk-node": "^0.210.0",
|
||||
"@opentelemetry/sdk-trace-base": "^2.4.0",
|
||||
|
||||
220
extensions/diagnostics-otel/src/service.test.ts
Normal file
220
extensions/diagnostics-otel/src/service.test.ts
Normal file
@@ -0,0 +1,220 @@
|
||||
import { beforeEach, describe, expect, test, vi } from "vitest";
|
||||
|
||||
const registerLogTransportMock = vi.hoisted(() => vi.fn());
|
||||
|
||||
const telemetryState = vi.hoisted(() => {
|
||||
const counters = new Map<string, { add: ReturnType<typeof vi.fn> }>();
|
||||
const histograms = new Map<string, { record: ReturnType<typeof vi.fn> }>();
|
||||
const tracer = {
|
||||
startSpan: vi.fn((_name: string, _opts?: unknown) => ({
|
||||
end: vi.fn(),
|
||||
setStatus: vi.fn(),
|
||||
})),
|
||||
};
|
||||
const meter = {
|
||||
createCounter: vi.fn((name: string) => {
|
||||
const counter = { add: vi.fn() };
|
||||
counters.set(name, counter);
|
||||
return counter;
|
||||
}),
|
||||
createHistogram: vi.fn((name: string) => {
|
||||
const histogram = { record: vi.fn() };
|
||||
histograms.set(name, histogram);
|
||||
return histogram;
|
||||
}),
|
||||
};
|
||||
return { counters, histograms, tracer, meter };
|
||||
});
|
||||
|
||||
const sdkStart = vi.hoisted(() => vi.fn().mockResolvedValue(undefined));
|
||||
const sdkShutdown = vi.hoisted(() => vi.fn().mockResolvedValue(undefined));
|
||||
const logEmit = vi.hoisted(() => vi.fn());
|
||||
const logShutdown = vi.hoisted(() => vi.fn().mockResolvedValue(undefined));
|
||||
|
||||
vi.mock("@opentelemetry/api", () => ({
|
||||
metrics: {
|
||||
getMeter: () => telemetryState.meter,
|
||||
},
|
||||
trace: {
|
||||
getTracer: () => telemetryState.tracer,
|
||||
},
|
||||
SpanStatusCode: {
|
||||
ERROR: 2,
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("@opentelemetry/sdk-node", () => ({
|
||||
NodeSDK: class {
|
||||
start = sdkStart;
|
||||
shutdown = sdkShutdown;
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("@opentelemetry/exporter-metrics-otlp-http", () => ({
|
||||
OTLPMetricExporter: class {},
|
||||
}));
|
||||
|
||||
vi.mock("@opentelemetry/exporter-trace-otlp-http", () => ({
|
||||
OTLPTraceExporter: class {},
|
||||
}));
|
||||
|
||||
vi.mock("@opentelemetry/exporter-logs-otlp-http", () => ({
|
||||
OTLPLogExporter: class {},
|
||||
}));
|
||||
|
||||
vi.mock("@opentelemetry/sdk-logs", () => ({
|
||||
BatchLogRecordProcessor: class {},
|
||||
LoggerProvider: class {
|
||||
addLogRecordProcessor = vi.fn();
|
||||
getLogger = vi.fn(() => ({
|
||||
emit: logEmit,
|
||||
}));
|
||||
shutdown = logShutdown;
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("@opentelemetry/sdk-metrics", () => ({
|
||||
PeriodicExportingMetricReader: class {},
|
||||
}));
|
||||
|
||||
vi.mock("@opentelemetry/sdk-trace-base", () => ({
|
||||
ParentBasedSampler: class {},
|
||||
TraceIdRatioBasedSampler: class {},
|
||||
}));
|
||||
|
||||
vi.mock("@opentelemetry/resources", () => ({
|
||||
Resource: class {
|
||||
// eslint-disable-next-line @typescript-eslint/no-useless-constructor
|
||||
constructor(_value?: unknown) {}
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("@opentelemetry/semantic-conventions", () => ({
|
||||
SemanticResourceAttributes: {
|
||||
SERVICE_NAME: "service.name",
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("clawdbot/plugin-sdk", async () => {
|
||||
const actual = await vi.importActual<typeof import("clawdbot/plugin-sdk")>("clawdbot/plugin-sdk");
|
||||
return {
|
||||
...actual,
|
||||
registerLogTransport: registerLogTransportMock,
|
||||
};
|
||||
});
|
||||
|
||||
import { createDiagnosticsOtelService } from "./service.js";
|
||||
import { emitDiagnosticEvent } from "clawdbot/plugin-sdk";
|
||||
|
||||
describe("diagnostics-otel service", () => {
|
||||
beforeEach(() => {
|
||||
telemetryState.counters.clear();
|
||||
telemetryState.histograms.clear();
|
||||
telemetryState.tracer.startSpan.mockClear();
|
||||
telemetryState.meter.createCounter.mockClear();
|
||||
telemetryState.meter.createHistogram.mockClear();
|
||||
sdkStart.mockClear();
|
||||
sdkShutdown.mockClear();
|
||||
logEmit.mockClear();
|
||||
logShutdown.mockClear();
|
||||
registerLogTransportMock.mockReset();
|
||||
});
|
||||
|
||||
test("records message-flow metrics and spans", async () => {
|
||||
const registeredTransports: Array<(logObj: Record<string, unknown>) => void> = [];
|
||||
const stopTransport = vi.fn();
|
||||
registerLogTransportMock.mockImplementation((transport) => {
|
||||
registeredTransports.push(transport);
|
||||
return stopTransport;
|
||||
});
|
||||
|
||||
const service = createDiagnosticsOtelService();
|
||||
await service.start({
|
||||
config: {
|
||||
diagnostics: {
|
||||
enabled: true,
|
||||
otel: {
|
||||
enabled: true,
|
||||
endpoint: "http://otel-collector:4318",
|
||||
protocol: "http/protobuf",
|
||||
traces: true,
|
||||
metrics: true,
|
||||
logs: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
logger: {
|
||||
info: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
debug: vi.fn(),
|
||||
},
|
||||
});
|
||||
|
||||
emitDiagnosticEvent({
|
||||
type: "webhook.received",
|
||||
channel: "telegram",
|
||||
updateType: "telegram-post",
|
||||
});
|
||||
emitDiagnosticEvent({
|
||||
type: "webhook.processed",
|
||||
channel: "telegram",
|
||||
updateType: "telegram-post",
|
||||
durationMs: 120,
|
||||
});
|
||||
emitDiagnosticEvent({
|
||||
type: "message.queued",
|
||||
channel: "telegram",
|
||||
source: "telegram",
|
||||
queueDepth: 2,
|
||||
});
|
||||
emitDiagnosticEvent({
|
||||
type: "message.processed",
|
||||
channel: "telegram",
|
||||
outcome: "completed",
|
||||
durationMs: 55,
|
||||
});
|
||||
emitDiagnosticEvent({
|
||||
type: "queue.lane.dequeue",
|
||||
lane: "main",
|
||||
queueSize: 3,
|
||||
waitMs: 10,
|
||||
});
|
||||
emitDiagnosticEvent({
|
||||
type: "session.stuck",
|
||||
state: "processing",
|
||||
ageMs: 125_000,
|
||||
});
|
||||
emitDiagnosticEvent({
|
||||
type: "run.attempt",
|
||||
runId: "run-1",
|
||||
attempt: 2,
|
||||
});
|
||||
|
||||
expect(telemetryState.counters.get("clawdbot.webhook.received")?.add).toHaveBeenCalled();
|
||||
expect(telemetryState.histograms.get("clawdbot.webhook.duration_ms")?.record).toHaveBeenCalled();
|
||||
expect(telemetryState.counters.get("clawdbot.message.queued")?.add).toHaveBeenCalled();
|
||||
expect(telemetryState.counters.get("clawdbot.message.processed")?.add).toHaveBeenCalled();
|
||||
expect(telemetryState.histograms.get("clawdbot.message.duration_ms")?.record).toHaveBeenCalled();
|
||||
expect(telemetryState.histograms.get("clawdbot.queue.wait_ms")?.record).toHaveBeenCalled();
|
||||
expect(telemetryState.counters.get("clawdbot.session.stuck")?.add).toHaveBeenCalled();
|
||||
expect(telemetryState.histograms.get("clawdbot.session.stuck_age_ms")?.record).toHaveBeenCalled();
|
||||
expect(telemetryState.counters.get("clawdbot.run.attempt")?.add).toHaveBeenCalled();
|
||||
|
||||
const spanNames = telemetryState.tracer.startSpan.mock.calls.map((call) => call[0]);
|
||||
expect(spanNames).toContain("clawdbot.webhook.processed");
|
||||
expect(spanNames).toContain("clawdbot.message.processed");
|
||||
expect(spanNames).toContain("clawdbot.session.stuck");
|
||||
|
||||
expect(registerLogTransportMock).toHaveBeenCalledTimes(1);
|
||||
expect(registeredTransports).toHaveLength(1);
|
||||
registeredTransports[0]?.({
|
||||
0: "{\"subsystem\":\"diagnostic\"}",
|
||||
1: "hello",
|
||||
_meta: { logLevelName: "INFO", date: new Date() },
|
||||
});
|
||||
expect(logEmit).toHaveBeenCalled();
|
||||
|
||||
await service.stop?.();
|
||||
});
|
||||
});
|
||||
@@ -1,17 +1,17 @@
|
||||
import { metrics, trace } from "@opentelemetry/api";
|
||||
import { metrics, trace, SpanStatusCode } from "@opentelemetry/api";
|
||||
import type { SeverityNumber } from "@opentelemetry/api-logs";
|
||||
import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http";
|
||||
import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http";
|
||||
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
|
||||
import { Resource } from "@opentelemetry/resources";
|
||||
import { BatchLogRecordProcessor, LoggerProvider } from "@opentelemetry/sdk-logs";
|
||||
import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
|
||||
import { NodeSDK } from "@opentelemetry/sdk-node";
|
||||
import { ParentBasedSampler, TraceIdRatioBasedSampler } from "@opentelemetry/sdk-trace-base";
|
||||
import { SemanticResourceAttributes } from "@opentelemetry/semantic-conventions";
|
||||
|
||||
import type {
|
||||
ClawdbotPluginService,
|
||||
DiagnosticUsageEvent,
|
||||
} from "clawdbot/plugin-sdk";
|
||||
import { onDiagnosticEvent } from "clawdbot/plugin-sdk";
|
||||
import type { ClawdbotPluginService, DiagnosticEventPayload } from "clawdbot/plugin-sdk";
|
||||
import { onDiagnosticEvent, registerLogTransport } from "clawdbot/plugin-sdk";
|
||||
|
||||
const DEFAULT_SERVICE_NAME = "clawdbot";
|
||||
|
||||
@@ -34,6 +34,8 @@ function resolveSampleRate(value: number | undefined): number | undefined {
|
||||
|
||||
export function createDiagnosticsOtelService(): ClawdbotPluginService {
|
||||
let sdk: NodeSDK | null = null;
|
||||
let logProvider: LoggerProvider | null = null;
|
||||
let stopLogTransport: (() => void) | null = null;
|
||||
let unsubscribe: (() => void) | null = null;
|
||||
|
||||
return {
|
||||
@@ -57,7 +59,8 @@ export function createDiagnosticsOtelService(): ClawdbotPluginService {
|
||||
|
||||
const tracesEnabled = otel.traces !== false;
|
||||
const metricsEnabled = otel.metrics !== false;
|
||||
if (!tracesEnabled && !metricsEnabled) return;
|
||||
const logsEnabled = otel.logs === true;
|
||||
if (!tracesEnabled && !metricsEnabled && !logsEnabled) return;
|
||||
|
||||
const resource = new Resource({
|
||||
[SemanticResourceAttributes.SERVICE_NAME]: serviceName,
|
||||
@@ -65,6 +68,7 @@ export function createDiagnosticsOtelService(): ClawdbotPluginService {
|
||||
|
||||
const traceUrl = resolveOtelUrl(endpoint, "v1/traces");
|
||||
const metricUrl = resolveOtelUrl(endpoint, "v1/metrics");
|
||||
const logUrl = resolveOtelUrl(endpoint, "v1/logs");
|
||||
const traceExporter = tracesEnabled
|
||||
? new OTLPTraceExporter({
|
||||
...(traceUrl ? { url: traceUrl } : {}),
|
||||
@@ -88,20 +92,31 @@ export function createDiagnosticsOtelService(): ClawdbotPluginService {
|
||||
})
|
||||
: undefined;
|
||||
|
||||
sdk = new NodeSDK({
|
||||
resource,
|
||||
...(traceExporter ? { traceExporter } : {}),
|
||||
...(metricReader ? { metricReader } : {}),
|
||||
...(sampleRate !== undefined
|
||||
? {
|
||||
sampler: new ParentBasedSampler({
|
||||
root: new TraceIdRatioBasedSampler(sampleRate),
|
||||
}),
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
if (tracesEnabled || metricsEnabled) {
|
||||
sdk = new NodeSDK({
|
||||
resource,
|
||||
...(traceExporter ? { traceExporter } : {}),
|
||||
...(metricReader ? { metricReader } : {}),
|
||||
...(sampleRate !== undefined
|
||||
? {
|
||||
sampler: new ParentBasedSampler({
|
||||
root: new TraceIdRatioBasedSampler(sampleRate),
|
||||
}),
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
|
||||
await sdk.start();
|
||||
await sdk.start();
|
||||
}
|
||||
|
||||
const logSeverityMap: Record<string, SeverityNumber> = {
|
||||
TRACE: 1 as SeverityNumber,
|
||||
DEBUG: 5 as SeverityNumber,
|
||||
INFO: 9 as SeverityNumber,
|
||||
WARN: 13 as SeverityNumber,
|
||||
ERROR: 17 as SeverityNumber,
|
||||
FATAL: 21 as SeverityNumber,
|
||||
};
|
||||
|
||||
const meter = metrics.getMeter("clawdbot");
|
||||
const tracer = trace.getTracer("clawdbot");
|
||||
@@ -122,38 +137,210 @@ export function createDiagnosticsOtelService(): ClawdbotPluginService {
|
||||
unit: "1",
|
||||
description: "Context window size and usage",
|
||||
});
|
||||
const webhookReceivedCounter = meter.createCounter("clawdbot.webhook.received", {
|
||||
unit: "1",
|
||||
description: "Webhook requests received",
|
||||
});
|
||||
const webhookErrorCounter = meter.createCounter("clawdbot.webhook.error", {
|
||||
unit: "1",
|
||||
description: "Webhook processing errors",
|
||||
});
|
||||
const webhookDurationHistogram = meter.createHistogram("clawdbot.webhook.duration_ms", {
|
||||
unit: "ms",
|
||||
description: "Webhook processing duration",
|
||||
});
|
||||
const messageQueuedCounter = meter.createCounter("clawdbot.message.queued", {
|
||||
unit: "1",
|
||||
description: "Messages queued for processing",
|
||||
});
|
||||
const messageProcessedCounter = meter.createCounter("clawdbot.message.processed", {
|
||||
unit: "1",
|
||||
description: "Messages processed by outcome",
|
||||
});
|
||||
const messageDurationHistogram = meter.createHistogram("clawdbot.message.duration_ms", {
|
||||
unit: "ms",
|
||||
description: "Message processing duration",
|
||||
});
|
||||
const queueDepthHistogram = meter.createHistogram("clawdbot.queue.depth", {
|
||||
unit: "1",
|
||||
description: "Queue depth on enqueue/dequeue",
|
||||
});
|
||||
const queueWaitHistogram = meter.createHistogram("clawdbot.queue.wait_ms", {
|
||||
unit: "ms",
|
||||
description: "Queue wait time before execution",
|
||||
});
|
||||
const laneEnqueueCounter = meter.createCounter("clawdbot.queue.lane.enqueue", {
|
||||
unit: "1",
|
||||
description: "Command queue lane enqueue events",
|
||||
});
|
||||
const laneDequeueCounter = meter.createCounter("clawdbot.queue.lane.dequeue", {
|
||||
unit: "1",
|
||||
description: "Command queue lane dequeue events",
|
||||
});
|
||||
const sessionStateCounter = meter.createCounter("clawdbot.session.state", {
|
||||
unit: "1",
|
||||
description: "Session state transitions",
|
||||
});
|
||||
const sessionStuckCounter = meter.createCounter("clawdbot.session.stuck", {
|
||||
unit: "1",
|
||||
description: "Sessions stuck in processing",
|
||||
});
|
||||
const sessionStuckAgeHistogram = meter.createHistogram("clawdbot.session.stuck_age_ms", {
|
||||
unit: "ms",
|
||||
description: "Age of stuck sessions",
|
||||
});
|
||||
const runAttemptCounter = meter.createCounter("clawdbot.run.attempt", {
|
||||
unit: "1",
|
||||
description: "Run attempts",
|
||||
});
|
||||
|
||||
unsubscribe = onDiagnosticEvent((evt) => {
|
||||
if (evt.type !== "model.usage") return;
|
||||
const usageEvent = evt as DiagnosticUsageEvent;
|
||||
if (logsEnabled) {
|
||||
const logExporter = new OTLPLogExporter({
|
||||
...(logUrl ? { url: logUrl } : {}),
|
||||
...(headers ? { headers } : {}),
|
||||
});
|
||||
logProvider = new LoggerProvider({ resource });
|
||||
logProvider.addLogRecordProcessor(
|
||||
new BatchLogRecordProcessor(logExporter, {
|
||||
...(typeof otel.flushIntervalMs === "number"
|
||||
? { scheduledDelayMillis: Math.max(1000, otel.flushIntervalMs) }
|
||||
: {}),
|
||||
}),
|
||||
);
|
||||
const otelLogger = logProvider.getLogger("clawdbot");
|
||||
|
||||
stopLogTransport = registerLogTransport((logObj) => {
|
||||
const safeStringify = (value: unknown) => {
|
||||
try {
|
||||
return JSON.stringify(value);
|
||||
} catch {
|
||||
return String(value);
|
||||
}
|
||||
};
|
||||
const meta = (logObj as Record<string, unknown>)._meta as
|
||||
| {
|
||||
logLevelName?: string;
|
||||
date?: Date;
|
||||
name?: string;
|
||||
parentNames?: string[];
|
||||
path?: {
|
||||
filePath?: string;
|
||||
fileLine?: string;
|
||||
fileColumn?: string;
|
||||
filePathWithLine?: string;
|
||||
method?: string;
|
||||
};
|
||||
}
|
||||
| undefined;
|
||||
const logLevelName = meta?.logLevelName ?? "INFO";
|
||||
const severityNumber = logSeverityMap[logLevelName] ?? (9 as SeverityNumber);
|
||||
|
||||
const numericArgs = Object.entries(logObj)
|
||||
.filter(([key]) => /^\d+$/.test(key))
|
||||
.sort((a, b) => Number(a[0]) - Number(b[0]))
|
||||
.map(([, value]) => value);
|
||||
|
||||
let bindings: Record<string, unknown> | undefined;
|
||||
if (typeof numericArgs[0] === "string" && numericArgs[0].trim().startsWith("{")) {
|
||||
try {
|
||||
const parsed = JSON.parse(numericArgs[0]);
|
||||
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
||||
bindings = parsed as Record<string, unknown>;
|
||||
numericArgs.shift();
|
||||
}
|
||||
} catch {
|
||||
// ignore malformed json bindings
|
||||
}
|
||||
}
|
||||
|
||||
let message = "";
|
||||
if (numericArgs.length > 0 && typeof numericArgs[numericArgs.length - 1] === "string") {
|
||||
message = String(numericArgs.pop());
|
||||
} else if (numericArgs.length === 1) {
|
||||
message = safeStringify(numericArgs[0]);
|
||||
numericArgs.length = 0;
|
||||
}
|
||||
if (!message) {
|
||||
message = "log";
|
||||
}
|
||||
|
||||
const attributes: Record<string, string | number | boolean> = {
|
||||
"clawdbot.log.level": logLevelName,
|
||||
};
|
||||
if (meta?.name) attributes["clawdbot.logger"] = meta.name;
|
||||
if (meta?.parentNames?.length) {
|
||||
attributes["clawdbot.logger.parents"] = meta.parentNames.join(".");
|
||||
}
|
||||
if (bindings) {
|
||||
for (const [key, value] of Object.entries(bindings)) {
|
||||
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
||||
attributes[`clawdbot.${key}`] = value;
|
||||
} else if (value != null) {
|
||||
attributes[`clawdbot.${key}`] = safeStringify(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (numericArgs.length > 0) {
|
||||
attributes["clawdbot.log.args"] = safeStringify(numericArgs);
|
||||
}
|
||||
if (meta?.path?.filePath) attributes["code.filepath"] = meta.path.filePath;
|
||||
if (meta?.path?.fileLine) attributes["code.lineno"] = Number(meta.path.fileLine);
|
||||
if (meta?.path?.method) attributes["code.function"] = meta.path.method;
|
||||
if (meta?.path?.filePathWithLine) {
|
||||
attributes["clawdbot.code.location"] = meta.path.filePathWithLine;
|
||||
}
|
||||
|
||||
otelLogger.emit({
|
||||
body: message,
|
||||
severityText: logLevelName,
|
||||
severityNumber,
|
||||
attributes,
|
||||
timestamp: meta?.date ?? new Date(),
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
const spanWithDuration = (
|
||||
name: string,
|
||||
attributes: Record<string, string | number>,
|
||||
durationMs?: number,
|
||||
) => {
|
||||
const startTime =
|
||||
typeof durationMs === "number" ? Date.now() - Math.max(0, durationMs) : undefined;
|
||||
const span = tracer.startSpan(name, {
|
||||
attributes,
|
||||
...(startTime ? { startTime } : {}),
|
||||
});
|
||||
return span;
|
||||
};
|
||||
|
||||
const recordModelUsage = (evt: Extract<DiagnosticEventPayload, { type: "model.usage" }>) => {
|
||||
const attrs = {
|
||||
"clawdbot.channel": usageEvent.channel ?? "unknown",
|
||||
"clawdbot.provider": usageEvent.provider ?? "unknown",
|
||||
"clawdbot.model": usageEvent.model ?? "unknown",
|
||||
"clawdbot.channel": evt.channel ?? "unknown",
|
||||
"clawdbot.provider": evt.provider ?? "unknown",
|
||||
"clawdbot.model": evt.model ?? "unknown",
|
||||
};
|
||||
|
||||
const usage = usageEvent.usage;
|
||||
const usage = evt.usage;
|
||||
if (usage.input) tokensCounter.add(usage.input, { ...attrs, "clawdbot.token": "input" });
|
||||
if (usage.output)
|
||||
tokensCounter.add(usage.output, { ...attrs, "clawdbot.token": "output" });
|
||||
if (usage.output) tokensCounter.add(usage.output, { ...attrs, "clawdbot.token": "output" });
|
||||
if (usage.cacheRead)
|
||||
tokensCounter.add(usage.cacheRead, { ...attrs, "clawdbot.token": "cache_read" });
|
||||
if (usage.cacheWrite)
|
||||
tokensCounter.add(usage.cacheWrite, { ...attrs, "clawdbot.token": "cache_write" });
|
||||
if (usage.promptTokens)
|
||||
tokensCounter.add(usage.promptTokens, { ...attrs, "clawdbot.token": "prompt" });
|
||||
if (usage.total)
|
||||
tokensCounter.add(usage.total, { ...attrs, "clawdbot.token": "total" });
|
||||
if (usage.total) tokensCounter.add(usage.total, { ...attrs, "clawdbot.token": "total" });
|
||||
|
||||
if (usageEvent.costUsd) costCounter.add(usageEvent.costUsd, attrs);
|
||||
if (usageEvent.durationMs) durationHistogram.record(usageEvent.durationMs, attrs);
|
||||
if (usageEvent.context?.limit)
|
||||
contextHistogram.record(usageEvent.context.limit, {
|
||||
if (evt.costUsd) costCounter.add(evt.costUsd, attrs);
|
||||
if (evt.durationMs) durationHistogram.record(evt.durationMs, attrs);
|
||||
if (evt.context?.limit)
|
||||
contextHistogram.record(evt.context.limit, {
|
||||
...attrs,
|
||||
"clawdbot.context": "limit",
|
||||
});
|
||||
if (usageEvent.context?.used)
|
||||
contextHistogram.record(usageEvent.context.used, {
|
||||
if (evt.context?.used)
|
||||
contextHistogram.record(evt.context.used, {
|
||||
...attrs,
|
||||
"clawdbot.context": "used",
|
||||
});
|
||||
@@ -161,8 +348,8 @@ export function createDiagnosticsOtelService(): ClawdbotPluginService {
|
||||
if (!tracesEnabled) return;
|
||||
const spanAttrs: Record<string, string | number> = {
|
||||
...attrs,
|
||||
"clawdbot.sessionKey": usageEvent.sessionKey ?? "",
|
||||
"clawdbot.sessionId": usageEvent.sessionId ?? "",
|
||||
"clawdbot.sessionKey": evt.sessionKey ?? "",
|
||||
"clawdbot.sessionId": evt.sessionId ?? "",
|
||||
"clawdbot.tokens.input": usage.input ?? 0,
|
||||
"clawdbot.tokens.output": usage.output ?? 0,
|
||||
"clawdbot.tokens.cache_read": usage.cacheRead ?? 0,
|
||||
@@ -170,23 +357,206 @@ export function createDiagnosticsOtelService(): ClawdbotPluginService {
|
||||
"clawdbot.tokens.total": usage.total ?? 0,
|
||||
};
|
||||
|
||||
const startTime = usageEvent.durationMs
|
||||
? Date.now() - Math.max(0, usageEvent.durationMs)
|
||||
: undefined;
|
||||
const span = tracer.startSpan("clawdbot.model.usage", {
|
||||
attributes: spanAttrs,
|
||||
...(startTime ? { startTime } : {}),
|
||||
});
|
||||
const span = spanWithDuration("clawdbot.model.usage", spanAttrs, evt.durationMs);
|
||||
span.end();
|
||||
};
|
||||
|
||||
const recordWebhookReceived = (
|
||||
evt: Extract<DiagnosticEventPayload, { type: "webhook.received" }>,
|
||||
) => {
|
||||
const attrs = {
|
||||
"clawdbot.channel": evt.channel ?? "unknown",
|
||||
"clawdbot.webhook": evt.updateType ?? "unknown",
|
||||
};
|
||||
webhookReceivedCounter.add(1, attrs);
|
||||
};
|
||||
|
||||
const recordWebhookProcessed = (
|
||||
evt: Extract<DiagnosticEventPayload, { type: "webhook.processed" }>,
|
||||
) => {
|
||||
const attrs = {
|
||||
"clawdbot.channel": evt.channel ?? "unknown",
|
||||
"clawdbot.webhook": evt.updateType ?? "unknown",
|
||||
};
|
||||
if (typeof evt.durationMs === "number") {
|
||||
webhookDurationHistogram.record(evt.durationMs, attrs);
|
||||
}
|
||||
if (!tracesEnabled) return;
|
||||
const spanAttrs: Record<string, string | number> = { ...attrs };
|
||||
if (evt.chatId !== undefined) spanAttrs["clawdbot.chatId"] = String(evt.chatId);
|
||||
const span = spanWithDuration("clawdbot.webhook.processed", spanAttrs, evt.durationMs);
|
||||
span.end();
|
||||
};
|
||||
|
||||
const recordWebhookError = (
|
||||
evt: Extract<DiagnosticEventPayload, { type: "webhook.error" }>,
|
||||
) => {
|
||||
const attrs = {
|
||||
"clawdbot.channel": evt.channel ?? "unknown",
|
||||
"clawdbot.webhook": evt.updateType ?? "unknown",
|
||||
};
|
||||
webhookErrorCounter.add(1, attrs);
|
||||
if (!tracesEnabled) return;
|
||||
const spanAttrs: Record<string, string | number> = {
|
||||
...attrs,
|
||||
"clawdbot.error": evt.error,
|
||||
};
|
||||
if (evt.chatId !== undefined) spanAttrs["clawdbot.chatId"] = String(evt.chatId);
|
||||
const span = tracer.startSpan("clawdbot.webhook.error", {
|
||||
attributes: spanAttrs,
|
||||
});
|
||||
span.setStatus({ code: SpanStatusCode.ERROR, message: evt.error });
|
||||
span.end();
|
||||
};
|
||||
|
||||
const recordMessageQueued = (
|
||||
evt: Extract<DiagnosticEventPayload, { type: "message.queued" }>,
|
||||
) => {
|
||||
const attrs = {
|
||||
"clawdbot.channel": evt.channel ?? "unknown",
|
||||
"clawdbot.source": evt.source ?? "unknown",
|
||||
};
|
||||
messageQueuedCounter.add(1, attrs);
|
||||
if (typeof evt.queueDepth === "number") {
|
||||
queueDepthHistogram.record(evt.queueDepth, attrs);
|
||||
}
|
||||
};
|
||||
|
||||
const recordMessageProcessed = (
|
||||
evt: Extract<DiagnosticEventPayload, { type: "message.processed" }>,
|
||||
) => {
|
||||
const attrs = {
|
||||
"clawdbot.channel": evt.channel ?? "unknown",
|
||||
"clawdbot.outcome": evt.outcome ?? "unknown",
|
||||
};
|
||||
messageProcessedCounter.add(1, attrs);
|
||||
if (typeof evt.durationMs === "number") {
|
||||
messageDurationHistogram.record(evt.durationMs, attrs);
|
||||
}
|
||||
if (!tracesEnabled) return;
|
||||
const spanAttrs: Record<string, string | number> = { ...attrs };
|
||||
if (evt.sessionKey) spanAttrs["clawdbot.sessionKey"] = evt.sessionKey;
|
||||
if (evt.sessionId) spanAttrs["clawdbot.sessionId"] = evt.sessionId;
|
||||
if (evt.chatId !== undefined) spanAttrs["clawdbot.chatId"] = String(evt.chatId);
|
||||
if (evt.messageId !== undefined) spanAttrs["clawdbot.messageId"] = String(evt.messageId);
|
||||
if (evt.reason) spanAttrs["clawdbot.reason"] = evt.reason;
|
||||
const span = spanWithDuration("clawdbot.message.processed", spanAttrs, evt.durationMs);
|
||||
if (evt.outcome === "error") {
|
||||
span.setStatus({ code: SpanStatusCode.ERROR, message: evt.error });
|
||||
}
|
||||
span.end();
|
||||
};
|
||||
|
||||
const recordLaneEnqueue = (
|
||||
evt: Extract<DiagnosticEventPayload, { type: "queue.lane.enqueue" }>,
|
||||
) => {
|
||||
const attrs = { "clawdbot.lane": evt.lane };
|
||||
laneEnqueueCounter.add(1, attrs);
|
||||
queueDepthHistogram.record(evt.queueSize, attrs);
|
||||
};
|
||||
|
||||
const recordLaneDequeue = (
|
||||
evt: Extract<DiagnosticEventPayload, { type: "queue.lane.dequeue" }>,
|
||||
) => {
|
||||
const attrs = { "clawdbot.lane": evt.lane };
|
||||
laneDequeueCounter.add(1, attrs);
|
||||
queueDepthHistogram.record(evt.queueSize, attrs);
|
||||
if (typeof evt.waitMs === "number") {
|
||||
queueWaitHistogram.record(evt.waitMs, attrs);
|
||||
}
|
||||
};
|
||||
|
||||
const recordSessionState = (
|
||||
evt: Extract<DiagnosticEventPayload, { type: "session.state" }>,
|
||||
) => {
|
||||
const attrs: Record<string, string> = { "clawdbot.state": evt.state };
|
||||
if (evt.reason) attrs["clawdbot.reason"] = evt.reason;
|
||||
sessionStateCounter.add(1, attrs);
|
||||
};
|
||||
|
||||
const recordSessionStuck = (
|
||||
evt: Extract<DiagnosticEventPayload, { type: "session.stuck" }>,
|
||||
) => {
|
||||
const attrs: Record<string, string> = { "clawdbot.state": evt.state };
|
||||
sessionStuckCounter.add(1, attrs);
|
||||
if (typeof evt.ageMs === "number") {
|
||||
sessionStuckAgeHistogram.record(evt.ageMs, attrs);
|
||||
}
|
||||
if (!tracesEnabled) return;
|
||||
const spanAttrs: Record<string, string | number> = { ...attrs };
|
||||
if (evt.sessionKey) spanAttrs["clawdbot.sessionKey"] = evt.sessionKey;
|
||||
if (evt.sessionId) spanAttrs["clawdbot.sessionId"] = evt.sessionId;
|
||||
spanAttrs["clawdbot.queueDepth"] = evt.queueDepth ?? 0;
|
||||
spanAttrs["clawdbot.ageMs"] = evt.ageMs;
|
||||
const span = tracer.startSpan("clawdbot.session.stuck", { attributes: spanAttrs });
|
||||
span.setStatus({ code: SpanStatusCode.ERROR, message: "session stuck" });
|
||||
span.end();
|
||||
};
|
||||
|
||||
const recordRunAttempt = (evt: Extract<DiagnosticEventPayload, { type: "run.attempt" }>) => {
|
||||
runAttemptCounter.add(1, { "clawdbot.attempt": evt.attempt });
|
||||
};
|
||||
|
||||
const recordHeartbeat = (
|
||||
evt: Extract<DiagnosticEventPayload, { type: "diagnostic.heartbeat" }>,
|
||||
) => {
|
||||
queueDepthHistogram.record(evt.queued, { "clawdbot.channel": "heartbeat" });
|
||||
};
|
||||
|
||||
unsubscribe = onDiagnosticEvent((evt: DiagnosticEventPayload) => {
|
||||
switch (evt.type) {
|
||||
case "model.usage":
|
||||
recordModelUsage(evt);
|
||||
return;
|
||||
case "webhook.received":
|
||||
recordWebhookReceived(evt);
|
||||
return;
|
||||
case "webhook.processed":
|
||||
recordWebhookProcessed(evt);
|
||||
return;
|
||||
case "webhook.error":
|
||||
recordWebhookError(evt);
|
||||
return;
|
||||
case "message.queued":
|
||||
recordMessageQueued(evt);
|
||||
return;
|
||||
case "message.processed":
|
||||
recordMessageProcessed(evt);
|
||||
return;
|
||||
case "queue.lane.enqueue":
|
||||
recordLaneEnqueue(evt);
|
||||
return;
|
||||
case "queue.lane.dequeue":
|
||||
recordLaneDequeue(evt);
|
||||
return;
|
||||
case "session.state":
|
||||
recordSessionState(evt);
|
||||
return;
|
||||
case "session.stuck":
|
||||
recordSessionStuck(evt);
|
||||
return;
|
||||
case "run.attempt":
|
||||
recordRunAttempt(evt);
|
||||
return;
|
||||
case "diagnostic.heartbeat":
|
||||
recordHeartbeat(evt);
|
||||
return;
|
||||
}
|
||||
});
|
||||
|
||||
if (otel.logs) {
|
||||
ctx.logger.warn("diagnostics-otel: logs exporter not wired yet");
|
||||
if (logsEnabled) {
|
||||
ctx.logger.info("diagnostics-otel: logs exporter enabled (OTLP/HTTP)");
|
||||
}
|
||||
},
|
||||
async stop() {
|
||||
unsubscribe?.();
|
||||
unsubscribe = null;
|
||||
stopLogTransport?.();
|
||||
stopLogTransport = null;
|
||||
if (logProvider) {
|
||||
await logProvider.shutdown().catch(() => undefined);
|
||||
logProvider = null;
|
||||
}
|
||||
if (sdk) {
|
||||
await sdk.shutdown().catch(() => undefined);
|
||||
sdk = null;
|
||||
|
||||
Reference in New Issue
Block a user