fix(webchat): support image-only sends

This commit is contained in:
Peter Steinberger
2026-01-26 05:32:29 +00:00
parent 9ba4b1e32b
commit 6859e1e6a6
11 changed files with 93 additions and 27 deletions

View File

@@ -34,6 +34,9 @@ Status: unreleased.
- Slack: clear ack reaction after streamed replies. (#2044) Thanks @fancyboi999. - Slack: clear ack reaction after streamed replies. (#2044) Thanks @fancyboi999.
- macOS: keep custom SSH usernames in remote target. (#2046) Thanks @algal. - macOS: keep custom SSH usernames in remote target. (#2046) Thanks @algal.
### Fixes
- Web UI: improve WebChat image paste previews and allow image-only sends. (#1925) Thanks @smartprogrammer93.
## 2026.1.24-3 ## 2026.1.24-3
### Fixes ### Fixes

View File

@@ -35,7 +35,7 @@ export const ChatHistoryParamsSchema = Type.Object(
export const ChatSendParamsSchema = Type.Object( export const ChatSendParamsSchema = Type.Object(
{ {
sessionKey: NonEmptyString, sessionKey: NonEmptyString,
message: NonEmptyString, message: Type.String(),
thinking: Type.Optional(Type.String()), thinking: Type.Optional(Type.String()),
deliver: Type.Optional(Type.Boolean()), deliver: Type.Optional(Type.Boolean()),
attachments: Type.Optional(Type.Array(Type.Unknown())), attachments: Type.Optional(Type.Array(Type.Unknown())),

View File

@@ -338,6 +338,15 @@ export const chatHandlers: GatewayRequestHandlers = {
: undefined, : undefined,
})) }))
.filter((a) => a.content) ?? []; .filter((a) => a.content) ?? [];
const rawMessage = p.message.trim();
if (!rawMessage && normalizedAttachments.length === 0) {
respond(
false,
undefined,
errorShape(ErrorCodes.INVALID_REQUEST, "message or attachment required"),
);
return;
}
let parsedMessage = p.message; let parsedMessage = p.message;
let parsedImages: ChatImageContent[] = []; let parsedImages: ChatImageContent[] = [];
if (normalizedAttachments.length > 0) { if (normalizedAttachments.length > 0) {

View File

@@ -208,6 +208,39 @@ describe("gateway server chat", () => {
| undefined; | undefined;
expect(imgOpts?.images).toEqual([{ type: "image", data: pngB64, mimeType: "image/png" }]); expect(imgOpts?.images).toEqual([{ type: "image", data: pngB64, mimeType: "image/png" }]);
const callsBeforeImageOnly = spy.mock.calls.length;
const reqIdOnly = "chat-img-only";
ws.send(
JSON.stringify({
type: "req",
id: reqIdOnly,
method: "chat.send",
params: {
sessionKey: "main",
message: "",
idempotencyKey: "idem-img-only",
attachments: [
{
type: "image",
mimeType: "image/png",
fileName: "dot.png",
content: `data:image/png;base64,${pngB64}`,
},
],
},
}),
);
const imgOnlyRes = await onceMessage(ws, (o) => o.type === "res" && o.id === reqIdOnly, 8000);
expect(imgOnlyRes.ok).toBe(true);
expect(imgOnlyRes.payload?.runId).toBeDefined();
await waitFor(() => spy.mock.calls.length > callsBeforeImageOnly, 8000);
const imgOnlyOpts = spy.mock.calls.at(-1)?.[1] as
| { images?: Array<{ type: string; data: string; mimeType: string }> }
| undefined;
expect(imgOnlyOpts?.images).toEqual([{ type: "image", data: pngB64, mimeType: "image/png" }]);
const historyDir = await fs.mkdtemp(path.join(os.tmpdir(), "clawdbot-gw-")); const historyDir = await fs.mkdtemp(path.join(os.tmpdir(), "clawdbot-gw-"));
tempDirs.push(historyDir); tempDirs.push(historyDir);
testState.sessionStorePath = path.join(historyDir, "sessions.json"); testState.sessionStorePath = path.join(historyDir, "sessions.json");

View File

@@ -1,4 +1,4 @@
import { abortChatRun, loadChatHistory, sendChatMessage, type ChatAttachment } from "./controllers/chat"; import { abortChatRun, loadChatHistory, sendChatMessage } from "./controllers/chat";
import { loadSessions } from "./controllers/sessions"; import { loadSessions } from "./controllers/sessions";
import { generateUUID } from "./uuid"; import { generateUUID } from "./uuid";
import { resetToolStream } from "./app-tool-stream"; import { resetToolStream } from "./app-tool-stream";
@@ -8,12 +8,13 @@ import { normalizeBasePath } from "./navigation";
import type { GatewayHelloOk } from "./gateway"; import type { GatewayHelloOk } from "./gateway";
import { parseAgentSessionKey } from "../../../src/sessions/session-key-utils.js"; import { parseAgentSessionKey } from "../../../src/sessions/session-key-utils.js";
import type { ClawdbotApp } from "./app"; import type { ClawdbotApp } from "./app";
import type { ChatAttachment, ChatQueueItem } from "./ui-types";
type ChatHost = { type ChatHost = {
connected: boolean; connected: boolean;
chatMessage: string; chatMessage: string;
chatAttachments: ChatAttachment[]; chatAttachments: ChatAttachment[];
chatQueue: Array<{ id: string; text: string; createdAt: number }>; chatQueue: ChatQueueItem[];
chatRunId: string | null; chatRunId: string | null;
chatSending: boolean; chatSending: boolean;
sessionKey: string; sessionKey: string;
@@ -46,15 +47,17 @@ export async function handleAbortChat(host: ChatHost) {
await abortChatRun(host as unknown as ClawdbotApp); await abortChatRun(host as unknown as ClawdbotApp);
} }
function enqueueChatMessage(host: ChatHost, text: string) { function enqueueChatMessage(host: ChatHost, text: string, attachments?: ChatAttachment[]) {
const trimmed = text.trim(); const trimmed = text.trim();
if (!trimmed) return; const hasAttachments = Boolean(attachments && attachments.length > 0);
if (!trimmed && !hasAttachments) return;
host.chatQueue = [ host.chatQueue = [
...host.chatQueue, ...host.chatQueue,
{ {
id: generateUUID(), id: generateUUID(),
text: trimmed, text: trimmed,
createdAt: Date.now(), createdAt: Date.now(),
attachments: hasAttachments ? attachments?.map((att) => ({ ...att })) : undefined,
}, },
]; ];
} }
@@ -62,19 +65,31 @@ function enqueueChatMessage(host: ChatHost, text: string) {
async function sendChatMessageNow( async function sendChatMessageNow(
host: ChatHost, host: ChatHost,
message: string, message: string,
opts?: { previousDraft?: string; restoreDraft?: boolean; attachments?: ChatAttachment[] }, opts?: {
previousDraft?: string;
restoreDraft?: boolean;
attachments?: ChatAttachment[];
previousAttachments?: ChatAttachment[];
restoreAttachments?: boolean;
},
) { ) {
resetToolStream(host as unknown as Parameters<typeof resetToolStream>[0]); resetToolStream(host as unknown as Parameters<typeof resetToolStream>[0]);
const ok = await sendChatMessage(host as unknown as ClawdbotApp, message, opts?.attachments); const ok = await sendChatMessage(host as unknown as ClawdbotApp, message, opts?.attachments);
if (!ok && opts?.previousDraft != null) { if (!ok && opts?.previousDraft != null) {
host.chatMessage = opts.previousDraft; host.chatMessage = opts.previousDraft;
} }
if (!ok && opts?.previousAttachments) {
host.chatAttachments = opts.previousAttachments;
}
if (ok) { if (ok) {
setLastActiveSessionKey(host as unknown as Parameters<typeof setLastActiveSessionKey>[0], host.sessionKey); setLastActiveSessionKey(host as unknown as Parameters<typeof setLastActiveSessionKey>[0], host.sessionKey);
} }
if (ok && opts?.restoreDraft && opts.previousDraft?.trim()) { if (ok && opts?.restoreDraft && opts.previousDraft?.trim()) {
host.chatMessage = opts.previousDraft; host.chatMessage = opts.previousDraft;
} }
if (ok && opts?.restoreAttachments && opts.previousAttachments?.length) {
host.chatAttachments = opts.previousAttachments;
}
scheduleChatScroll(host as unknown as Parameters<typeof scheduleChatScroll>[0]); scheduleChatScroll(host as unknown as Parameters<typeof scheduleChatScroll>[0]);
if (ok && !host.chatRunId) { if (ok && !host.chatRunId) {
void flushChatQueue(host); void flushChatQueue(host);
@@ -87,7 +102,7 @@ async function flushChatQueue(host: ChatHost) {
const [next, ...rest] = host.chatQueue; const [next, ...rest] = host.chatQueue;
if (!next) return; if (!next) return;
host.chatQueue = rest; host.chatQueue = rest;
const ok = await sendChatMessageNow(host, next.text); const ok = await sendChatMessageNow(host, next.text, { attachments: next.attachments });
if (!ok) { if (!ok) {
host.chatQueue = [next, ...host.chatQueue]; host.chatQueue = [next, ...host.chatQueue];
} }
@@ -106,7 +121,8 @@ export async function handleSendChat(
const previousDraft = host.chatMessage; const previousDraft = host.chatMessage;
const message = (messageOverride ?? host.chatMessage).trim(); const message = (messageOverride ?? host.chatMessage).trim();
const attachments = host.chatAttachments ?? []; const attachments = host.chatAttachments ?? [];
const hasAttachments = attachments.length > 0; const attachmentsToSend = messageOverride == null ? attachments : [];
const hasAttachments = attachmentsToSend.length > 0;
// Allow sending with just attachments (no message text required) // Allow sending with just attachments (no message text required)
if (!message && !hasAttachments) return; if (!message && !hasAttachments) return;
@@ -123,14 +139,16 @@ export async function handleSendChat(
} }
if (isChatBusy(host)) { if (isChatBusy(host)) {
enqueueChatMessage(host, message); enqueueChatMessage(host, message, attachmentsToSend);
return; return;
} }
await sendChatMessageNow(host, message, { await sendChatMessageNow(host, message, {
previousDraft: messageOverride == null ? previousDraft : undefined, previousDraft: messageOverride == null ? previousDraft : undefined,
restoreDraft: Boolean(messageOverride && opts?.restoreDraft), restoreDraft: Boolean(messageOverride && opts?.restoreDraft),
attachments: hasAttachments ? attachments : undefined, attachments: hasAttachments ? attachmentsToSend : undefined,
previousAttachments: messageOverride == null ? attachments : undefined,
restoreAttachments: Boolean(messageOverride && opts?.restoreDraft),
}); });
} }

View File

@@ -431,6 +431,7 @@ export function renderApp(state: AppViewState) {
onSessionKeyChange: (next) => { onSessionKeyChange: (next) => {
state.sessionKey = next; state.sessionKey = next;
state.chatMessage = ""; state.chatMessage = "";
state.chatAttachments = [];
state.chatStream = null; state.chatStream = null;
state.chatStreamStartedAt = null; state.chatStreamStartedAt = null;
state.chatRunId = null; state.chatRunId = null;

View File

@@ -19,7 +19,7 @@ import type {
SkillStatusReport, SkillStatusReport,
StatusSummary, StatusSummary,
} from "./types"; } from "./types";
import type { ChatQueueItem, CronFormState } from "./ui-types"; import type { ChatAttachment, ChatQueueItem, CronFormState } from "./ui-types";
import type { EventLogEntry } from "./app-events"; import type { EventLogEntry } from "./app-events";
import type { SkillMessage } from "./controllers/skills"; import type { SkillMessage } from "./controllers/skills";
import type { import type {
@@ -49,6 +49,7 @@ export type AppViewState = {
chatLoading: boolean; chatLoading: boolean;
chatSending: boolean; chatSending: boolean;
chatMessage: string; chatMessage: string;
chatAttachments: ChatAttachment[];
chatMessages: unknown[]; chatMessages: unknown[];
chatToolMessages: unknown[]; chatToolMessages: unknown[];
chatStream: string | null; chatStream: string | null;

View File

@@ -24,7 +24,7 @@ import type {
StatusSummary, StatusSummary,
NostrProfile, NostrProfile,
} from "./types"; } from "./types";
import { type ChatQueueItem, type CronFormState } from "./ui-types"; import { type ChatAttachment, type ChatQueueItem, type CronFormState } from "./ui-types";
import type { EventLogEntry } from "./app-events"; import type { EventLogEntry } from "./app-events";
import { DEFAULT_CRON_FORM, DEFAULT_LOG_LEVEL_FILTERS } from "./app-defaults"; import { DEFAULT_CRON_FORM, DEFAULT_LOG_LEVEL_FILTERS } from "./app-defaults";
import type { import type {
@@ -129,7 +129,7 @@ export class ClawdbotApp extends LitElement {
@state() chatAvatarUrl: string | null = null; @state() chatAvatarUrl: string | null = null;
@state() chatThinkingLevel: string | null = null; @state() chatThinkingLevel: string | null = null;
@state() chatQueue: ChatQueueItem[] = []; @state() chatQueue: ChatQueueItem[] = [];
@state() chatAttachments: Array<{ id: string; dataUrl: string; mimeType: string }> = []; @state() chatAttachments: ChatAttachment[] = [];
// Sidebar state for tool output viewing // Sidebar state for tool output viewing
@state() sidebarOpen = false; @state() sidebarOpen = false;
@state() sidebarContent: string | null = null; @state() sidebarContent: string | null = null;

View File

@@ -1,12 +1,7 @@
import { extractText } from "../chat/message-extract"; import { extractText } from "../chat/message-extract";
import type { GatewayBrowserClient } from "../gateway"; import type { GatewayBrowserClient } from "../gateway";
import { generateUUID } from "../uuid"; import { generateUUID } from "../uuid";
import type { ChatAttachment } from "../ui-types";
export type ChatAttachment = {
id: string;
dataUrl: string;
mimeType: string;
};
export type ChatState = { export type ChatState = {
client: GatewayBrowserClient | null; client: GatewayBrowserClient | null;

View File

@@ -1,7 +1,14 @@
export type ChatAttachment = {
id: string;
dataUrl: string;
mimeType: string;
};
export type ChatQueueItem = { export type ChatQueueItem = {
id: string; id: string;
text: string; text: string;
createdAt: number; createdAt: number;
attachments?: ChatAttachment[];
}; };
export const CRON_CHANNEL_LAST = "last"; export const CRON_CHANNEL_LAST = "last";

View File

@@ -1,7 +1,7 @@
import { html, nothing } from "lit"; import { html, nothing } from "lit";
import { repeat } from "lit/directives/repeat.js"; import { repeat } from "lit/directives/repeat.js";
import type { SessionsListResult } from "../types"; import type { SessionsListResult } from "../types";
import type { ChatQueueItem } from "../ui-types"; import type { ChatAttachment, ChatQueueItem } from "../ui-types";
import type { ChatItem, MessageGroup } from "../types/chat-types"; import type { ChatItem, MessageGroup } from "../types/chat-types";
import { icons } from "../icons"; import { icons } from "../icons";
import { import {
@@ -22,12 +22,6 @@ export type CompactionIndicatorStatus = {
completedAt: number | null; completedAt: number | null;
}; };
export type ChatAttachment = {
id: string;
dataUrl: string;
mimeType: string;
};
export type ChatProps = { export type ChatProps = {
sessionKey: string; sessionKey: string;
onSessionKeyChange: (next: string) => void; onSessionKeyChange: (next: string) => void;
@@ -305,7 +299,12 @@ export function renderChat(props: ChatProps) {
${props.queue.map( ${props.queue.map(
(item) => html` (item) => html`
<div class="chat-queue__item"> <div class="chat-queue__item">
<div class="chat-queue__text">${item.text}</div> <div class="chat-queue__text">
${item.text ||
(item.attachments?.length
? `Image (${item.attachments.length})`
: "")}
</div>
<button <button
class="btn chat-queue__remove" class="btn chat-queue__remove"
type="button" type="button"