fix(webchat): support image-only sends
This commit is contained in:
@@ -34,6 +34,9 @@ Status: unreleased.
|
|||||||
- Slack: clear ack reaction after streamed replies. (#2044) Thanks @fancyboi999.
|
- Slack: clear ack reaction after streamed replies. (#2044) Thanks @fancyboi999.
|
||||||
- macOS: keep custom SSH usernames in remote target. (#2046) Thanks @algal.
|
- macOS: keep custom SSH usernames in remote target. (#2046) Thanks @algal.
|
||||||
|
|
||||||
|
### Fixes
|
||||||
|
- Web UI: improve WebChat image paste previews and allow image-only sends. (#1925) Thanks @smartprogrammer93.
|
||||||
|
|
||||||
## 2026.1.24-3
|
## 2026.1.24-3
|
||||||
|
|
||||||
### Fixes
|
### Fixes
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ export const ChatHistoryParamsSchema = Type.Object(
|
|||||||
export const ChatSendParamsSchema = Type.Object(
|
export const ChatSendParamsSchema = Type.Object(
|
||||||
{
|
{
|
||||||
sessionKey: NonEmptyString,
|
sessionKey: NonEmptyString,
|
||||||
message: NonEmptyString,
|
message: Type.String(),
|
||||||
thinking: Type.Optional(Type.String()),
|
thinking: Type.Optional(Type.String()),
|
||||||
deliver: Type.Optional(Type.Boolean()),
|
deliver: Type.Optional(Type.Boolean()),
|
||||||
attachments: Type.Optional(Type.Array(Type.Unknown())),
|
attachments: Type.Optional(Type.Array(Type.Unknown())),
|
||||||
|
|||||||
@@ -338,6 +338,15 @@ export const chatHandlers: GatewayRequestHandlers = {
|
|||||||
: undefined,
|
: undefined,
|
||||||
}))
|
}))
|
||||||
.filter((a) => a.content) ?? [];
|
.filter((a) => a.content) ?? [];
|
||||||
|
const rawMessage = p.message.trim();
|
||||||
|
if (!rawMessage && normalizedAttachments.length === 0) {
|
||||||
|
respond(
|
||||||
|
false,
|
||||||
|
undefined,
|
||||||
|
errorShape(ErrorCodes.INVALID_REQUEST, "message or attachment required"),
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
let parsedMessage = p.message;
|
let parsedMessage = p.message;
|
||||||
let parsedImages: ChatImageContent[] = [];
|
let parsedImages: ChatImageContent[] = [];
|
||||||
if (normalizedAttachments.length > 0) {
|
if (normalizedAttachments.length > 0) {
|
||||||
|
|||||||
@@ -208,6 +208,39 @@ describe("gateway server chat", () => {
|
|||||||
| undefined;
|
| undefined;
|
||||||
expect(imgOpts?.images).toEqual([{ type: "image", data: pngB64, mimeType: "image/png" }]);
|
expect(imgOpts?.images).toEqual([{ type: "image", data: pngB64, mimeType: "image/png" }]);
|
||||||
|
|
||||||
|
const callsBeforeImageOnly = spy.mock.calls.length;
|
||||||
|
const reqIdOnly = "chat-img-only";
|
||||||
|
ws.send(
|
||||||
|
JSON.stringify({
|
||||||
|
type: "req",
|
||||||
|
id: reqIdOnly,
|
||||||
|
method: "chat.send",
|
||||||
|
params: {
|
||||||
|
sessionKey: "main",
|
||||||
|
message: "",
|
||||||
|
idempotencyKey: "idem-img-only",
|
||||||
|
attachments: [
|
||||||
|
{
|
||||||
|
type: "image",
|
||||||
|
mimeType: "image/png",
|
||||||
|
fileName: "dot.png",
|
||||||
|
content: `data:image/png;base64,${pngB64}`,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const imgOnlyRes = await onceMessage(ws, (o) => o.type === "res" && o.id === reqIdOnly, 8000);
|
||||||
|
expect(imgOnlyRes.ok).toBe(true);
|
||||||
|
expect(imgOnlyRes.payload?.runId).toBeDefined();
|
||||||
|
|
||||||
|
await waitFor(() => spy.mock.calls.length > callsBeforeImageOnly, 8000);
|
||||||
|
const imgOnlyOpts = spy.mock.calls.at(-1)?.[1] as
|
||||||
|
| { images?: Array<{ type: string; data: string; mimeType: string }> }
|
||||||
|
| undefined;
|
||||||
|
expect(imgOnlyOpts?.images).toEqual([{ type: "image", data: pngB64, mimeType: "image/png" }]);
|
||||||
|
|
||||||
const historyDir = await fs.mkdtemp(path.join(os.tmpdir(), "clawdbot-gw-"));
|
const historyDir = await fs.mkdtemp(path.join(os.tmpdir(), "clawdbot-gw-"));
|
||||||
tempDirs.push(historyDir);
|
tempDirs.push(historyDir);
|
||||||
testState.sessionStorePath = path.join(historyDir, "sessions.json");
|
testState.sessionStorePath = path.join(historyDir, "sessions.json");
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { abortChatRun, loadChatHistory, sendChatMessage, type ChatAttachment } from "./controllers/chat";
|
import { abortChatRun, loadChatHistory, sendChatMessage } from "./controllers/chat";
|
||||||
import { loadSessions } from "./controllers/sessions";
|
import { loadSessions } from "./controllers/sessions";
|
||||||
import { generateUUID } from "./uuid";
|
import { generateUUID } from "./uuid";
|
||||||
import { resetToolStream } from "./app-tool-stream";
|
import { resetToolStream } from "./app-tool-stream";
|
||||||
@@ -8,12 +8,13 @@ import { normalizeBasePath } from "./navigation";
|
|||||||
import type { GatewayHelloOk } from "./gateway";
|
import type { GatewayHelloOk } from "./gateway";
|
||||||
import { parseAgentSessionKey } from "../../../src/sessions/session-key-utils.js";
|
import { parseAgentSessionKey } from "../../../src/sessions/session-key-utils.js";
|
||||||
import type { ClawdbotApp } from "./app";
|
import type { ClawdbotApp } from "./app";
|
||||||
|
import type { ChatAttachment, ChatQueueItem } from "./ui-types";
|
||||||
|
|
||||||
type ChatHost = {
|
type ChatHost = {
|
||||||
connected: boolean;
|
connected: boolean;
|
||||||
chatMessage: string;
|
chatMessage: string;
|
||||||
chatAttachments: ChatAttachment[];
|
chatAttachments: ChatAttachment[];
|
||||||
chatQueue: Array<{ id: string; text: string; createdAt: number }>;
|
chatQueue: ChatQueueItem[];
|
||||||
chatRunId: string | null;
|
chatRunId: string | null;
|
||||||
chatSending: boolean;
|
chatSending: boolean;
|
||||||
sessionKey: string;
|
sessionKey: string;
|
||||||
@@ -46,15 +47,17 @@ export async function handleAbortChat(host: ChatHost) {
|
|||||||
await abortChatRun(host as unknown as ClawdbotApp);
|
await abortChatRun(host as unknown as ClawdbotApp);
|
||||||
}
|
}
|
||||||
|
|
||||||
function enqueueChatMessage(host: ChatHost, text: string) {
|
function enqueueChatMessage(host: ChatHost, text: string, attachments?: ChatAttachment[]) {
|
||||||
const trimmed = text.trim();
|
const trimmed = text.trim();
|
||||||
if (!trimmed) return;
|
const hasAttachments = Boolean(attachments && attachments.length > 0);
|
||||||
|
if (!trimmed && !hasAttachments) return;
|
||||||
host.chatQueue = [
|
host.chatQueue = [
|
||||||
...host.chatQueue,
|
...host.chatQueue,
|
||||||
{
|
{
|
||||||
id: generateUUID(),
|
id: generateUUID(),
|
||||||
text: trimmed,
|
text: trimmed,
|
||||||
createdAt: Date.now(),
|
createdAt: Date.now(),
|
||||||
|
attachments: hasAttachments ? attachments?.map((att) => ({ ...att })) : undefined,
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
@@ -62,19 +65,31 @@ function enqueueChatMessage(host: ChatHost, text: string) {
|
|||||||
async function sendChatMessageNow(
|
async function sendChatMessageNow(
|
||||||
host: ChatHost,
|
host: ChatHost,
|
||||||
message: string,
|
message: string,
|
||||||
opts?: { previousDraft?: string; restoreDraft?: boolean; attachments?: ChatAttachment[] },
|
opts?: {
|
||||||
|
previousDraft?: string;
|
||||||
|
restoreDraft?: boolean;
|
||||||
|
attachments?: ChatAttachment[];
|
||||||
|
previousAttachments?: ChatAttachment[];
|
||||||
|
restoreAttachments?: boolean;
|
||||||
|
},
|
||||||
) {
|
) {
|
||||||
resetToolStream(host as unknown as Parameters<typeof resetToolStream>[0]);
|
resetToolStream(host as unknown as Parameters<typeof resetToolStream>[0]);
|
||||||
const ok = await sendChatMessage(host as unknown as ClawdbotApp, message, opts?.attachments);
|
const ok = await sendChatMessage(host as unknown as ClawdbotApp, message, opts?.attachments);
|
||||||
if (!ok && opts?.previousDraft != null) {
|
if (!ok && opts?.previousDraft != null) {
|
||||||
host.chatMessage = opts.previousDraft;
|
host.chatMessage = opts.previousDraft;
|
||||||
}
|
}
|
||||||
|
if (!ok && opts?.previousAttachments) {
|
||||||
|
host.chatAttachments = opts.previousAttachments;
|
||||||
|
}
|
||||||
if (ok) {
|
if (ok) {
|
||||||
setLastActiveSessionKey(host as unknown as Parameters<typeof setLastActiveSessionKey>[0], host.sessionKey);
|
setLastActiveSessionKey(host as unknown as Parameters<typeof setLastActiveSessionKey>[0], host.sessionKey);
|
||||||
}
|
}
|
||||||
if (ok && opts?.restoreDraft && opts.previousDraft?.trim()) {
|
if (ok && opts?.restoreDraft && opts.previousDraft?.trim()) {
|
||||||
host.chatMessage = opts.previousDraft;
|
host.chatMessage = opts.previousDraft;
|
||||||
}
|
}
|
||||||
|
if (ok && opts?.restoreAttachments && opts.previousAttachments?.length) {
|
||||||
|
host.chatAttachments = opts.previousAttachments;
|
||||||
|
}
|
||||||
scheduleChatScroll(host as unknown as Parameters<typeof scheduleChatScroll>[0]);
|
scheduleChatScroll(host as unknown as Parameters<typeof scheduleChatScroll>[0]);
|
||||||
if (ok && !host.chatRunId) {
|
if (ok && !host.chatRunId) {
|
||||||
void flushChatQueue(host);
|
void flushChatQueue(host);
|
||||||
@@ -87,7 +102,7 @@ async function flushChatQueue(host: ChatHost) {
|
|||||||
const [next, ...rest] = host.chatQueue;
|
const [next, ...rest] = host.chatQueue;
|
||||||
if (!next) return;
|
if (!next) return;
|
||||||
host.chatQueue = rest;
|
host.chatQueue = rest;
|
||||||
const ok = await sendChatMessageNow(host, next.text);
|
const ok = await sendChatMessageNow(host, next.text, { attachments: next.attachments });
|
||||||
if (!ok) {
|
if (!ok) {
|
||||||
host.chatQueue = [next, ...host.chatQueue];
|
host.chatQueue = [next, ...host.chatQueue];
|
||||||
}
|
}
|
||||||
@@ -106,7 +121,8 @@ export async function handleSendChat(
|
|||||||
const previousDraft = host.chatMessage;
|
const previousDraft = host.chatMessage;
|
||||||
const message = (messageOverride ?? host.chatMessage).trim();
|
const message = (messageOverride ?? host.chatMessage).trim();
|
||||||
const attachments = host.chatAttachments ?? [];
|
const attachments = host.chatAttachments ?? [];
|
||||||
const hasAttachments = attachments.length > 0;
|
const attachmentsToSend = messageOverride == null ? attachments : [];
|
||||||
|
const hasAttachments = attachmentsToSend.length > 0;
|
||||||
|
|
||||||
// Allow sending with just attachments (no message text required)
|
// Allow sending with just attachments (no message text required)
|
||||||
if (!message && !hasAttachments) return;
|
if (!message && !hasAttachments) return;
|
||||||
@@ -123,14 +139,16 @@ export async function handleSendChat(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (isChatBusy(host)) {
|
if (isChatBusy(host)) {
|
||||||
enqueueChatMessage(host, message);
|
enqueueChatMessage(host, message, attachmentsToSend);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
await sendChatMessageNow(host, message, {
|
await sendChatMessageNow(host, message, {
|
||||||
previousDraft: messageOverride == null ? previousDraft : undefined,
|
previousDraft: messageOverride == null ? previousDraft : undefined,
|
||||||
restoreDraft: Boolean(messageOverride && opts?.restoreDraft),
|
restoreDraft: Boolean(messageOverride && opts?.restoreDraft),
|
||||||
attachments: hasAttachments ? attachments : undefined,
|
attachments: hasAttachments ? attachmentsToSend : undefined,
|
||||||
|
previousAttachments: messageOverride == null ? attachments : undefined,
|
||||||
|
restoreAttachments: Boolean(messageOverride && opts?.restoreDraft),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -431,6 +431,7 @@ export function renderApp(state: AppViewState) {
|
|||||||
onSessionKeyChange: (next) => {
|
onSessionKeyChange: (next) => {
|
||||||
state.sessionKey = next;
|
state.sessionKey = next;
|
||||||
state.chatMessage = "";
|
state.chatMessage = "";
|
||||||
|
state.chatAttachments = [];
|
||||||
state.chatStream = null;
|
state.chatStream = null;
|
||||||
state.chatStreamStartedAt = null;
|
state.chatStreamStartedAt = null;
|
||||||
state.chatRunId = null;
|
state.chatRunId = null;
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ import type {
|
|||||||
SkillStatusReport,
|
SkillStatusReport,
|
||||||
StatusSummary,
|
StatusSummary,
|
||||||
} from "./types";
|
} from "./types";
|
||||||
import type { ChatQueueItem, CronFormState } from "./ui-types";
|
import type { ChatAttachment, ChatQueueItem, CronFormState } from "./ui-types";
|
||||||
import type { EventLogEntry } from "./app-events";
|
import type { EventLogEntry } from "./app-events";
|
||||||
import type { SkillMessage } from "./controllers/skills";
|
import type { SkillMessage } from "./controllers/skills";
|
||||||
import type {
|
import type {
|
||||||
@@ -49,6 +49,7 @@ export type AppViewState = {
|
|||||||
chatLoading: boolean;
|
chatLoading: boolean;
|
||||||
chatSending: boolean;
|
chatSending: boolean;
|
||||||
chatMessage: string;
|
chatMessage: string;
|
||||||
|
chatAttachments: ChatAttachment[];
|
||||||
chatMessages: unknown[];
|
chatMessages: unknown[];
|
||||||
chatToolMessages: unknown[];
|
chatToolMessages: unknown[];
|
||||||
chatStream: string | null;
|
chatStream: string | null;
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ import type {
|
|||||||
StatusSummary,
|
StatusSummary,
|
||||||
NostrProfile,
|
NostrProfile,
|
||||||
} from "./types";
|
} from "./types";
|
||||||
import { type ChatQueueItem, type CronFormState } from "./ui-types";
|
import { type ChatAttachment, type ChatQueueItem, type CronFormState } from "./ui-types";
|
||||||
import type { EventLogEntry } from "./app-events";
|
import type { EventLogEntry } from "./app-events";
|
||||||
import { DEFAULT_CRON_FORM, DEFAULT_LOG_LEVEL_FILTERS } from "./app-defaults";
|
import { DEFAULT_CRON_FORM, DEFAULT_LOG_LEVEL_FILTERS } from "./app-defaults";
|
||||||
import type {
|
import type {
|
||||||
@@ -129,7 +129,7 @@ export class ClawdbotApp extends LitElement {
|
|||||||
@state() chatAvatarUrl: string | null = null;
|
@state() chatAvatarUrl: string | null = null;
|
||||||
@state() chatThinkingLevel: string | null = null;
|
@state() chatThinkingLevel: string | null = null;
|
||||||
@state() chatQueue: ChatQueueItem[] = [];
|
@state() chatQueue: ChatQueueItem[] = [];
|
||||||
@state() chatAttachments: Array<{ id: string; dataUrl: string; mimeType: string }> = [];
|
@state() chatAttachments: ChatAttachment[] = [];
|
||||||
// Sidebar state for tool output viewing
|
// Sidebar state for tool output viewing
|
||||||
@state() sidebarOpen = false;
|
@state() sidebarOpen = false;
|
||||||
@state() sidebarContent: string | null = null;
|
@state() sidebarContent: string | null = null;
|
||||||
|
|||||||
@@ -1,12 +1,7 @@
|
|||||||
import { extractText } from "../chat/message-extract";
|
import { extractText } from "../chat/message-extract";
|
||||||
import type { GatewayBrowserClient } from "../gateway";
|
import type { GatewayBrowserClient } from "../gateway";
|
||||||
import { generateUUID } from "../uuid";
|
import { generateUUID } from "../uuid";
|
||||||
|
import type { ChatAttachment } from "../ui-types";
|
||||||
export type ChatAttachment = {
|
|
||||||
id: string;
|
|
||||||
dataUrl: string;
|
|
||||||
mimeType: string;
|
|
||||||
};
|
|
||||||
|
|
||||||
export type ChatState = {
|
export type ChatState = {
|
||||||
client: GatewayBrowserClient | null;
|
client: GatewayBrowserClient | null;
|
||||||
|
|||||||
@@ -1,7 +1,14 @@
|
|||||||
|
export type ChatAttachment = {
|
||||||
|
id: string;
|
||||||
|
dataUrl: string;
|
||||||
|
mimeType: string;
|
||||||
|
};
|
||||||
|
|
||||||
export type ChatQueueItem = {
|
export type ChatQueueItem = {
|
||||||
id: string;
|
id: string;
|
||||||
text: string;
|
text: string;
|
||||||
createdAt: number;
|
createdAt: number;
|
||||||
|
attachments?: ChatAttachment[];
|
||||||
};
|
};
|
||||||
|
|
||||||
export const CRON_CHANNEL_LAST = "last";
|
export const CRON_CHANNEL_LAST = "last";
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import { html, nothing } from "lit";
|
import { html, nothing } from "lit";
|
||||||
import { repeat } from "lit/directives/repeat.js";
|
import { repeat } from "lit/directives/repeat.js";
|
||||||
import type { SessionsListResult } from "../types";
|
import type { SessionsListResult } from "../types";
|
||||||
import type { ChatQueueItem } from "../ui-types";
|
import type { ChatAttachment, ChatQueueItem } from "../ui-types";
|
||||||
import type { ChatItem, MessageGroup } from "../types/chat-types";
|
import type { ChatItem, MessageGroup } from "../types/chat-types";
|
||||||
import { icons } from "../icons";
|
import { icons } from "../icons";
|
||||||
import {
|
import {
|
||||||
@@ -22,12 +22,6 @@ export type CompactionIndicatorStatus = {
|
|||||||
completedAt: number | null;
|
completedAt: number | null;
|
||||||
};
|
};
|
||||||
|
|
||||||
export type ChatAttachment = {
|
|
||||||
id: string;
|
|
||||||
dataUrl: string;
|
|
||||||
mimeType: string;
|
|
||||||
};
|
|
||||||
|
|
||||||
export type ChatProps = {
|
export type ChatProps = {
|
||||||
sessionKey: string;
|
sessionKey: string;
|
||||||
onSessionKeyChange: (next: string) => void;
|
onSessionKeyChange: (next: string) => void;
|
||||||
@@ -305,7 +299,12 @@ export function renderChat(props: ChatProps) {
|
|||||||
${props.queue.map(
|
${props.queue.map(
|
||||||
(item) => html`
|
(item) => html`
|
||||||
<div class="chat-queue__item">
|
<div class="chat-queue__item">
|
||||||
<div class="chat-queue__text">${item.text}</div>
|
<div class="chat-queue__text">
|
||||||
|
${item.text ||
|
||||||
|
(item.attachments?.length
|
||||||
|
? `Image (${item.attachments.length})`
|
||||||
|
: "")}
|
||||||
|
</div>
|
||||||
<button
|
<button
|
||||||
class="btn chat-queue__remove"
|
class="btn chat-queue__remove"
|
||||||
type="button"
|
type="button"
|
||||||
|
|||||||
Reference in New Issue
Block a user