let5see/clawdbot
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

fix: gate web chat/talk on mobile nodes

Browse Source
This commit is contained in:
Peter Steinberger
2025-12-30 22:05:17 +01:00
parent a2a26b26fb
commit 7e40147aa3
9 changed files with 314 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -8,6 +8,7 @@ coverage
.worktrees/
.DS_Store
**/.DS_Store
ui/src/ui/__screenshots__/
# Bun build artifacts
*.bun-build

1
CHANGELOG.md
View File

@@ -27,6 +27,7 @@
- Chat UI: clear streaming/tool bubbles when external runs finish, preventing duplicate assistant bubbles.
- Chat UI: user bubbles use `ui.seamColor` (fallback to a calmer default blue).
- Control UI: sync sidebar navigation with the URL for deep-linking, and auto-scroll chat to the latest message.
- Control UI: disable Web Chat + Talk when no iOS/Android node is connected; refreshed Web Chat styling and keyboard send.
- Talk Mode: wait for chat history to surface the assistant reply before starting TTS (macOS/iOS/Android).
- iOS Talk Mode: fix chat completion wait to time out even if no events arrive (prevents “Thinking…” hangs).
- iOS Talk Mode: keep recognition running during playback to support interrupt-on-speech.

43
src/gateway/server.ts
View File

@@ -1648,6 +1648,19 @@ export async function startGatewayServer(
let bridge: Awaited<ReturnType<typeof startNodeBridgeServer>> | null = null;
const bridgeNodeSubscriptions = new Map<string, Set<string>>();
const bridgeSessionSubscribers = new Map<string, Set<string>>();
const isMobilePlatform = (platform: unknown): boolean => {
const p = typeof platform === "string" ? platform.trim().toLowerCase() : "";
if (!p) return false;
return (
p.startsWith("ios") || p.startsWith("ipados") || p.startsWith("android")
);
};
const hasConnectedMobileNode = (): boolean => {
const connected = bridge?.listConnected?.() ?? [];
return connected.some((n) => isMobilePlatform(n.platform));
};
try {
await new Promise<void>((resolve, reject) => {
const onError = (err: NodeJS.ErrnoException) => {
@@ -4094,6 +4107,21 @@ export async function startGatewayServer(
break;
}
case "chat.send": {
if (
client &&
isWebchatConnect(client.connect) &&
!hasConnectedMobileNode()
) {
respond(
false,
undefined,
errorShape(
ErrorCodes.UNAVAILABLE,
"web chat disabled: no connected iOS/Android nodes",
),
);
break;
}
const params = (req.params ?? {}) as Record<string, unknown>;
if (!validateChatSendParams(params)) {
respond(
@@ -4645,6 +4673,21 @@ export async function startGatewayServer(
break;
}
case "talk.mode": {
if (
client &&
isWebchatConnect(client.connect) &&
!hasConnectedMobileNode()
) {
respond(
false,
undefined,
errorShape(
ErrorCodes.UNAVAILABLE,
"talk disabled: no connected iOS/Android nodes",
),
);
break;
}
const params = (req.params ?? {}) as Record<string, unknown>;
if (!validateTalkModeParams(params)) {
respond(

198
ui/src/styles/components.css
View File

@@ -361,64 +361,166 @@
background: rgba(0, 0, 0, 0.2);
}
.messages {
display: grid;
gap: 10px;
max-height: 60vh;
overflow: auto;
padding: 8px;
min-width: 0;
border-radius: 12px;
background: rgba(0, 0, 0, 0.2);
}
.chat-messages {
margin-top: 8px;
padding: 6px;
}
.msg {
border: 1px solid var(--border);
background: rgba(0, 0, 0, 0.2);
border-radius: 14px;
padding: 10px 12px;
min-width: 0;
}
.msg .meta {
font-size: 12px;
color: var(--muted);
.chat-header {
display: flex;
justify-content: space-between;
align-items: flex-end;
gap: 12px;
flex-wrap: wrap;
}
.chat-header__left {
display: flex;
align-items: flex-end;
gap: 12px;
flex-wrap: wrap;
min-width: 0;
}
.chat-header__right {
display: flex;
align-items: center;
gap: 10px;
margin-bottom: 6px;
}
.msg.user {
border-color: rgba(255, 255, 255, 0.14);
.chat-session {
min-width: 240px;
}
.msg.assistant {
border-color: rgba(255, 122, 61, 0.25);
background: rgba(255, 122, 61, 0.08);
.chat-thread {
margin-top: 12px;
display: flex;
flex-direction: column;
gap: 12px;
max-height: 60vh;
overflow: auto;
padding: 14px 12px;
min-width: 0;
border-radius: 16px;
border: 1px solid rgba(255, 255, 255, 0.08);
background: linear-gradient(
180deg,
rgba(0, 0, 0, 0.18) 0%,
rgba(0, 0, 0, 0.26) 100%
);
}
.msgContent {
:root[data-theme="light"] .chat-thread {
border-color: rgba(16, 24, 40, 0.12);
background: linear-gradient(
180deg,
rgba(16, 24, 40, 0.03) 0%,
rgba(16, 24, 40, 0.06) 100%
);
}
.chat-line {
display: flex;
}
.chat-line.user {
justify-content: flex-end;
}
.chat-line.assistant,
.chat-line.other {
justify-content: flex-start;
}
.chat-msg {
display: grid;
gap: 6px;
max-width: min(720px, 82%);
}
.chat-line.user .chat-msg {
justify-items: end;
}
.chat-bubble {
border: 1px solid var(--border);
background: rgba(0, 0, 0, 0.24);
border-radius: 18px;
padding: 10px 12px;
min-width: 0;
box-shadow: 0 12px 26px rgba(0, 0, 0, 0.22);
}
:root[data-theme="light"] .chat-bubble {
background: rgba(255, 255, 255, 0.85);
box-shadow: 0 12px 26px rgba(16, 24, 40, 0.08);
}
.chat-line.user .chat-bubble {
border-color: rgba(255, 122, 61, 0.35);
background: linear-gradient(
135deg,
rgba(255, 122, 61, 0.24) 0%,
rgba(255, 122, 61, 0.12) 100%
);
}
.chat-line.assistant .chat-bubble {
border-color: rgba(54, 207, 201, 0.16);
background: linear-gradient(
135deg,
rgba(54, 207, 201, 0.08) 0%,
rgba(0, 0, 0, 0.22) 100%
);
}
:root[data-theme="light"] .chat-line.assistant .chat-bubble {
background: linear-gradient(
135deg,
rgba(27, 185, 177, 0.12) 0%,
rgba(255, 255, 255, 0.85) 100%
);
}
@keyframes chatStreamPulse {
0% {
box-shadow: 0 12px 26px rgba(0, 0, 0, 0.22), 0 0 0 0 rgba(54, 207, 201, 0);
}
60% {
box-shadow: 0 12px 26px rgba(0, 0, 0, 0.22), 0 0 0 6px rgba(54, 207, 201, 0.06);
}
100% {
box-shadow: 0 12px 26px rgba(0, 0, 0, 0.22), 0 0 0 0 rgba(54, 207, 201, 0);
}
}
.chat-bubble.streaming {
border-color: rgba(54, 207, 201, 0.32);
animation: chatStreamPulse 1.6s ease-in-out infinite;
}
@media (prefers-reduced-motion: reduce) {
.chat-bubble.streaming {
animation: none;
}
}
.chat-text {
white-space: pre-wrap;
overflow-wrap: anywhere;
word-break: break-word;
}
.compose {
display: grid;
gap: 10px;
.chat-stamp {
font-size: 11px;
color: var(--muted);
}
.chat-line.user .chat-stamp {
text-align: right;
}
.chat-compose {
margin-top: 8px;
margin-top: 12px;
display: grid;
grid-template-columns: minmax(0, 1fr) auto;
align-items: end;
gap: 8px;
gap: 10px;
}
.chat-compose__field {
@@ -426,8 +528,18 @@
}
.chat-compose__field textarea {
min-height: 120px;
padding: 8px 10px;
min-height: 72px;
padding: 10px 12px;
border-radius: 14px;
resize: vertical;
white-space: pre-wrap;
font-family: var(--font-body);
line-height: 1.45;
}
.chat-compose__field textarea:disabled {
opacity: 0.7;
cursor: not-allowed;
}
.chat-compose__actions {
@@ -436,6 +548,10 @@
}
@media (max-width: 900px) {
.chat-session {
min-width: 200px;
}
.chat-compose {
grid-template-columns: 1fr;
}

12
ui/src/ui/app-render.ts
View File

@@ -147,6 +147,16 @@ export function renderApp(state: AppViewState) {
const presenceCount = state.presenceEntries.length;
const sessionsCount = state.sessionsResult?.count ?? null;
const cronNext = state.cronStatus?.nextWakeAtMs ?? null;
const hasConnectedMobileNode = state.nodes.some((n) => {
if (!Boolean(n.connected)) return false;
const p = typeof n.platform === "string" ? n.platform.trim().toLowerCase() : "";
return p.startsWith("ios") || p.startsWith("ipados") || p.startsWith("android");
});
const chatDisabledReason = !state.connected
? "Disconnected from gateway."
: hasConnectedMobileNode
? null
: "No connected iOS/Android node — Web Chat + Talk are disabled.";
return html`
<div class="shell">
@@ -322,6 +332,8 @@ export function renderApp(state: AppViewState) {
stream: state.chatStream,
draft: state.chatMessage,
connected: state.connected,
canSend: state.connected && hasConnectedMobileNode,
disabledReason: chatDisabledReason,
onRefresh: () => loadChatHistory(state),
onDraftChange: (next) => (state.chatMessage = next),
onSend: () => state.handleSendChat(),

32
ui/src/ui/app.ts
View File

@@ -168,6 +168,7 @@ export class ClawdisApp extends LitElement {
client: GatewayBrowserClient | null = null;
private chatScrollFrame: number | null = null;
private nodesPollInterval: number | null = null;
basePath = "";
private popStateHandler = () => this.onPopState();
private themeMedia: MediaQueryList | null = null;
@@ -185,10 +186,12 @@ export class ClawdisApp extends LitElement {
this.attachThemeListener();
window.addEventListener("popstate", this.popStateHandler);
this.connect();
this.startNodesPolling();
}
disconnectedCallback() {
window.removeEventListener("popstate", this.popStateHandler);
this.stopNodesPolling();
this.detachThemeListener();
super.disconnectedCallback();
}
@@ -221,6 +224,7 @@ export class ClawdisApp extends LitElement {
this.connected = true;
this.hello = hello;
this.applySnapshot(hello);
void loadNodes(this, { quiet: true });
void this.refreshActiveTab();
},
onClose: ({ code, reason }) => {
@@ -239,12 +243,37 @@ export class ClawdisApp extends LitElement {
if (this.chatScrollFrame) cancelAnimationFrame(this.chatScrollFrame);
this.chatScrollFrame = requestAnimationFrame(() => {
this.chatScrollFrame = null;
const container = this.querySelector(".messages") as HTMLElement | null;
const container = this.querySelector(".chat-thread") as HTMLElement | null;
if (!container) return;
container.scrollTop = container.scrollHeight;
});
}
private startNodesPolling() {
if (this.nodesPollInterval != null) return;
this.nodesPollInterval = window.setInterval(
() => void loadNodes(this, { quiet: true }),
5000,
);
}
private stopNodesPolling() {
if (this.nodesPollInterval == null) return;
clearInterval(this.nodesPollInterval);
this.nodesPollInterval = null;
}
private hasConnectedMobileNode() {
return this.nodes.some((n) => {
if (!Boolean(n.connected)) return false;
const p =
typeof n.platform === "string" ? n.platform.trim().toLowerCase() : "";
return (
p.startsWith("ios") || p.startsWith("ipados") || p.startsWith("android")
);
});
}
private onEvent(evt: GatewayEventFrame) {
this.eventLog = [
{ ts: Date.now(), event: evt.event, payload: evt.payload },
@@ -427,6 +456,7 @@ export class ClawdisApp extends LitElement {
}
async handleSendChat() {
if (!this.connected || !this.hasConnectedMobileNode()) return;
await sendChat(this);
void loadChatHistory(this);
}

11
ui/src/ui/controllers/nodes.ts
View File

@@ -8,19 +8,22 @@ export type NodesState = {
lastError: string | null;
};
export async function loadNodes(state: NodesState) {
export async function loadNodes(
state: NodesState,
opts?: { quiet?: boolean },
) {
if (!state.client || !state.connected) return;
if (state.nodesLoading) return;
state.nodesLoading = true;
state.lastError = null;
if (!opts?.quiet) state.lastError = null;
try {
const res = (await state.client.request("node.list", {})) as {
nodes?: Array<Record<string, unknown>>;
};
state.nodes = Array.isArray(res.nodes) ? res.nodes : [];
} catch (err) {
state.lastError = String(err);
if (!opts?.quiet) state.lastError = String(err);
} finally {
state.nodesLoading = false;
}
}

4
ui/src/ui/navigation.browser.test.ts
View File

@@ -68,7 +68,7 @@ describe("control UI routing", () => {
const app = mountApp("/chat");
await app.updateComplete;
const initialContainer = app.querySelector(".messages") as HTMLElement | null;
const initialContainer = app.querySelector(".chat-thread") as HTMLElement | null;
expect(initialContainer).not.toBeNull();
if (!initialContainer) return;
initialContainer.style.maxHeight = "180px";
@@ -83,7 +83,7 @@ describe("control UI routing", () => {
await app.updateComplete;
await nextFrame();
const container = app.querySelector(".messages") as HTMLElement | null;
const container = app.querySelector(".chat-thread") as HTMLElement | null;
expect(container).not.toBeNull();
if (!container) return;
const maxScroll = container.scrollHeight - container.clientHeight;

86
ui/src/ui/views/chat.ts
View File

@@ -10,58 +10,90 @@ export type ChatProps = {
stream: string | null;
draft: string;
connected: boolean;
canSend: boolean;
disabledReason: string | null;
onRefresh: () => void;
onDraftChange: (next: string) => void;
onSend: () => void;
};
export function renderChat(props: ChatProps) {
const canInteract = props.connected;
const canCompose = props.canSend && !props.sending;
const composePlaceholder = (() => {
if (!props.connected) return "Connect to the gateway to start chatting…";
if (!props.canSend) return "Connect an iOS/Android node to enable Web Chat + Talk…";
return "Message (⌘↩ to send)";
})();
return html`
<section class="card">
<div class="row" style="justify-content: space-between;">
<div class="row">
<label class="field" style="min-width: 220px;">
<section class="card chat">
<div class="chat-header">
<div class="chat-header__left">
<label class="field chat-session">
<span>Session Key</span>
<input
.value=${props.sessionKey}
?disabled=${!canInteract}
@input=${(e: Event) =>
props.onSessionKeyChange((e.target as HTMLInputElement).value)}
/>
</label>
<button class="btn" ?disabled=${props.loading} @click=${props.onRefresh}>
<button
class="btn"
?disabled=${props.loading || !canInteract}
@click=${props.onRefresh}
>
${props.loading ? "Loading…" : "Refresh"}
</button>
</div>
<div class="muted">
Thinking: ${props.thinkingLevel ?? "inherit"}
<div class="chat-header__right">
<div class="muted">Thinking: ${props.thinkingLevel ?? "inherit"}</div>
</div>
</div>
<div class="messages chat-messages">
${props.disabledReason
? html`<div class="callout" style="margin-top: 12px;">
${props.disabledReason}
</div>`
: nothing}
<div class="chat-thread" role="log" aria-live="polite">
${props.loading ? html`<div class="muted">Loading chat…</div>` : nothing}
${props.messages.map((m) => renderMessage(m))}
${props.stream
? html`${renderMessage({
role: "assistant",
content: [{ type: "text", text: props.stream }],
})}`
? renderMessage(
{
role: "assistant",
content: [{ type: "text", text: props.stream }],
timestamp: Date.now(),
},
{ streaming: true },
)
: nothing}
</div>
<div class="compose chat-compose">
<div class="chat-compose">
<label class="field chat-compose__field">
<span>Message</span>
<textarea
.value=${props.draft}
?disabled=${!props.canSend}
@keydown=${(e: KeyboardEvent) => {
if (e.key !== "Enter") return;
if (!e.metaKey && !e.ctrlKey) return;
e.preventDefault();
if (canCompose) props.onSend();
}}
@input=${(e: Event) =>
props.onDraftChange((e.target as HTMLTextAreaElement).value)}
placeholder="Ask the model…"
placeholder=${composePlaceholder}
></textarea>
</label>
<div class="row chat-compose__actions">
<button
class="btn primary"
?disabled=${props.sending || !props.connected}
?disabled=${!props.canSend || props.sending}
@click=${props.onSend}
>
${props.sending ? "Sending…" : "Send"}
@@ -72,7 +104,7 @@ export function renderChat(props: ChatProps) {
`;
}
function renderMessage(message: unknown) {
function renderMessage(message: unknown, opts?: { streaming?: boolean }) {
const m = message as Record<string, unknown>;
const role = typeof m.role === "string" ? m.role : "unknown";
const text =
@@ -81,18 +113,20 @@ function renderMessage(message: unknown) {
? m.content
: JSON.stringify(message, null, 2));
const ts =
typeof m.timestamp === "number"
? new Date(m.timestamp).toLocaleTimeString()
: "";
const klass = role === "assistant" ? "assistant" : role === "user" ? "user" : "";
const timestamp =
typeof m.timestamp === "number" ? new Date(m.timestamp).toLocaleTimeString() : "";
const klass = role === "assistant" ? "assistant" : role === "user" ? "user" : "other";
const who = role === "assistant" ? "Assistant" : role === "user" ? "You" : role;
return html`
<div class="msg ${klass}">
<div class="meta">
<span class="mono">${role}</span>
<span class="mono">${ts}</span>
<div class="chat-line ${klass}">
<div class="chat-msg">
<div class="chat-bubble ${opts?.streaming ? "streaming" : ""}">
<div class="chat-text">${text}</div>
</div>
<div class="chat-stamp mono">
${who}${timestamp ? html` · ${timestamp}` : nothing}
</div>
</div>
<div class="msgContent">${text}</div>
</div>
`;
}