fix: gate web chat/talk on mobile nodes

This commit is contained in:
Peter Steinberger
2025-12-30 22:05:17 +01:00
parent a2a26b26fb
commit 7e40147aa3
9 changed files with 314 additions and 74 deletions

1
.gitignore vendored
View File

@@ -8,6 +8,7 @@ coverage
.worktrees/ .worktrees/
.DS_Store .DS_Store
**/.DS_Store **/.DS_Store
ui/src/ui/__screenshots__/
# Bun build artifacts # Bun build artifacts
*.bun-build *.bun-build

View File

@@ -27,6 +27,7 @@
- Chat UI: clear streaming/tool bubbles when external runs finish, preventing duplicate assistant bubbles. - Chat UI: clear streaming/tool bubbles when external runs finish, preventing duplicate assistant bubbles.
- Chat UI: user bubbles use `ui.seamColor` (fallback to a calmer default blue). - Chat UI: user bubbles use `ui.seamColor` (fallback to a calmer default blue).
- Control UI: sync sidebar navigation with the URL for deep-linking, and auto-scroll chat to the latest message. - Control UI: sync sidebar navigation with the URL for deep-linking, and auto-scroll chat to the latest message.
- Control UI: disable Web Chat + Talk when no iOS/Android node is connected; refreshed Web Chat styling and keyboard send.
- Talk Mode: wait for chat history to surface the assistant reply before starting TTS (macOS/iOS/Android). - Talk Mode: wait for chat history to surface the assistant reply before starting TTS (macOS/iOS/Android).
- iOS Talk Mode: fix chat completion wait to time out even if no events arrive (prevents “Thinking…” hangs). - iOS Talk Mode: fix chat completion wait to time out even if no events arrive (prevents “Thinking…” hangs).
- iOS Talk Mode: keep recognition running during playback to support interrupt-on-speech. - iOS Talk Mode: keep recognition running during playback to support interrupt-on-speech.

View File

@@ -1648,6 +1648,19 @@ export async function startGatewayServer(
let bridge: Awaited<ReturnType<typeof startNodeBridgeServer>> | null = null; let bridge: Awaited<ReturnType<typeof startNodeBridgeServer>> | null = null;
const bridgeNodeSubscriptions = new Map<string, Set<string>>(); const bridgeNodeSubscriptions = new Map<string, Set<string>>();
const bridgeSessionSubscribers = new Map<string, Set<string>>(); const bridgeSessionSubscribers = new Map<string, Set<string>>();
const isMobilePlatform = (platform: unknown): boolean => {
const p = typeof platform === "string" ? platform.trim().toLowerCase() : "";
if (!p) return false;
return (
p.startsWith("ios") || p.startsWith("ipados") || p.startsWith("android")
);
};
const hasConnectedMobileNode = (): boolean => {
const connected = bridge?.listConnected?.() ?? [];
return connected.some((n) => isMobilePlatform(n.platform));
};
try { try {
await new Promise<void>((resolve, reject) => { await new Promise<void>((resolve, reject) => {
const onError = (err: NodeJS.ErrnoException) => { const onError = (err: NodeJS.ErrnoException) => {
@@ -4094,6 +4107,21 @@ export async function startGatewayServer(
break; break;
} }
case "chat.send": { case "chat.send": {
if (
client &&
isWebchatConnect(client.connect) &&
!hasConnectedMobileNode()
) {
respond(
false,
undefined,
errorShape(
ErrorCodes.UNAVAILABLE,
"web chat disabled: no connected iOS/Android nodes",
),
);
break;
}
const params = (req.params ?? {}) as Record<string, unknown>; const params = (req.params ?? {}) as Record<string, unknown>;
if (!validateChatSendParams(params)) { if (!validateChatSendParams(params)) {
respond( respond(
@@ -4645,6 +4673,21 @@ export async function startGatewayServer(
break; break;
} }
case "talk.mode": { case "talk.mode": {
if (
client &&
isWebchatConnect(client.connect) &&
!hasConnectedMobileNode()
) {
respond(
false,
undefined,
errorShape(
ErrorCodes.UNAVAILABLE,
"talk disabled: no connected iOS/Android nodes",
),
);
break;
}
const params = (req.params ?? {}) as Record<string, unknown>; const params = (req.params ?? {}) as Record<string, unknown>;
if (!validateTalkModeParams(params)) { if (!validateTalkModeParams(params)) {
respond( respond(

View File

@@ -361,64 +361,166 @@
background: rgba(0, 0, 0, 0.2); background: rgba(0, 0, 0, 0.2);
} }
.messages { .chat-header {
display: grid;
gap: 10px;
max-height: 60vh;
overflow: auto;
padding: 8px;
min-width: 0;
border-radius: 12px;
background: rgba(0, 0, 0, 0.2);
}
.chat-messages {
margin-top: 8px;
padding: 6px;
}
.msg {
border: 1px solid var(--border);
background: rgba(0, 0, 0, 0.2);
border-radius: 14px;
padding: 10px 12px;
min-width: 0;
}
.msg .meta {
font-size: 12px;
color: var(--muted);
display: flex; display: flex;
justify-content: space-between; justify-content: space-between;
align-items: flex-end;
gap: 12px;
flex-wrap: wrap;
}
.chat-header__left {
display: flex;
align-items: flex-end;
gap: 12px;
flex-wrap: wrap;
min-width: 0;
}
.chat-header__right {
display: flex;
align-items: center;
gap: 10px; gap: 10px;
margin-bottom: 6px;
} }
.msg.user { .chat-session {
border-color: rgba(255, 255, 255, 0.14); min-width: 240px;
} }
.msg.assistant { .chat-thread {
border-color: rgba(255, 122, 61, 0.25); margin-top: 12px;
background: rgba(255, 122, 61, 0.08); display: flex;
flex-direction: column;
gap: 12px;
max-height: 60vh;
overflow: auto;
padding: 14px 12px;
min-width: 0;
border-radius: 16px;
border: 1px solid rgba(255, 255, 255, 0.08);
background: linear-gradient(
180deg,
rgba(0, 0, 0, 0.18) 0%,
rgba(0, 0, 0, 0.26) 100%
);
} }
.msgContent { :root[data-theme="light"] .chat-thread {
border-color: rgba(16, 24, 40, 0.12);
background: linear-gradient(
180deg,
rgba(16, 24, 40, 0.03) 0%,
rgba(16, 24, 40, 0.06) 100%
);
}
.chat-line {
display: flex;
}
.chat-line.user {
justify-content: flex-end;
}
.chat-line.assistant,
.chat-line.other {
justify-content: flex-start;
}
.chat-msg {
display: grid;
gap: 6px;
max-width: min(720px, 82%);
}
.chat-line.user .chat-msg {
justify-items: end;
}
.chat-bubble {
border: 1px solid var(--border);
background: rgba(0, 0, 0, 0.24);
border-radius: 18px;
padding: 10px 12px;
min-width: 0;
box-shadow: 0 12px 26px rgba(0, 0, 0, 0.22);
}
:root[data-theme="light"] .chat-bubble {
background: rgba(255, 255, 255, 0.85);
box-shadow: 0 12px 26px rgba(16, 24, 40, 0.08);
}
.chat-line.user .chat-bubble {
border-color: rgba(255, 122, 61, 0.35);
background: linear-gradient(
135deg,
rgba(255, 122, 61, 0.24) 0%,
rgba(255, 122, 61, 0.12) 100%
);
}
.chat-line.assistant .chat-bubble {
border-color: rgba(54, 207, 201, 0.16);
background: linear-gradient(
135deg,
rgba(54, 207, 201, 0.08) 0%,
rgba(0, 0, 0, 0.22) 100%
);
}
:root[data-theme="light"] .chat-line.assistant .chat-bubble {
background: linear-gradient(
135deg,
rgba(27, 185, 177, 0.12) 0%,
rgba(255, 255, 255, 0.85) 100%
);
}
@keyframes chatStreamPulse {
0% {
box-shadow: 0 12px 26px rgba(0, 0, 0, 0.22), 0 0 0 0 rgba(54, 207, 201, 0);
}
60% {
box-shadow: 0 12px 26px rgba(0, 0, 0, 0.22), 0 0 0 6px rgba(54, 207, 201, 0.06);
}
100% {
box-shadow: 0 12px 26px rgba(0, 0, 0, 0.22), 0 0 0 0 rgba(54, 207, 201, 0);
}
}
.chat-bubble.streaming {
border-color: rgba(54, 207, 201, 0.32);
animation: chatStreamPulse 1.6s ease-in-out infinite;
}
@media (prefers-reduced-motion: reduce) {
.chat-bubble.streaming {
animation: none;
}
}
.chat-text {
white-space: pre-wrap; white-space: pre-wrap;
overflow-wrap: anywhere; overflow-wrap: anywhere;
word-break: break-word; word-break: break-word;
} }
.compose { .chat-stamp {
display: grid; font-size: 11px;
gap: 10px; color: var(--muted);
}
.chat-line.user .chat-stamp {
text-align: right;
} }
.chat-compose { .chat-compose {
margin-top: 8px; margin-top: 12px;
display: grid;
grid-template-columns: minmax(0, 1fr) auto; grid-template-columns: minmax(0, 1fr) auto;
align-items: end; align-items: end;
gap: 8px; gap: 10px;
} }
.chat-compose__field { .chat-compose__field {
@@ -426,8 +528,18 @@
} }
.chat-compose__field textarea { .chat-compose__field textarea {
min-height: 120px; min-height: 72px;
padding: 8px 10px; padding: 10px 12px;
border-radius: 14px;
resize: vertical;
white-space: pre-wrap;
font-family: var(--font-body);
line-height: 1.45;
}
.chat-compose__field textarea:disabled {
opacity: 0.7;
cursor: not-allowed;
} }
.chat-compose__actions { .chat-compose__actions {
@@ -436,6 +548,10 @@
} }
@media (max-width: 900px) { @media (max-width: 900px) {
.chat-session {
min-width: 200px;
}
.chat-compose { .chat-compose {
grid-template-columns: 1fr; grid-template-columns: 1fr;
} }

View File

@@ -147,6 +147,16 @@ export function renderApp(state: AppViewState) {
const presenceCount = state.presenceEntries.length; const presenceCount = state.presenceEntries.length;
const sessionsCount = state.sessionsResult?.count ?? null; const sessionsCount = state.sessionsResult?.count ?? null;
const cronNext = state.cronStatus?.nextWakeAtMs ?? null; const cronNext = state.cronStatus?.nextWakeAtMs ?? null;
const hasConnectedMobileNode = state.nodes.some((n) => {
if (!Boolean(n.connected)) return false;
const p = typeof n.platform === "string" ? n.platform.trim().toLowerCase() : "";
return p.startsWith("ios") || p.startsWith("ipados") || p.startsWith("android");
});
const chatDisabledReason = !state.connected
? "Disconnected from gateway."
: hasConnectedMobileNode
? null
: "No connected iOS/Android node — Web Chat + Talk are disabled.";
return html` return html`
<div class="shell"> <div class="shell">
@@ -322,6 +332,8 @@ export function renderApp(state: AppViewState) {
stream: state.chatStream, stream: state.chatStream,
draft: state.chatMessage, draft: state.chatMessage,
connected: state.connected, connected: state.connected,
canSend: state.connected && hasConnectedMobileNode,
disabledReason: chatDisabledReason,
onRefresh: () => loadChatHistory(state), onRefresh: () => loadChatHistory(state),
onDraftChange: (next) => (state.chatMessage = next), onDraftChange: (next) => (state.chatMessage = next),
onSend: () => state.handleSendChat(), onSend: () => state.handleSendChat(),

View File

@@ -168,6 +168,7 @@ export class ClawdisApp extends LitElement {
client: GatewayBrowserClient | null = null; client: GatewayBrowserClient | null = null;
private chatScrollFrame: number | null = null; private chatScrollFrame: number | null = null;
private nodesPollInterval: number | null = null;
basePath = ""; basePath = "";
private popStateHandler = () => this.onPopState(); private popStateHandler = () => this.onPopState();
private themeMedia: MediaQueryList | null = null; private themeMedia: MediaQueryList | null = null;
@@ -185,10 +186,12 @@ export class ClawdisApp extends LitElement {
this.attachThemeListener(); this.attachThemeListener();
window.addEventListener("popstate", this.popStateHandler); window.addEventListener("popstate", this.popStateHandler);
this.connect(); this.connect();
this.startNodesPolling();
} }
disconnectedCallback() { disconnectedCallback() {
window.removeEventListener("popstate", this.popStateHandler); window.removeEventListener("popstate", this.popStateHandler);
this.stopNodesPolling();
this.detachThemeListener(); this.detachThemeListener();
super.disconnectedCallback(); super.disconnectedCallback();
} }
@@ -221,6 +224,7 @@ export class ClawdisApp extends LitElement {
this.connected = true; this.connected = true;
this.hello = hello; this.hello = hello;
this.applySnapshot(hello); this.applySnapshot(hello);
void loadNodes(this, { quiet: true });
void this.refreshActiveTab(); void this.refreshActiveTab();
}, },
onClose: ({ code, reason }) => { onClose: ({ code, reason }) => {
@@ -239,12 +243,37 @@ export class ClawdisApp extends LitElement {
if (this.chatScrollFrame) cancelAnimationFrame(this.chatScrollFrame); if (this.chatScrollFrame) cancelAnimationFrame(this.chatScrollFrame);
this.chatScrollFrame = requestAnimationFrame(() => { this.chatScrollFrame = requestAnimationFrame(() => {
this.chatScrollFrame = null; this.chatScrollFrame = null;
const container = this.querySelector(".messages") as HTMLElement | null; const container = this.querySelector(".chat-thread") as HTMLElement | null;
if (!container) return; if (!container) return;
container.scrollTop = container.scrollHeight; container.scrollTop = container.scrollHeight;
}); });
} }
private startNodesPolling() {
if (this.nodesPollInterval != null) return;
this.nodesPollInterval = window.setInterval(
() => void loadNodes(this, { quiet: true }),
5000,
);
}
private stopNodesPolling() {
if (this.nodesPollInterval == null) return;
clearInterval(this.nodesPollInterval);
this.nodesPollInterval = null;
}
private hasConnectedMobileNode() {
return this.nodes.some((n) => {
if (!Boolean(n.connected)) return false;
const p =
typeof n.platform === "string" ? n.platform.trim().toLowerCase() : "";
return (
p.startsWith("ios") || p.startsWith("ipados") || p.startsWith("android")
);
});
}
private onEvent(evt: GatewayEventFrame) { private onEvent(evt: GatewayEventFrame) {
this.eventLog = [ this.eventLog = [
{ ts: Date.now(), event: evt.event, payload: evt.payload }, { ts: Date.now(), event: evt.event, payload: evt.payload },
@@ -427,6 +456,7 @@ export class ClawdisApp extends LitElement {
} }
async handleSendChat() { async handleSendChat() {
if (!this.connected || !this.hasConnectedMobileNode()) return;
await sendChat(this); await sendChat(this);
void loadChatHistory(this); void loadChatHistory(this);
} }

View File

@@ -8,19 +8,22 @@ export type NodesState = {
lastError: string | null; lastError: string | null;
}; };
export async function loadNodes(state: NodesState) { export async function loadNodes(
state: NodesState,
opts?: { quiet?: boolean },
) {
if (!state.client || !state.connected) return; if (!state.client || !state.connected) return;
if (state.nodesLoading) return;
state.nodesLoading = true; state.nodesLoading = true;
state.lastError = null; if (!opts?.quiet) state.lastError = null;
try { try {
const res = (await state.client.request("node.list", {})) as { const res = (await state.client.request("node.list", {})) as {
nodes?: Array<Record<string, unknown>>; nodes?: Array<Record<string, unknown>>;
}; };
state.nodes = Array.isArray(res.nodes) ? res.nodes : []; state.nodes = Array.isArray(res.nodes) ? res.nodes : [];
} catch (err) { } catch (err) {
state.lastError = String(err); if (!opts?.quiet) state.lastError = String(err);
} finally { } finally {
state.nodesLoading = false; state.nodesLoading = false;
} }
} }

View File

@@ -68,7 +68,7 @@ describe("control UI routing", () => {
const app = mountApp("/chat"); const app = mountApp("/chat");
await app.updateComplete; await app.updateComplete;
const initialContainer = app.querySelector(".messages") as HTMLElement | null; const initialContainer = app.querySelector(".chat-thread") as HTMLElement | null;
expect(initialContainer).not.toBeNull(); expect(initialContainer).not.toBeNull();
if (!initialContainer) return; if (!initialContainer) return;
initialContainer.style.maxHeight = "180px"; initialContainer.style.maxHeight = "180px";
@@ -83,7 +83,7 @@ describe("control UI routing", () => {
await app.updateComplete; await app.updateComplete;
await nextFrame(); await nextFrame();
const container = app.querySelector(".messages") as HTMLElement | null; const container = app.querySelector(".chat-thread") as HTMLElement | null;
expect(container).not.toBeNull(); expect(container).not.toBeNull();
if (!container) return; if (!container) return;
const maxScroll = container.scrollHeight - container.clientHeight; const maxScroll = container.scrollHeight - container.clientHeight;

View File

@@ -10,58 +10,90 @@ export type ChatProps = {
stream: string | null; stream: string | null;
draft: string; draft: string;
connected: boolean; connected: boolean;
canSend: boolean;
disabledReason: string | null;
onRefresh: () => void; onRefresh: () => void;
onDraftChange: (next: string) => void; onDraftChange: (next: string) => void;
onSend: () => void; onSend: () => void;
}; };
export function renderChat(props: ChatProps) { export function renderChat(props: ChatProps) {
const canInteract = props.connected;
const canCompose = props.canSend && !props.sending;
const composePlaceholder = (() => {
if (!props.connected) return "Connect to the gateway to start chatting…";
if (!props.canSend) return "Connect an iOS/Android node to enable Web Chat + Talk…";
return "Message (⌘↩ to send)";
})();
return html` return html`
<section class="card"> <section class="card chat">
<div class="row" style="justify-content: space-between;"> <div class="chat-header">
<div class="row"> <div class="chat-header__left">
<label class="field" style="min-width: 220px;"> <label class="field chat-session">
<span>Session Key</span> <span>Session Key</span>
<input <input
.value=${props.sessionKey} .value=${props.sessionKey}
?disabled=${!canInteract}
@input=${(e: Event) => @input=${(e: Event) =>
props.onSessionKeyChange((e.target as HTMLInputElement).value)} props.onSessionKeyChange((e.target as HTMLInputElement).value)}
/> />
</label> </label>
<button class="btn" ?disabled=${props.loading} @click=${props.onRefresh}> <button
class="btn"
?disabled=${props.loading || !canInteract}
@click=${props.onRefresh}
>
${props.loading ? "Loading…" : "Refresh"} ${props.loading ? "Loading…" : "Refresh"}
</button> </button>
</div> </div>
<div class="muted"> <div class="chat-header__right">
Thinking: ${props.thinkingLevel ?? "inherit"} <div class="muted">Thinking: ${props.thinkingLevel ?? "inherit"}</div>
</div> </div>
</div> </div>
<div class="messages chat-messages"> ${props.disabledReason
? html`<div class="callout" style="margin-top: 12px;">
${props.disabledReason}
</div>`
: nothing}
<div class="chat-thread" role="log" aria-live="polite">
${props.loading ? html`<div class="muted">Loading chat…</div>` : nothing} ${props.loading ? html`<div class="muted">Loading chat…</div>` : nothing}
${props.messages.map((m) => renderMessage(m))} ${props.messages.map((m) => renderMessage(m))}
${props.stream ${props.stream
? html`${renderMessage({ ? renderMessage(
role: "assistant", {
content: [{ type: "text", text: props.stream }], role: "assistant",
})}` content: [{ type: "text", text: props.stream }],
timestamp: Date.now(),
},
{ streaming: true },
)
: nothing} : nothing}
</div> </div>
<div class="compose chat-compose"> <div class="chat-compose">
<label class="field chat-compose__field"> <label class="field chat-compose__field">
<span>Message</span> <span>Message</span>
<textarea <textarea
.value=${props.draft} .value=${props.draft}
?disabled=${!props.canSend}
@keydown=${(e: KeyboardEvent) => {
if (e.key !== "Enter") return;
if (!e.metaKey && !e.ctrlKey) return;
e.preventDefault();
if (canCompose) props.onSend();
}}
@input=${(e: Event) => @input=${(e: Event) =>
props.onDraftChange((e.target as HTMLTextAreaElement).value)} props.onDraftChange((e.target as HTMLTextAreaElement).value)}
placeholder="Ask the model…" placeholder=${composePlaceholder}
></textarea> ></textarea>
</label> </label>
<div class="row chat-compose__actions"> <div class="row chat-compose__actions">
<button <button
class="btn primary" class="btn primary"
?disabled=${props.sending || !props.connected} ?disabled=${!props.canSend || props.sending}
@click=${props.onSend} @click=${props.onSend}
> >
${props.sending ? "Sending…" : "Send"} ${props.sending ? "Sending…" : "Send"}
@@ -72,7 +104,7 @@ export function renderChat(props: ChatProps) {
`; `;
} }
function renderMessage(message: unknown) { function renderMessage(message: unknown, opts?: { streaming?: boolean }) {
const m = message as Record<string, unknown>; const m = message as Record<string, unknown>;
const role = typeof m.role === "string" ? m.role : "unknown"; const role = typeof m.role === "string" ? m.role : "unknown";
const text = const text =
@@ -81,18 +113,20 @@ function renderMessage(message: unknown) {
? m.content ? m.content
: JSON.stringify(message, null, 2)); : JSON.stringify(message, null, 2));
const ts = const timestamp =
typeof m.timestamp === "number" typeof m.timestamp === "number" ? new Date(m.timestamp).toLocaleTimeString() : "";
? new Date(m.timestamp).toLocaleTimeString() const klass = role === "assistant" ? "assistant" : role === "user" ? "user" : "other";
: ""; const who = role === "assistant" ? "Assistant" : role === "user" ? "You" : role;
const klass = role === "assistant" ? "assistant" : role === "user" ? "user" : "";
return html` return html`
<div class="msg ${klass}"> <div class="chat-line ${klass}">
<div class="meta"> <div class="chat-msg">
<span class="mono">${role}</span> <div class="chat-bubble ${opts?.streaming ? "streaming" : ""}">
<span class="mono">${ts}</span> <div class="chat-text">${text}</div>
</div>
<div class="chat-stamp mono">
${who}${timestamp ? html` · ${timestamp}` : nothing}
</div>
</div> </div>
<div class="msgContent">${text}</div>
</div> </div>
`; `;
} }