Merge branch 'steipete:main' into main

This commit is contained in:
Dan
2026-01-02 19:53:16 +02:00
committed by GitHub
13 changed files with 429 additions and 41 deletions

View File

@@ -73,6 +73,7 @@
- macOS settings: colorize provider status subtitles to distinguish healthy vs degraded states.
- macOS: keep config writes on the main actor to satisfy Swift concurrency rules.
- macOS menu: show multi-line gateway error details, add an always-visible gateway row, avoid duplicate gateway status rows, suppress transient `cancelled` device refresh errors, and auto-recover the control channel on disconnect.
- macOS menu: show session last-used timestamps in the list and add recent-message previews in session submenus.
- macOS: log health refresh failures and recovery to make gateway issues easier to diagnose.
- macOS codesign: skip hardened runtime for ad-hoc signing and avoid empty options args (#70) — thanks @petter-b
- macOS codesign: include camera entitlement so permission prompts work in the menu bar app.

View File

@@ -192,7 +192,6 @@ actor CameraController {
func listDevices() -> [CameraDeviceInfo] {
let types: [AVCaptureDevice.DeviceType] = [
.builtInWideAngleCamera,
.externalUnknown,
]
let session = AVCaptureDevice.DiscoverySession(
deviceTypes: types,
@@ -308,7 +307,8 @@ actor CameraController {
private nonisolated static func sleepDelayMs(_ delayMs: Int) async {
guard delayMs > 0 else { return }
let ns = UInt64(min(delayMs, 10_000)) * 1_000_000
let maxDelayMs = 10 * 1000
let ns = UInt64(min(delayMs, maxDelayMs)) * UInt64(NSEC_PER_MSEC)
try? await Task.sleep(nanoseconds: ns)
}
}

View File

@@ -300,6 +300,25 @@ final class MenuSessionsInjector: NSObject, NSMenuDelegate {
return item
}
private func makeSessionPreviewItem(
sessionKey: String,
title: String,
width: CGFloat,
maxLines: Int) -> NSMenuItem
{
let item = NSMenuItem()
item.tag = self.tag
item.isEnabled = false
let view = AnyView(SessionMenuPreviewView(
sessionKey: sessionKey,
width: width,
maxItems: 10,
maxLines: maxLines,
title: title))
item.view = self.makeHostedView(rootView: view, width: width, highlighted: false)
return item
}
private func makeMessageItem(text: String, symbolName: String, width: CGFloat) -> NSMenuItem {
let view = AnyView(
Label(text, systemImage: symbolName)
@@ -361,6 +380,19 @@ final class MenuSessionsInjector: NSObject, NSMenuDelegate {
private func buildSubmenu(for row: SessionRow, storePath: String) -> NSMenu {
let menu = NSMenu()
let width = self.submenuWidth()
menu.addItem(self.makeSessionPreviewItem(
sessionKey: row.key,
title: "Recent messages (last 10)",
width: width,
maxLines: 3))
let morePreview = NSMenuItem(title: "More preview…", action: nil, keyEquivalent: "")
morePreview.submenu = self.buildPreviewSubmenu(sessionKey: row.key, width: width)
menu.addItem(morePreview)
menu.addItem(NSMenuItem.separator())
let thinking = NSMenuItem(title: "Thinking", action: nil, keyEquivalent: "")
thinking.submenu = self.buildThinkingMenu(for: row)
@@ -455,6 +487,16 @@ final class MenuSessionsInjector: NSObject, NSMenuDelegate {
return menu
}
private func buildPreviewSubmenu(sessionKey: String, width: CGFloat) -> NSMenu {
let menu = NSMenu()
menu.addItem(self.makeSessionPreviewItem(
sessionKey: sessionKey,
title: "Recent messages (expanded)",
width: width,
maxLines: 8))
return menu
}
private func buildNodesOverflowMenu(entries: [NodeInfo], width: CGFloat) -> NSMenu {
let menu = NSMenu()
for entry in entries {
@@ -705,6 +747,16 @@ final class MenuSessionsInjector: NSObject, NSMenuDelegate {
return self.currentMenuWidth(for: menu)
}
private func submenuWidth() -> CGFloat {
if let openWidth = self.menuOpenWidth {
return max(300, openWidth)
}
if let cached = self.lastKnownMenuWidth {
return max(300, cached)
}
return self.fallbackWidth
}
private func menuWindowWidth(for menu: NSMenu) -> CGFloat? {
var menuWindow: NSWindow?
for item in menu.items {

View File

@@ -45,7 +45,7 @@ struct SessionMenuLabelView: View {
Spacer(minLength: 8)
Text(self.row.tokens.contextSummaryShort)
Text("\(self.row.tokens.contextSummaryShort) · \(self.row.ageText)")
.font(.caption.monospacedDigit())
.foregroundStyle(self.secondaryTextColor)
.lineLimit(1)

View File

@@ -0,0 +1,267 @@
import ClawdisChatUI
import ClawdisKit
import SwiftUI
private struct SessionPreviewItem: Identifiable, Sendable {
let id: String
let role: PreviewRole
let text: String
}
private enum PreviewRole: String, Sendable {
case user
case assistant
case tool
case system
case other
var label: String {
switch self {
case .user: "User"
case .assistant: "Agent"
case .tool: "Tool"
case .system: "System"
case .other: "Other"
}
}
}
private actor SessionPreviewCache {
static let shared = SessionPreviewCache()
private struct CacheEntry {
let items: [SessionPreviewItem]
let updatedAt: Date
}
private var entries: [String: CacheEntry] = [:]
func cachedItems(for sessionKey: String, maxAge: TimeInterval) -> [SessionPreviewItem]? {
guard let entry = self.entries[sessionKey] else { return nil }
guard Date().timeIntervalSince(entry.updatedAt) < maxAge else { return nil }
return entry.items
}
func store(items: [SessionPreviewItem], for sessionKey: String) {
self.entries[sessionKey] = CacheEntry(items: items, updatedAt: Date())
}
}
struct SessionMenuPreviewView: View {
let sessionKey: String
let width: CGFloat
let maxItems: Int
let maxLines: Int
let title: String
@Environment(\.menuItemHighlighted) private var isHighlighted
@State private var items: [SessionPreviewItem] = []
@State private var status: LoadStatus = .loading
private enum LoadStatus: Equatable {
case loading
case ready
case empty
case error(String)
}
private var primaryColor: Color {
self.isHighlighted ? Color(nsColor: .selectedMenuItemTextColor) : .primary
}
private var secondaryColor: Color {
self.isHighlighted ? Color(nsColor: .selectedMenuItemTextColor).opacity(0.85) : .secondary
}
var body: some View {
VStack(alignment: .leading, spacing: 8) {
HStack(alignment: .firstTextBaseline, spacing: 6) {
Text(self.title)
.font(.caption.weight(.semibold))
.foregroundStyle(self.secondaryColor)
Spacer(minLength: 8)
}
switch self.status {
case .loading:
Text("Loading preview…")
.font(.caption)
.foregroundStyle(self.secondaryColor)
case .empty:
Text("No recent messages")
.font(.caption)
.foregroundStyle(self.secondaryColor)
case let .error(message):
Text(message)
.font(.caption)
.foregroundStyle(self.secondaryColor)
case .ready:
VStack(alignment: .leading, spacing: 6) {
ForEach(self.items) { item in
self.previewRow(item)
}
}
}
}
.padding(.vertical, 6)
.padding(.leading, 18)
.padding(.trailing, 12)
.frame(width: max(1, self.width), alignment: .leading)
.task(id: self.sessionKey) {
await self.loadPreview()
}
}
@ViewBuilder
private func previewRow(_ item: SessionPreviewItem) -> some View {
HStack(alignment: .top, spacing: 8) {
Text(item.role.label)
.font(.caption2.monospacedDigit())
.foregroundStyle(self.roleColor(item.role))
.frame(width: 50, alignment: .leading)
Text(item.text)
.font(.caption)
.foregroundStyle(self.primaryColor)
.multilineTextAlignment(.leading)
.lineLimit(self.maxLines)
.truncationMode(.tail)
.fixedSize(horizontal: false, vertical: true)
}
}
private func roleColor(_ role: PreviewRole) -> Color {
if self.isHighlighted { return Color(nsColor: .selectedMenuItemTextColor).opacity(0.9) }
switch role {
case .user: return .accentColor
case .assistant: return .secondary
case .tool: return .orange
case .system: return .gray
case .other: return .secondary
}
}
private func loadPreview() async {
if let cached = await SessionPreviewCache.shared.cachedItems(for: self.sessionKey, maxAge: 12) {
await MainActor.run {
self.items = cached
self.status = cached.isEmpty ? .empty : .ready
}
return
}
await MainActor.run {
self.status = .loading
}
do {
let payload = try await GatewayConnection.shared.chatHistory(sessionKey: self.sessionKey)
let built = Self.previewItems(from: payload, maxItems: self.maxItems)
await SessionPreviewCache.shared.store(items: built, for: self.sessionKey)
await MainActor.run {
self.items = built
self.status = built.isEmpty ? .empty : .ready
}
} catch {
await MainActor.run {
self.status = .error("Preview unavailable")
}
}
}
private static func previewItems(
from payload: ClawdisChatHistoryPayload,
maxItems: Int) -> [SessionPreviewItem]
{
let raw: [ClawdisKit.AnyCodable] = payload.messages ?? []
let messages = self.decodeMessages(raw)
let built = messages.compactMap { message -> SessionPreviewItem? in
guard let text = self.previewText(for: message) else { return nil }
let isTool = self.isToolCall(message)
let role = self.previewRole(message.role, isTool: isTool)
let id = "\(message.timestamp ?? 0)-\(UUID().uuidString)"
return SessionPreviewItem(id: id, role: role, text: text)
}
let trimmed = built.suffix(maxItems)
return Array(trimmed.reversed())
}
private static func decodeMessages(_ raw: [ClawdisKit.AnyCodable]) -> [ClawdisChatMessage] {
raw.compactMap { item in
guard let data = try? JSONEncoder().encode(item) else { return nil }
return try? JSONDecoder().decode(ClawdisChatMessage.self, from: data)
}
}
private static func previewRole(_ raw: String, isTool: Bool) -> PreviewRole {
if isTool { return .tool }
switch raw.lowercased() {
case "user": return .user
case "assistant": return .assistant
case "system": return .system
case "tool": return .tool
default: return .other
}
}
private static func previewText(for message: ClawdisChatMessage) -> String? {
let text = message.content.compactMap(\.text).joined(separator: "\n")
.trimmingCharacters(in: .whitespacesAndNewlines)
if !text.isEmpty { return text }
let toolNames = self.toolNames(for: message)
if !toolNames.isEmpty {
let shown = toolNames.prefix(2)
let overflow = toolNames.count - shown.count
var label = "call \(shown.joined(separator: ", "))"
if overflow > 0 { label += " +\(overflow)" }
return label
}
if let media = self.mediaSummary(for: message) {
return media
}
return nil
}
private static func isToolCall(_ message: ClawdisChatMessage) -> Bool {
if message.toolName?.nonEmpty != nil { return true }
return message.content.contains { $0.name?.nonEmpty != nil || $0.type?.lowercased() == "toolcall" }
}
private static func toolNames(for message: ClawdisChatMessage) -> [String] {
var names: [String] = []
for content in message.content {
if let name = content.name?.nonEmpty {
names.append(name)
}
}
if let toolName = message.toolName?.nonEmpty {
names.append(toolName)
}
return Self.dedupePreservingOrder(names)
}
private static func mediaSummary(for message: ClawdisChatMessage) -> String? {
let types = message.content.compactMap { content -> String? in
let raw = content.type?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
guard let raw, !raw.isEmpty else { return nil }
if raw == "text" || raw == "toolcall" { return nil }
return raw
}
guard let first = types.first else { return nil }
return "[\(first)]"
}
private static func dedupePreservingOrder(_ values: [String]) -> [String] {
var seen = Set<String>()
var result: [String] = []
for value in values where !seen.contains(value) {
seen.insert(value)
result.append(value)
}
return result
}
}

View File

@@ -440,7 +440,7 @@ private struct ChatComposerTextView: NSViewRepresentable {
// Always allow clearing the text (e.g. after send), even while editing.
// Only skip other updates while editing to avoid cursor jumps.
let shouldClear = self.text.isEmpty && !textView.string.isEmpty
if isEditing && !shouldClear { return }
if isEditing, !shouldClear { return }
if textView.string != self.text {
context.coordinator.isProgrammaticUpdate = true

View File

@@ -1,6 +1,8 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
const callGateway = vi.fn();
const { callGateway } = vi.hoisted(() => ({
callGateway: vi.fn(),
}));
vi.mock("../gateway/call.js", () => ({ callGateway }));
vi.mock("../media/image-ops.js", () => ({

View File

@@ -865,20 +865,6 @@ function createCanvasTool(): AnyAgentTool {
Number.isFinite(params.quality)
? params.quality
: undefined;
const delayMs =
typeof params.delayMs === "number" &&
Number.isFinite(params.delayMs)
? params.delayMs
: undefined;
const deviceId =
typeof params.deviceId === "string" && params.deviceId.trim()
? params.deviceId.trim()
: undefined;
const delayMs =
typeof params.delayMs === "number" &&
Number.isFinite(params.delayMs)
? params.delayMs
: undefined;
const raw = (await invoke("canvas.snapshot", {
format,
maxWidth,
@@ -889,8 +875,7 @@ function createCanvasTool(): AnyAgentTool {
ext: payload.format === "jpeg" ? "jpg" : payload.format,
});
await writeBase64ToFile(filePath, payload.base64);
const mimeType =
imageMimeFromFormat(payload.format) ?? "image/png";
const mimeType = imageMimeFromFormat(payload.format) ?? "image/png";
return await imageResult({
label: "canvas:snapshot",
path: filePath,
@@ -1139,6 +1124,15 @@ function createNodesTool(): AnyAgentTool {
Number.isFinite(params.quality)
? params.quality
: undefined;
const delayMs =
typeof params.delayMs === "number" &&
Number.isFinite(params.delayMs)
? params.delayMs
: undefined;
const deviceId =
typeof params.deviceId === "string" && params.deviceId.trim()
? params.deviceId.trim()
: undefined;
const content: AgentToolResult<unknown>["content"] = [];
const details: Array<Record<string, unknown>> = [];
@@ -1158,10 +1152,23 @@ function createNodesTool(): AnyAgentTool {
idempotencyKey: crypto.randomUUID(),
})) as { payload?: unknown };
const payload = parseCameraSnapPayload(raw?.payload);
const normalizedFormat = payload.format.toLowerCase();
if (
normalizedFormat !== "jpg" &&
normalizedFormat !== "jpeg" &&
normalizedFormat !== "png"
) {
throw new Error(
`unsupported camera.snap format: ${payload.format}`,
);
}
const isJpeg =
normalizedFormat === "jpg" || normalizedFormat === "jpeg";
const filePath = cameraTempPath({
kind: "snap",
facing,
ext: payload.format === "jpeg" ? "jpg" : payload.format,
ext: isJpeg ? "jpg" : "png",
});
await writeBase64ToFile(filePath, payload.base64);
content.push({ type: "text", text: `MEDIA:${filePath}` });
@@ -1169,7 +1176,8 @@ function createNodesTool(): AnyAgentTool {
type: "image",
data: payload.base64,
mimeType:
imageMimeFromFormat(payload.format) ?? "image/png",
imageMimeFromFormat(payload.format) ??
(isJpeg ? "image/jpeg" : "image/png"),
});
details.push({
facing,

View File

@@ -32,4 +32,32 @@ describe("tool image sanitizing", () => {
expect(size).toBeLessThanOrEqual(5 * 1024 * 1024);
expect(image.mimeType).toBe("image/jpeg");
}, 20_000);
it("corrects mismatched jpeg mimeType", async () => {
const jpeg = await sharp({
create: {
width: 10,
height: 10,
channels: 3,
background: { r: 255, g: 0, b: 0 },
},
})
.jpeg()
.toBuffer();
const blocks = [
{
type: "image" as const,
data: jpeg.toString("base64"),
mimeType: "image/png",
},
];
const out = await sanitizeContentBlocksImages(blocks, "test");
const image = out.find((b) => b.type === "image");
if (!image || image.type !== "image") {
throw new Error("expected image block");
}
expect(image.mimeType).toBe("image/jpeg");
});
});

View File

@@ -31,6 +31,15 @@ function isTextBlock(block: unknown): block is TextContentBlock {
return rec.type === "text" && typeof rec.text === "string";
}
function inferMimeTypeFromBase64(base64: string): string | undefined {
const trimmed = base64.trim();
if (!trimmed) return undefined;
if (trimmed.startsWith("/9j/")) return "image/jpeg";
if (trimmed.startsWith("iVBOR")) return "image/png";
if (trimmed.startsWith("R0lGOD")) return "image/gif";
return undefined;
}
async function resizeImageBase64IfNeeded(params: {
base64: string;
mimeType: string;
@@ -127,13 +136,19 @@ export async function sanitizeContentBlocksImages(
}
try {
const inferredMimeType = inferMimeTypeFromBase64(data);
const mimeType = inferredMimeType ?? block.mimeType;
const resized = await resizeImageBase64IfNeeded({
base64: data,
mimeType: block.mimeType,
mimeType,
maxDimensionPx,
maxBytes,
});
out.push({ ...block, data: resized.base64, mimeType: resized.mimeType });
out.push({
...block,
data: resized.base64,
mimeType: resized.resized ? resized.mimeType : mimeType,
});
} catch (err) {
out.push({
type: "text",

View File

@@ -43,6 +43,8 @@ type NodesRpcOpts = {
format?: string;
maxWidth?: string;
quality?: string;
delayMs?: string;
deviceId?: string;
duration?: string;
screen?: string;
fps?: string;
@@ -888,7 +890,9 @@ export function registerNodesCli(program: Command) {
const name =
typeof device.name === "string" ? device.name : "Unknown Camera";
const position =
typeof device.position === "string" ? device.position : "unspecified";
typeof device.position === "string"
? device.position
: "unspecified";
defaultRuntime.log(`${name} (${position})${id ? `${id}` : ""}`);
}
} catch (err) {
@@ -908,7 +912,10 @@ export function registerNodesCli(program: Command) {
.option("--device-id <id>", "Camera device id (from nodes camera list)")
.option("--max-width <px>", "Max width in px (optional)")
.option("--quality <0-1>", "JPEG quality (default 0.9)")
.option("--delay-ms <ms>", "Delay before capture in ms (macOS default 2000)")
.option(
"--delay-ms <ms>",
"Delay before capture in ms (macOS default 2000)",
)
.option(
"--invoke-timeout <ms>",
"Node invoke timeout in ms (default 20000)",
@@ -940,7 +947,9 @@ export function registerNodesCli(program: Command) {
const delayMs = opts.delayMs
? Number.parseInt(String(opts.delayMs), 10)
: undefined;
const deviceId = opts.deviceId ? String(opts.deviceId).trim() : undefined;
const deviceId = opts.deviceId
? String(opts.deviceId).trim()
: undefined;
const timeoutMs = opts.invokeTimeout
? Number.parseInt(String(opts.invokeTimeout), 10)
: undefined;
@@ -1037,20 +1046,22 @@ export function registerNodesCli(program: Command) {
const timeoutMs = opts.invokeTimeout
? Number.parseInt(String(opts.invokeTimeout), 10)
: undefined;
const deviceId = opts.deviceId ? String(opts.deviceId).trim() : undefined;
const deviceId = opts.deviceId
? String(opts.deviceId).trim()
: undefined;
const invokeParams: Record<string, unknown> = {
nodeId,
command: "camera.clip",
params: {
facing,
durationMs: Number.isFinite(durationMs) ? durationMs : undefined,
includeAudio,
format: "mp4",
deviceId: deviceId || undefined,
},
idempotencyKey: randomIdempotencyKey(),
};
params: {
facing,
durationMs: Number.isFinite(durationMs) ? durationMs : undefined,
includeAudio,
format: "mp4",
deviceId: deviceId || undefined,
},
idempotencyKey: randomIdempotencyKey(),
};
if (typeof timeoutMs === "number" && Number.isFinite(timeoutMs)) {
invokeParams.timeoutMs = timeoutMs;
}

View File

@@ -419,7 +419,9 @@ export async function monitorDiscordProvider(opts: MonitorDiscordOpts = {}) {
return;
}
const channelName =
interaction.channel && "name" in interaction.channel
interaction.channel &&
"name" in interaction.channel &&
typeof interaction.channel.name === "string"
? interaction.channel.name
: undefined;
const channelSlug = channelName
@@ -459,7 +461,9 @@ export async function monitorDiscordProvider(opts: MonitorDiscordOpts = {}) {
}
} else if (isGroupDm) {
const channelName =
interaction.channel && "name" in interaction.channel
interaction.channel &&
"name" in interaction.channel &&
typeof interaction.channel.name === "string"
? interaction.channel.name
: undefined;
const channelSlug = channelName

View File

@@ -26,7 +26,7 @@ export async function callGateway<T = unknown>(
const timeoutMs = opts.timeoutMs ?? 10_000;
const config = loadConfig();
const isRemoteMode = config.gateway?.mode === "remote";
const remote = isRemoteMode ? config.gateway.remote : undefined;
const remote = isRemoteMode ? config.gateway?.remote : undefined;
const url =
(typeof opts.url === "string" && opts.url.trim().length > 0
? opts.url.trim()