Merge remote-tracking branch 'origin/main' into upstream-preview-nix-2025-12-20
This commit is contained in:
144
scripts/bench-model.ts
Normal file
144
scripts/bench-model.ts
Normal file
@@ -0,0 +1,144 @@
|
||||
import { completeSimple, getModel, type Model } from "@mariozechner/pi-ai";
|
||||
|
||||
type Usage = {
|
||||
input?: number;
|
||||
output?: number;
|
||||
cacheRead?: number;
|
||||
cacheWrite?: number;
|
||||
totalTokens?: number;
|
||||
};
|
||||
|
||||
type RunResult = {
|
||||
durationMs: number;
|
||||
usage?: Usage;
|
||||
};
|
||||
|
||||
const DEFAULT_PROMPT =
|
||||
"Reply with a single word: ok. No punctuation or extra text.";
|
||||
const DEFAULT_RUNS = 10;
|
||||
|
||||
function parseArg(flag: string): string | undefined {
|
||||
const idx = process.argv.indexOf(flag);
|
||||
if (idx === -1) return undefined;
|
||||
return process.argv[idx + 1];
|
||||
}
|
||||
|
||||
function parseRuns(raw: string | undefined): number {
|
||||
if (!raw) return DEFAULT_RUNS;
|
||||
const parsed = Number(raw);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) return DEFAULT_RUNS;
|
||||
return Math.floor(parsed);
|
||||
}
|
||||
|
||||
function median(values: number[]): number {
|
||||
if (values.length === 0) return 0;
|
||||
const sorted = [...values].sort((a, b) => a - b);
|
||||
const mid = Math.floor(sorted.length / 2);
|
||||
if (sorted.length % 2 === 0) {
|
||||
return Math.round((sorted[mid - 1] + sorted[mid]) / 2);
|
||||
}
|
||||
return sorted[mid];
|
||||
}
|
||||
|
||||
async function runModel(opts: {
|
||||
label: string;
|
||||
model: Model<any>;
|
||||
apiKey: string;
|
||||
runs: number;
|
||||
prompt: string;
|
||||
}): Promise<RunResult[]> {
|
||||
const results: RunResult[] = [];
|
||||
for (let i = 0; i < opts.runs; i += 1) {
|
||||
const started = Date.now();
|
||||
const res = await completeSimple(
|
||||
opts.model,
|
||||
{
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: opts.prompt,
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
],
|
||||
},
|
||||
{ apiKey: opts.apiKey, maxTokens: 64 },
|
||||
);
|
||||
const durationMs = Date.now() - started;
|
||||
results.push({ durationMs, usage: res.usage });
|
||||
console.log(
|
||||
`${opts.label} run ${i + 1}/${opts.runs}: ${durationMs}ms`,
|
||||
);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const runs = parseRuns(parseArg("--runs"));
|
||||
const prompt = parseArg("--prompt") ?? DEFAULT_PROMPT;
|
||||
|
||||
const anthropicKey = process.env.ANTHROPIC_API_KEY?.trim();
|
||||
const minimaxKey = process.env.MINIMAX_API_KEY?.trim();
|
||||
if (!anthropicKey) {
|
||||
throw new Error("Missing ANTHROPIC_API_KEY in environment.");
|
||||
}
|
||||
if (!minimaxKey) {
|
||||
throw new Error("Missing MINIMAX_API_KEY in environment.");
|
||||
}
|
||||
|
||||
const minimaxBaseUrl =
|
||||
process.env.MINIMAX_BASE_URL?.trim() || "https://api.minimax.io/v1";
|
||||
const minimaxModelId =
|
||||
process.env.MINIMAX_MODEL?.trim() || "minimax-m2.1";
|
||||
|
||||
const minimaxModel: Model<"openai-completions"> = {
|
||||
id: minimaxModelId,
|
||||
name: `MiniMax ${minimaxModelId}`,
|
||||
api: "openai-completions",
|
||||
provider: "minimax",
|
||||
baseUrl: minimaxBaseUrl,
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 200000,
|
||||
maxTokens: 8192,
|
||||
};
|
||||
const opusModel = getModel("anthropic", "claude-opus-4-5");
|
||||
|
||||
console.log(`Prompt: ${prompt}`);
|
||||
console.log(`Runs: ${runs}`);
|
||||
console.log("");
|
||||
|
||||
const minimaxResults = await runModel({
|
||||
label: "minimax",
|
||||
model: minimaxModel,
|
||||
apiKey: minimaxKey,
|
||||
runs,
|
||||
prompt,
|
||||
});
|
||||
const opusResults = await runModel({
|
||||
label: "opus",
|
||||
model: opusModel,
|
||||
apiKey: anthropicKey,
|
||||
runs,
|
||||
prompt,
|
||||
});
|
||||
|
||||
const summarize = (label: string, results: RunResult[]) => {
|
||||
const durations = results.map((r) => r.durationMs);
|
||||
const med = median(durations);
|
||||
const min = Math.min(...durations);
|
||||
const max = Math.max(...durations);
|
||||
return { label, med, min, max };
|
||||
};
|
||||
|
||||
const summary = [summarize("minimax", minimaxResults), summarize("opus", opusResults)];
|
||||
console.log("");
|
||||
console.log("Summary (ms):");
|
||||
for (const row of summary) {
|
||||
console.log(
|
||||
`${row.label.padEnd(7)} median=${row.med} min=${row.min} max=${row.max}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
await main();
|
||||
@@ -33,6 +33,13 @@ else
|
||||
echo "📦 Skipping TS build (SKIP_TSC=1)"
|
||||
fi
|
||||
|
||||
if [[ "${SKIP_UI_BUILD:-0}" != "1" ]]; then
|
||||
echo "🖥 Building Control UI (pnpm ui:build)"
|
||||
(cd "$ROOT_DIR" && pnpm ui:build)
|
||||
else
|
||||
echo "🖥 Skipping Control UI build (SKIP_UI_BUILD=1)"
|
||||
fi
|
||||
|
||||
cd "$ROOT_DIR/apps/macos"
|
||||
|
||||
echo "🔨 Building $PRODUCT ($BUILD_CONFIG)"
|
||||
@@ -86,6 +93,10 @@ fi
|
||||
echo "🖼 Copying app icon"
|
||||
cp "$ROOT_DIR/apps/macos/Sources/Clawdis/Resources/Clawdis.icns" "$APP_ROOT/Contents/Resources/Clawdis.icns"
|
||||
|
||||
echo "📦 Copying device model resources"
|
||||
rm -rf "$APP_ROOT/Contents/Resources/DeviceModels"
|
||||
cp -R "$ROOT_DIR/apps/macos/Sources/Clawdis/Resources/DeviceModels" "$APP_ROOT/Contents/Resources/DeviceModels"
|
||||
|
||||
RELAY_DIR="$APP_ROOT/Contents/Resources/Relay"
|
||||
|
||||
if [[ "${SKIP_GATEWAY_PACKAGE:-0}" != "1" ]]; then
|
||||
@@ -109,6 +120,10 @@ if [[ "${SKIP_GATEWAY_PACKAGE:-0}" != "1" ]]; then
|
||||
rm -rf "$RELAY_DIR/a2ui"
|
||||
cp -R "$ROOT_DIR/src/canvas-host/a2ui" "$RELAY_DIR/a2ui"
|
||||
|
||||
echo "🎛 Copying Control UI assets"
|
||||
rm -rf "$RELAY_DIR/control-ui"
|
||||
cp -R "$ROOT_DIR/dist/control-ui" "$RELAY_DIR/control-ui"
|
||||
|
||||
echo "🧠 Copying bundled skills"
|
||||
rm -rf "$RELAY_DIR/skills"
|
||||
cp -R "$ROOT_DIR/skills" "$RELAY_DIR/skills"
|
||||
|
||||
@@ -10,6 +10,11 @@ DEBUG_PROCESS_PATTERN="${ROOT_DIR}/apps/macos/.build/debug/Clawdis"
|
||||
LOCAL_PROCESS_PATTERN="${ROOT_DIR}/apps/macos/.build-local/debug/Clawdis"
|
||||
RELEASE_PROCESS_PATTERN="${ROOT_DIR}/apps/macos/.build/release/Clawdis"
|
||||
LAUNCH_AGENT="${HOME}/Library/LaunchAgents/com.steipete.clawdis.plist"
|
||||
LOCK_KEY="$(printf '%s' "${ROOT_DIR}" | shasum -a 256 | cut -c1-8)"
|
||||
LOCK_DIR="${TMPDIR:-/tmp}/clawdis-restart-${LOCK_KEY}"
|
||||
LOCK_PID_FILE="${LOCK_DIR}/pid"
|
||||
WAIT_FOR_LOCK=0
|
||||
LOG_PATH="${CLAWDIS_RESTART_LOG:-/tmp/clawdis-restart.log}"
|
||||
|
||||
log() { printf '%s\n' "$*"; }
|
||||
fail() { printf 'ERROR: %s\n' "$*" >&2; exit 1; }
|
||||
@@ -25,6 +30,60 @@ run_step() {
|
||||
fi
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
if [[ -d "${LOCK_DIR}" ]]; then
|
||||
rm -rf "${LOCK_DIR}"
|
||||
fi
|
||||
}
|
||||
|
||||
acquire_lock() {
|
||||
while true; do
|
||||
if mkdir "${LOCK_DIR}" 2>/dev/null; then
|
||||
echo "$$" > "${LOCK_PID_FILE}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local existing_pid=""
|
||||
if [[ -f "${LOCK_PID_FILE}" ]]; then
|
||||
existing_pid="$(cat "${LOCK_PID_FILE}" 2>/dev/null || true)"
|
||||
fi
|
||||
|
||||
if [[ -n "${existing_pid}" ]] && kill -0 "${existing_pid}" 2>/dev/null; then
|
||||
if [[ "${WAIT_FOR_LOCK}" == "1" ]]; then
|
||||
log "==> Another restart is running (pid ${existing_pid}); waiting..."
|
||||
while kill -0 "${existing_pid}" 2>/dev/null; do
|
||||
sleep 1
|
||||
done
|
||||
continue
|
||||
fi
|
||||
log "==> Another restart is running (pid ${existing_pid}); re-run with --wait."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
rm -rf "${LOCK_DIR}"
|
||||
done
|
||||
}
|
||||
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
for arg in "$@"; do
|
||||
case "${arg}" in
|
||||
--wait|-w) WAIT_FOR_LOCK=1 ;;
|
||||
--help|-h)
|
||||
log "Usage: $(basename "$0") [--wait]"
|
||||
exit 0
|
||||
;;
|
||||
*) ;;
|
||||
esac
|
||||
done
|
||||
|
||||
mkdir -p "$(dirname "$LOG_PATH")"
|
||||
rm -f "$LOG_PATH"
|
||||
exec > >(tee "$LOG_PATH") 2>&1
|
||||
log "==> Log: ${LOG_PATH}"
|
||||
|
||||
acquire_lock
|
||||
|
||||
kill_all_clawdis() {
|
||||
for _ in {1..10}; do
|
||||
pkill -f "${APP_PROCESS_PATTERN}" 2>/dev/null || true
|
||||
@@ -102,5 +161,5 @@ sleep 1.5
|
||||
if pgrep -f "${APP_PROCESS_PATTERN}" >/dev/null 2>&1; then
|
||||
log "OK: Clawdis is running."
|
||||
else
|
||||
fail "App exited immediately. Check /tmp/clawdis.log or Console.app (User Reports)."
|
||||
fail "App exited immediately. Check ${LOG_PATH} or Console.app (User Reports)."
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user