"""Model client for AI inference using OpenAI-compatible API.""" import json from dataclasses import dataclass, field from typing import Any from openai import OpenAI @dataclass class ModelConfig: """Configuration for the AI model.""" base_url: str = "http://localhost:8000/v1" api_key: str = "EMPTY" model_name: str = "autoglm-phone-9b" max_tokens: int = 3000 temperature: float = 0.0 top_p: float = 0.85 frequency_penalty: float = 0.2 extra_body: dict[str, Any] = field(default_factory=dict) @dataclass class ModelResponse: """Response from the AI model.""" thinking: str action: str raw_content: str class ModelClient: """ Client for interacting with OpenAI-compatible vision-language models. Args: config: Model configuration. """ def __init__(self, config: ModelConfig | None = None): self.config = config or ModelConfig() self.client = OpenAI(base_url=self.config.base_url, api_key=self.config.api_key) def request(self, messages: list[dict[str, Any]]) -> ModelResponse: """ Send a request to the model. Args: messages: List of message dictionaries in OpenAI format. Returns: ModelResponse containing thinking and action. Raises: ValueError: If the response cannot be parsed. """ response = self.client.chat.completions.create( messages=messages, model=self.config.model_name, max_tokens=self.config.max_tokens, temperature=self.config.temperature, top_p=self.config.top_p, frequency_penalty=self.config.frequency_penalty, extra_body=self.config.extra_body, stream=False, ) raw_content = response.choices[0].message.content # Parse thinking and action from response thinking, action = self._parse_response(raw_content) return ModelResponse(thinking=thinking, action=action, raw_content=raw_content) def _parse_response(self, content: str) -> tuple[str, str]: """ Parse the model response into thinking and action parts. Parsing rules: 1. If content contains 'finish(message=', everything before is thinking, everything from 'finish(message=' onwards is action. 2. If rule 1 doesn't apply but content contains 'do(action=', everything before is thinking, everything from 'do(action=' onwards is action. 3. Fallback: If content contains '', use legacy parsing with XML tags. 4. Otherwise, return empty thinking and full content as action. Args: content: Raw response content. Returns: Tuple of (thinking, action). """ # Rule 1: Check for finish(message= if "finish(message=" in content: parts = content.split("finish(message=", 1) thinking = parts[0].strip() action = "finish(message=" + parts[1] return thinking, action # Rule 2: Check for do(action= if "do(action=" in content: parts = content.split("do(action=", 1) thinking = parts[0].strip() action = "do(action=" + parts[1] return thinking, action # Rule 3: Fallback to legacy XML tag parsing if "" in content: parts = content.split("", 1) thinking = parts[0].replace("", "").replace("", "").strip() action = parts[1].replace("", "").strip() return thinking, action # Rule 4: No markers found, return content as action return "", content class MessageBuilder: """Helper class for building conversation messages.""" @staticmethod def create_system_message(content: str) -> dict[str, Any]: """Create a system message.""" return {"role": "system", "content": content} @staticmethod def create_user_message( text: str, image_base64: str | None = None ) -> dict[str, Any]: """ Create a user message with optional image. Args: text: Text content. image_base64: Optional base64-encoded image. Returns: Message dictionary. """ content = [] if image_base64: content.append( { "type": "image_url", "image_url": {"url": f"data:image/png;base64,{image_base64}"}, } ) content.append({"type": "text", "text": text}) return {"role": "user", "content": content} @staticmethod def create_assistant_message(content: str) -> dict[str, Any]: """Create an assistant message.""" return {"role": "assistant", "content": content} @staticmethod def remove_images_from_message(message: dict[str, Any]) -> dict[str, Any]: """ Remove image content from a message to save context space. Args: message: Message dictionary. Returns: Message with images removed. """ if isinstance(message.get("content"), list): message["content"] = [ item for item in message["content"] if item.get("type") == "text" ] return message @staticmethod def build_screen_info(current_app: str, **extra_info) -> str: """ Build screen info string for the model. Args: current_app: Current app name. **extra_info: Additional info to include. Returns: JSON string with screen info. """ info = {"current_app": current_app, **extra_info} return json.dumps(info, ensure_ascii=False)