支持鸿蒙OSNEXT_HDC

This commit is contained in:
floatingstarZ
2025-12-16 14:28:49 +08:00
parent 4d427bcd31
commit 95f5921887
12 changed files with 2314 additions and 102 deletions

View File

@@ -0,0 +1,53 @@
"""HDC utilities for HarmonyOS device interaction."""
from phone_agent.hdc.connection import (
HDCConnection,
ConnectionType,
DeviceInfo,
list_devices,
quick_connect,
set_hdc_verbose,
)
from phone_agent.hdc.device import (
back,
double_tap,
get_current_app,
home,
launch_app,
long_press,
swipe,
tap,
)
from phone_agent.hdc.input import (
clear_text,
detect_and_set_adb_keyboard,
restore_keyboard,
type_text,
)
from phone_agent.hdc.screenshot import get_screenshot
__all__ = [
# Screenshot
"get_screenshot",
# Input
"type_text",
"clear_text",
"detect_and_set_adb_keyboard",
"restore_keyboard",
# Device control
"get_current_app",
"tap",
"swipe",
"back",
"home",
"double_tap",
"long_press",
"launch_app",
# Connection management
"HDCConnection",
"DeviceInfo",
"ConnectionType",
"quick_connect",
"list_devices",
"set_hdc_verbose",
]

View File

@@ -0,0 +1,381 @@
"""HDC connection management for HarmonyOS devices."""
import os
import subprocess
import time
from dataclasses import dataclass
from enum import Enum
from typing import Optional
from phone_agent.config.timing import TIMING_CONFIG
# Global flag to control HDC command output
_HDC_VERBOSE = os.getenv("HDC_VERBOSE", "false").lower() in ("true", "1", "yes")
def _run_hdc_command(cmd: list, **kwargs) -> subprocess.CompletedProcess:
"""
Run HDC command with optional verbose output.
Args:
cmd: Command list to execute.
**kwargs: Additional arguments for subprocess.run.
Returns:
CompletedProcess result.
"""
if _HDC_VERBOSE:
print(f"[HDC] Running command: {' '.join(cmd)}")
result = subprocess.run(cmd, **kwargs)
if _HDC_VERBOSE and result.returncode != 0:
print(f"[HDC] Command failed with return code {result.returncode}")
if hasattr(result, 'stderr') and result.stderr:
print(f"[HDC] Error: {result.stderr}")
return result
def set_hdc_verbose(verbose: bool):
"""Set HDC verbose mode globally."""
global _HDC_VERBOSE
_HDC_VERBOSE = verbose
class ConnectionType(Enum):
"""Type of HDC connection."""
USB = "usb"
WIFI = "wifi"
REMOTE = "remote"
@dataclass
class DeviceInfo:
"""Information about a connected device."""
device_id: str
status: str
connection_type: ConnectionType
model: str | None = None
harmony_version: str | None = None
class HDCConnection:
"""
Manages HDC connections to HarmonyOS devices.
Supports USB, WiFi, and remote TCP/IP connections.
Example:
>>> conn = HDCConnection()
>>> # Connect to remote device
>>> conn.connect("192.168.1.100:5555")
>>> # List devices
>>> devices = conn.list_devices()
>>> # Disconnect
>>> conn.disconnect("192.168.1.100:5555")
"""
def __init__(self, hdc_path: str = "hdc"):
"""
Initialize HDC connection manager.
Args:
hdc_path: Path to HDC executable.
"""
self.hdc_path = hdc_path
def connect(self, address: str, timeout: int = 10) -> tuple[bool, str]:
"""
Connect to a remote device via TCP/IP.
Args:
address: Device address in format "host:port" (e.g., "192.168.1.100:5555").
timeout: Connection timeout in seconds.
Returns:
Tuple of (success, message).
Note:
The remote device must have TCP/IP debugging enabled.
"""
# Validate address format
if ":" not in address:
address = f"{address}:5555" # Default HDC port
try:
result = _run_hdc_command(
[self.hdc_path, "tconn", address],
capture_output=True,
text=True,
timeout=timeout,
)
output = result.stdout + result.stderr
if "Connect OK" in output or "connected" in output.lower():
return True, f"Connected to {address}"
elif "already connected" in output.lower():
return True, f"Already connected to {address}"
else:
return False, output.strip()
except subprocess.TimeoutExpired:
return False, f"Connection timeout after {timeout}s"
except Exception as e:
return False, f"Connection error: {e}"
def disconnect(self, address: str | None = None) -> tuple[bool, str]:
"""
Disconnect from a remote device.
Args:
address: Device address to disconnect. If None, disconnects all.
Returns:
Tuple of (success, message).
"""
try:
if address:
cmd = [self.hdc_path, "tdisconn", address]
else:
# HDC doesn't have a "disconnect all" command, so we need to list and disconnect each
devices = self.list_devices()
for device in devices:
if ":" in device.device_id: # Remote device
_run_hdc_command(
[self.hdc_path, "tdisconn", device.device_id],
capture_output=True,
text=True,
timeout=5
)
return True, "Disconnected all remote devices"
result = _run_hdc_command(cmd, capture_output=True, text=True, encoding="utf-8", timeout=5)
output = result.stdout + result.stderr
return True, output.strip() or "Disconnected"
except Exception as e:
return False, f"Disconnect error: {e}"
def list_devices(self) -> list[DeviceInfo]:
"""
List all connected devices.
Returns:
List of DeviceInfo objects.
"""
try:
result = _run_hdc_command(
[self.hdc_path, "list", "targets"],
capture_output=True,
text=True,
timeout=5,
)
devices = []
for line in result.stdout.strip().split("\n"):
if not line.strip():
continue
# HDC output format: device_id (status)
# Example: "192.168.1.100:5555" or "FMR0223C13000649"
device_id = line.strip()
# Determine connection type
if ":" in device_id:
conn_type = ConnectionType.REMOTE
else:
conn_type = ConnectionType.USB
# HDC doesn't provide detailed status in list command
# We assume "Connected" status for devices that appear
devices.append(
DeviceInfo(
device_id=device_id,
status="device",
connection_type=conn_type,
model=None,
)
)
return devices
except Exception as e:
print(f"Error listing devices: {e}")
return []
def get_device_info(self, device_id: str | None = None) -> DeviceInfo | None:
"""
Get detailed information about a device.
Args:
device_id: Device ID. If None, uses first available device.
Returns:
DeviceInfo or None if not found.
"""
devices = self.list_devices()
if not devices:
return None
if device_id is None:
return devices[0]
for device in devices:
if device.device_id == device_id:
return device
return None
def is_connected(self, device_id: str | None = None) -> bool:
"""
Check if a device is connected.
Args:
device_id: Device ID to check. If None, checks if any device is connected.
Returns:
True if connected, False otherwise.
"""
devices = self.list_devices()
if not devices:
return False
if device_id is None:
return len(devices) > 0
return any(d.device_id == device_id for d in devices)
def enable_tcpip(
self, port: int = 5555, device_id: str | None = None
) -> tuple[bool, str]:
"""
Enable TCP/IP debugging on a USB-connected device.
This allows subsequent wireless connections to the device.
Args:
port: TCP port for HDC (default: 5555).
device_id: Device ID. If None, uses first available device.
Returns:
Tuple of (success, message).
Note:
The device must be connected via USB first.
After this, you can disconnect USB and connect via WiFi.
"""
try:
cmd = [self.hdc_path]
if device_id:
cmd.extend(["-t", device_id])
cmd.extend(["tmode", "port", str(port)])
result = _run_hdc_command(cmd, capture_output=True, text=True, encoding="utf-8", timeout=10)
output = result.stdout + result.stderr
if result.returncode == 0 or "success" in output.lower():
time.sleep(TIMING_CONFIG.connection.adb_restart_delay)
return True, f"TCP/IP mode enabled on port {port}"
else:
return False, output.strip()
except Exception as e:
return False, f"Error enabling TCP/IP: {e}"
def get_device_ip(self, device_id: str | None = None) -> str | None:
"""
Get the IP address of a connected device.
Args:
device_id: Device ID. If None, uses first available device.
Returns:
IP address string or None if not found.
"""
try:
cmd = [self.hdc_path]
if device_id:
cmd.extend(["-t", device_id])
cmd.extend(["shell", "ifconfig"])
result = _run_hdc_command(cmd, capture_output=True, text=True, encoding="utf-8", timeout=5)
# Parse IP from ifconfig output
for line in result.stdout.split("\n"):
if "inet addr:" in line or "inet " in line:
parts = line.strip().split()
for i, part in enumerate(parts):
if "addr:" in part:
ip = part.split(":")[1]
# Filter out localhost
if not ip.startswith("127."):
return ip
elif part == "inet" and i + 1 < len(parts):
ip = parts[i + 1].split("/")[0]
if not ip.startswith("127."):
return ip
return None
except Exception as e:
print(f"Error getting device IP: {e}")
return None
def restart_server(self) -> tuple[bool, str]:
"""
Restart the HDC server.
Returns:
Tuple of (success, message).
"""
try:
# Kill server
_run_hdc_command(
[self.hdc_path, "kill"], capture_output=True, timeout=5
)
time.sleep(TIMING_CONFIG.connection.server_restart_delay)
# Start server (HDC auto-starts when running commands)
_run_hdc_command(
[self.hdc_path, "start", "-r"], capture_output=True, timeout=5
)
return True, "HDC server restarted"
except Exception as e:
return False, f"Error restarting server: {e}"
def quick_connect(address: str) -> tuple[bool, str]:
"""
Quick helper to connect to a remote device.
Args:
address: Device address (e.g., "192.168.1.100" or "192.168.1.100:5555").
Returns:
Tuple of (success, message).
"""
conn = HDCConnection()
return conn.connect(address)
def list_devices() -> list[DeviceInfo]:
"""
Quick helper to list connected devices.
Returns:
List of DeviceInfo objects.
"""
conn = HDCConnection()
return conn.list_devices()

269
phone_agent/hdc/device.py Normal file
View File

@@ -0,0 +1,269 @@
"""Device control utilities for HarmonyOS automation."""
import os
import subprocess
import time
from typing import List, Optional, Tuple
from phone_agent.config.apps_harmonyos import APP_PACKAGES
from phone_agent.config.timing import TIMING_CONFIG
from phone_agent.hdc.connection import _run_hdc_command
def get_current_app(device_id: str | None = None) -> str:
"""
Get the currently focused app name.
Args:
device_id: Optional HDC device ID for multi-device setups.
Returns:
The app name if recognized, otherwise "System Home".
"""
hdc_prefix = _get_hdc_prefix(device_id)
result = _run_hdc_command(
hdc_prefix + ["shell", "hidumper", "-s", "WindowManagerService", "-a", "-a"],
capture_output=True,
text=True,
encoding="utf-8"
)
output = result.stdout
if not output:
raise ValueError("No output from hidumper")
# Parse window focus info
for line in output.split("\n"):
if "focused" in line.lower() or "current" in line.lower():
for app_name, package in APP_PACKAGES.items():
if package in line:
return app_name
return "System Home"
def tap(
x: int, y: int, device_id: str | None = None, delay: float | None = None
) -> None:
"""
Tap at the specified coordinates.
Args:
x: X coordinate.
y: Y coordinate.
device_id: Optional HDC device ID.
delay: Delay in seconds after tap. If None, uses configured default.
"""
if delay is None:
delay = TIMING_CONFIG.device.default_tap_delay
hdc_prefix = _get_hdc_prefix(device_id)
# HarmonyOS uses uitest uiInput click
_run_hdc_command(
hdc_prefix + ["shell", "uitest", "uiInput", "click", str(x), str(y)],
capture_output=True
)
time.sleep(delay)
def double_tap(
x: int, y: int, device_id: str | None = None, delay: float | None = None
) -> None:
"""
Double tap at the specified coordinates.
Args:
x: X coordinate.
y: Y coordinate.
device_id: Optional HDC device ID.
delay: Delay in seconds after double tap. If None, uses configured default.
"""
if delay is None:
delay = TIMING_CONFIG.device.default_double_tap_delay
hdc_prefix = _get_hdc_prefix(device_id)
# HarmonyOS uses uitest uiInput doubleClick
_run_hdc_command(
hdc_prefix + ["shell", "uitest", "uiInput", "doubleClick", str(x), str(y)],
capture_output=True
)
time.sleep(delay)
def long_press(
x: int,
y: int,
duration_ms: int = 3000,
device_id: str | None = None,
delay: float | None = None,
) -> None:
"""
Long press at the specified coordinates.
Args:
x: X coordinate.
y: Y coordinate.
duration_ms: Duration of press in milliseconds (note: HarmonyOS longClick may not support duration).
device_id: Optional HDC device ID.
delay: Delay in seconds after long press. If None, uses configured default.
"""
if delay is None:
delay = TIMING_CONFIG.device.default_long_press_delay
hdc_prefix = _get_hdc_prefix(device_id)
# HarmonyOS uses uitest uiInput longClick
# Note: longClick may have a fixed duration, duration_ms parameter might not be supported
_run_hdc_command(
hdc_prefix + ["shell", "uitest", "uiInput", "longClick", str(x), str(y)],
capture_output=True,
)
time.sleep(delay)
def swipe(
start_x: int,
start_y: int,
end_x: int,
end_y: int,
duration_ms: int | None = None,
device_id: str | None = None,
delay: float | None = None,
) -> None:
"""
Swipe from start to end coordinates.
Args:
start_x: Starting X coordinate.
start_y: Starting Y coordinate.
end_x: Ending X coordinate.
end_y: Ending Y coordinate.
duration_ms: Duration of swipe in milliseconds (auto-calculated if None).
device_id: Optional HDC device ID.
delay: Delay in seconds after swipe. If None, uses configured default.
"""
if delay is None:
delay = TIMING_CONFIG.device.default_swipe_delay
hdc_prefix = _get_hdc_prefix(device_id)
if duration_ms is None:
# Calculate duration based on distance
dist_sq = (start_x - end_x) ** 2 + (start_y - end_y) ** 2
duration_ms = int(dist_sq / 1000)
duration_ms = max(500, min(duration_ms, 1000)) # Clamp between 500-1000ms
# HarmonyOS uses uitest uiInput swipe
# Format: swipe startX startY endX endY duration
_run_hdc_command(
hdc_prefix
+ [
"shell",
"uitest",
"uiInput",
"swipe",
str(start_x),
str(start_y),
str(end_x),
str(end_y),
str(duration_ms),
],
capture_output=True,
)
time.sleep(delay)
def back(device_id: str | None = None, delay: float | None = None) -> None:
"""
Press the back button.
Args:
device_id: Optional HDC device ID.
delay: Delay in seconds after pressing back. If None, uses configured default.
"""
if delay is None:
delay = TIMING_CONFIG.device.default_back_delay
hdc_prefix = _get_hdc_prefix(device_id)
# HarmonyOS uses uitest uiInput keyEvent Back
_run_hdc_command(
hdc_prefix + ["shell", "uitest", "uiInput", "keyEvent", "Back"],
capture_output=True
)
time.sleep(delay)
def home(device_id: str | None = None, delay: float | None = None) -> None:
"""
Press the home button.
Args:
device_id: Optional HDC device ID.
delay: Delay in seconds after pressing home. If None, uses configured default.
"""
if delay is None:
delay = TIMING_CONFIG.device.default_home_delay
hdc_prefix = _get_hdc_prefix(device_id)
# HarmonyOS uses uitest uiInput keyEvent Home
_run_hdc_command(
hdc_prefix + ["shell", "uitest", "uiInput", "keyEvent", "Home"],
capture_output=True
)
time.sleep(delay)
def launch_app(
app_name: str, device_id: str | None = None, delay: float | None = None
) -> bool:
"""
Launch an app by name.
Args:
app_name: The app name (must be in APP_PACKAGES).
device_id: Optional HDC device ID.
delay: Delay in seconds after launching. If None, uses configured default.
Returns:
True if app was launched, False if app not found.
"""
if delay is None:
delay = TIMING_CONFIG.device.default_launch_delay
if app_name not in APP_PACKAGES:
print(f"[HDC] App '{app_name}' not found in HarmonyOS app list")
print(f"[HDC] Available apps: {', '.join(sorted(APP_PACKAGES.keys())[:10])}...")
return False
hdc_prefix = _get_hdc_prefix(device_id)
bundle = APP_PACKAGES[app_name]
# HarmonyOS uses 'aa start' command to launch apps
# Format: aa start -b {bundle} -a {ability}
# Most HarmonyOS apps use "EntryAbility" as the main ability name
_run_hdc_command(
hdc_prefix
+ [
"shell",
"aa",
"start",
"-b",
bundle,
"-a",
"EntryAbility",
],
capture_output=True,
)
time.sleep(delay)
return True
def _get_hdc_prefix(device_id: str | None) -> list:
"""Get HDC command prefix with optional device specifier."""
if device_id:
return ["hdc", "-t", device_id]
return ["hdc"]

136
phone_agent/hdc/input.py Normal file
View File

@@ -0,0 +1,136 @@
"""Input utilities for HarmonyOS device text input."""
import base64
import subprocess
from typing import Optional
from phone_agent.hdc.connection import _run_hdc_command
def type_text(text: str, device_id: str | None = None, x: int = None, y: int = None) -> None:
"""
Type text into the currently focused input field.
Args:
text: The text to type.
device_id: Optional HDC device ID for multi-device setups.
x: Optional X coordinate for input field (deprecated, kept for compatibility).
y: Optional Y coordinate for input field (deprecated, kept for compatibility).
Note:
HarmonyOS uses: hdc shell uitest uiInput text "文本内容"
This command works without coordinates when input field is focused.
Recommendation: Click on the input field first to focus it, then use this function.
"""
hdc_prefix = _get_hdc_prefix(device_id)
# Escape special characters for shell (keep quotes for proper text handling)
# The text will be wrapped in quotes in the command
escaped_text = text.replace('"', '\\"').replace("$", "\\$")
try:
# HarmonyOS uitest uiInput text command
# Format: hdc shell uitest uiInput text "文本内容"
_run_hdc_command(
hdc_prefix + ["shell", "uitest", "uiInput", "text", escaped_text],
capture_output=True,
text=True,
)
except Exception as e:
print(f"[HDC] Text input failed: {e}")
# Fallback: try with coordinates if provided (for older HarmonyOS versions)
if x is not None and y is not None:
try:
_run_hdc_command(
hdc_prefix + ["shell", "uitest", "uiInput", "inputText", str(x), str(y), escaped_text],
capture_output=True,
text=True,
)
except Exception:
pass
def clear_text(device_id: str | None = None) -> None:
"""
Clear text in the currently focused input field.
Args:
device_id: Optional HDC device ID for multi-device setups.
Note:
This method uses repeated delete key events to clear text.
For HarmonyOS, you might also use select all + delete for better efficiency.
"""
hdc_prefix = _get_hdc_prefix(device_id)
# Ctrl+A to select all (key code 2072 for Ctrl, 2017 for A)
# Then delete
_run_hdc_command(
hdc_prefix + ["shell", "uitest", "uiInput", "keyEvent", "2072", "2017"],
capture_output=True,
text=True,
)
_run_hdc_command(
hdc_prefix + ["shell", "uitest", "uiInput", "keyEvent", "2055"], # Delete key
capture_output=True,
text=True,
)
def detect_and_set_adb_keyboard(device_id: str | None = None) -> str:
"""
Detect current keyboard and switch to ADB Keyboard if available.
Args:
device_id: Optional HDC device ID for multi-device setups.
Returns:
The original keyboard IME identifier for later restoration.
Note:
This is a placeholder. HarmonyOS may not support ADB Keyboard.
If there's a similar tool for HarmonyOS, integrate it here.
"""
hdc_prefix = _get_hdc_prefix(device_id)
# Get current IME (if HarmonyOS supports this)
try:
result = _run_hdc_command(
hdc_prefix + ["shell", "settings", "get", "secure", "default_input_method"],
capture_output=True,
text=True,
)
current_ime = (result.stdout + result.stderr).strip()
# If ADB Keyboard equivalent exists for HarmonyOS, switch to it
# For now, we'll just return the current IME
return current_ime
except Exception:
return ""
def restore_keyboard(ime: str, device_id: str | None = None) -> None:
"""
Restore the original keyboard IME.
Args:
ime: The IME identifier to restore.
device_id: Optional HDC device ID for multi-device setups.
"""
if not ime:
return
hdc_prefix = _get_hdc_prefix(device_id)
try:
_run_hdc_command(
hdc_prefix + ["shell", "ime", "set", ime], capture_output=True, text=True
)
except Exception:
pass
def _get_hdc_prefix(device_id: str | None) -> list:
"""Get HDC command prefix with optional device specifier."""
if device_id:
return ["hdc", "-t", device_id]
return ["hdc"]

View File

@@ -0,0 +1,125 @@
"""Screenshot utilities for capturing HarmonyOS device screen."""
import base64
import os
import subprocess
import tempfile
import uuid
from dataclasses import dataclass
from io import BytesIO
from typing import Tuple
from PIL import Image
from phone_agent.hdc.connection import _run_hdc_command
@dataclass
class Screenshot:
"""Represents a captured screenshot."""
base64_data: str
width: int
height: int
is_sensitive: bool = False
def get_screenshot(device_id: str | None = None, timeout: int = 10) -> Screenshot:
"""
Capture a screenshot from the connected HarmonyOS device.
Args:
device_id: Optional HDC device ID for multi-device setups.
timeout: Timeout in seconds for screenshot operations.
Returns:
Screenshot object containing base64 data and dimensions.
Note:
If the screenshot fails (e.g., on sensitive screens like payment pages),
a black fallback image is returned with is_sensitive=True.
"""
temp_path = os.path.join(tempfile.gettempdir(), f"screenshot_{uuid.uuid4()}.png")
hdc_prefix = _get_hdc_prefix(device_id)
try:
# Execute screenshot command
# HarmonyOS HDC only supports JPEG format
remote_path = "/data/local/tmp/tmp_screenshot.jpeg"
# Try method 1: hdc shell screenshot (newer HarmonyOS versions)
result = _run_hdc_command(
hdc_prefix + ["shell", "screenshot", remote_path],
capture_output=True,
text=True,
timeout=timeout,
)
# Check for screenshot failure (sensitive screen)
output = result.stdout + result.stderr
if "fail" in output.lower() or "error" in output.lower() or "not found" in output.lower():
# Try method 2: snapshot_display (older versions or different devices)
result = _run_hdc_command(
hdc_prefix + ["shell", "snapshot_display", "-f", remote_path],
capture_output=True,
text=True,
timeout=timeout,
)
output = result.stdout + result.stderr
if "fail" in output.lower() or "error" in output.lower():
return _create_fallback_screenshot(is_sensitive=True)
# Pull screenshot to local temp path
# Note: remote file is JPEG, but PIL can open it regardless of local extension
_run_hdc_command(
hdc_prefix + ["file", "recv", remote_path, temp_path],
capture_output=True,
text=True,
timeout=5,
)
if not os.path.exists(temp_path):
return _create_fallback_screenshot(is_sensitive=False)
# Read JPEG image and convert to PNG for model inference
# PIL automatically detects the image format from file content
img = Image.open(temp_path)
width, height = img.size
buffered = BytesIO()
img.save(buffered, format="PNG")
base64_data = base64.b64encode(buffered.getvalue()).decode("utf-8")
# Cleanup
os.remove(temp_path)
return Screenshot(
base64_data=base64_data, width=width, height=height, is_sensitive=False
)
except Exception as e:
print(f"Screenshot error: {e}")
return _create_fallback_screenshot(is_sensitive=False)
def _get_hdc_prefix(device_id: str | None) -> list:
"""Get HDC command prefix with optional device specifier."""
if device_id:
return ["hdc", "-t", device_id]
return ["hdc"]
def _create_fallback_screenshot(is_sensitive: bool) -> Screenshot:
"""Create a black fallback image when screenshot fails."""
default_width, default_height = 1080, 2400
black_img = Image.new("RGB", (default_width, default_height), color="black")
buffered = BytesIO()
black_img.save(buffered, format="PNG")
base64_data = base64.b64encode(buffered.getvalue()).decode("utf-8")
return Screenshot(
base64_data=base64_data,
width=default_width,
height=default_height,
is_sensitive=is_sensitive,
)