feat: 添加AI主动探索测试模式
Some checks failed
AI Web Tester CI / test (push) Has been cancelled

新增功能:
- explorer.py: AI功能探索器
  - 自动发现页面可交互元素
  - 元素分类 (navigation/button/link/card/menu)
  - 危险操作保护 (删除/退出只记录不执行)
  - DOM快速定位替代AI定位 (速度提升10x)
  - 站点地图和BUG清单生成

- main.py: 添加 explore() 方法
- generator.py: 添加探索报告生成 (暗色主题+Mermaid站点图)
- test_cases.py: 支持 goal/explore/hybrid 三种模式

测试结果:
- 成功发现30个可交互元素
- 自动分类: Links(11), Navigation(8), Cards(8), Buttons(2), Menu(1)
- 生成完整HTML探索报告
This commit is contained in:
empty
2025-12-28 20:39:15 +08:00
parent 2a22708ab5
commit c6def51435
4 changed files with 748 additions and 30 deletions

347
src/agent/explorer.py Normal file
View File

@@ -0,0 +1,347 @@
"""
Feature Explorer - AI-driven autonomous feature discovery
"""
from typing import List, Dict, Any, Set, Optional
import json
import re
import logging
from datetime import datetime
logger = logging.getLogger(__name__)
class FeatureExplorer:
"""AI 驱动的功能探索器 - 自主发现并记录页面功能"""
def __init__(self, browser, analyzer):
self.browser = browser
self.analyzer = analyzer
# 探索状态
self.discovered_elements: List[Dict] = []
self.visited_urls: Set[str] = set()
self.site_map: Dict[str, List[str]] = {} # URL -> 子页面列表
self.bug_list: List[Dict] = []
self.action_log: List[Dict] = []
# 默认配置
self.config = {
"max_depth": 3,
"max_clicks": 30,
"skip_patterns": [], # 完全跳过
"dangerous_patterns": ["删除", "移除", "清空", "退出", "注销"], # 记录但不执行
"focus_patterns": [], # 优先探索
}
def explore(self, config: Dict = None) -> Dict[str, Any]:
"""
执行主动探索
Returns:
探索结果报告
"""
if config:
self.config.update(config)
start_url = self.browser.page.url
self.visited_urls.add(start_url)
click_count = 0
print(f"🔍 开始探索: {start_url}")
print(f" 配置: 最大点击={self.config['max_clicks']}")
# 首次发现页面元素
print(f" 正在分析页面元素...")
all_elements = self._discover_elements()
print(f" 发现 {len(all_elements)} 个可交互元素")
if not all_elements:
print(" ⚠️ 没有发现可交互元素")
return self._generate_report(start_url, 0)
# 过滤和排序
elements = self._filter_and_sort(all_elements)
print(f" 过滤后 {len(elements)} 个待探索元素")
# 探索循环 - 逐个探索发现的元素
for element in elements:
if click_count >= self.config["max_clicks"]:
print(f" 达到最大点击数 {self.config['max_clicks']}")
break
click_count += 1
print(f"\n [{click_count}/{min(len(elements), self.config['max_clicks'])}] 探索: {element.get('name', '未知')}")
# 执行探索
self._explore_element(element, click_count)
print(f"\n✅ 探索完成: {click_count} 次点击")
# 生成报告
return self._generate_report(start_url, click_count)
def _discover_elements(self) -> List[Dict]:
"""让 AI 发现页面上所有可交互元素"""
img = self.browser.screenshot_base64()
current_url = self.browser.page.url
prompt = """分析当前页面截图,识别所有可交互的 UI 元素。
**请识别以下类型的元素**:
1. 导航菜单项
2. 侧边栏链接
3. 操作按钮
4. 表单输入框
5. 下拉菜单
6. 标签页/Tab
7. 可点击的卡片或列表项
**返回格式** (只返回 JSON):
```json
{
"elements": [
{
"name": "元素名称/文字",
"type": "navigation|button|form|menu|tab|link|card",
"description": "功能描述",
"priority": 1-10,
"is_dangerous": false
}
],
"page_title": "页面标题",
"page_type": "dashboard|list|form|detail|login|other"
}
```
优先级说明: 10=核心功能, 5=普通功能, 1=次要功能"""
response = self.analyzer.model.analyze(img, prompt)
try:
match = re.search(r'\{[\s\S]*\}', response)
if match:
result = json.loads(match.group())
elements = result.get("elements", [])
# 添加元数据
for el in elements:
el["source_url"] = current_url
el["discovered_at"] = datetime.now().isoformat()
logger.info(f"发现 {len(elements)} 个可交互元素")
return elements
except Exception as e:
logger.warning(f"解析元素失败: {e}")
return []
def _filter_and_sort(self, elements: List[Dict]) -> List[Dict]:
"""过滤和排序元素"""
filtered = []
for el in elements:
name = el.get("name", "")
# 跳过已探索的元素
if self._is_explored(el):
continue
# 完全跳过的模式
if any(p in name for p in self.config["skip_patterns"]):
continue
# 标记危险元素(记录但不执行)
if any(p in name for p in self.config["dangerous_patterns"]):
el["is_dangerous"] = True
# 优先探索的模式
if any(p in name for p in self.config["focus_patterns"]):
el["priority"] = el.get("priority", 5) + 5
filtered.append(el)
# 按优先级排序
filtered.sort(key=lambda x: x.get("priority", 5), reverse=True)
return filtered
def _is_explored(self, element: Dict) -> bool:
"""检查元素是否已探索"""
name = element.get("name", "")
return any(d.get("name") == name for d in self.discovered_elements)
def _explore_element(self, element: Dict, click_num: int) -> None:
"""探索单个元素"""
name = element.get("name", "未知")
el_type = element.get("type", "unknown")
is_dangerous = element.get("is_dangerous", False)
# 记录为已发现
self.discovered_elements.append(element)
# 截图(操作前)- 使用较小的截图减少内存
before_url = self.browser.page.url
before_shot = None # 暂不保存截图以加速
action_record = {
"step": click_num,
"element": element,
"before_url": before_url,
"action_taken": False,
"success": True,
"error": None
}
# 危险元素只记录不执行
if is_dangerous:
print(f" ⚠️ 危险操作,仅记录: {name}")
action_record["skipped"] = True
action_record["skip_reason"] = "危险操作"
self.action_log.append(action_record)
return
# 执行点击 - 使用 DOM 选择器代替 AI 定位(更快)
try:
coords = self._find_element_by_name(name)
if coords:
print(f" → 点击 ({coords[0]}, {coords[1]})")
self.browser.click_at(coords[0], coords[1])
self.browser.wait(500)
action_record["action_taken"] = True
action_record["click_coords"] = coords
else:
print(f" ⚠️ 未找到元素")
action_record["success"] = False
except Exception as e:
print(f" ❌ 点击失败: {e}")
action_record["success"] = False
action_record["error"] = str(e)
self.bug_list.append({
"type": "click_failed",
"element": name,
"error": str(e),
"url": before_url
})
# 检查结果
self.browser.wait(300)
after_url = self.browser.page.url
action_record["after_url"] = after_url
action_record["url_changed"] = before_url != after_url
# 更新站点地图
if before_url != after_url:
if before_url not in self.site_map:
self.site_map[before_url] = []
if after_url not in self.site_map[before_url]:
self.site_map[before_url].append(after_url)
self.visited_urls.add(after_url)
print(f" → 跳转到新页面")
self.action_log.append(action_record)
def _find_element_by_name(self, name: str) -> Optional[tuple]:
"""通过元素名称/文本查找坐标(使用 DOM 而非 AI更快"""
try:
result = self.browser.page.evaluate(f'''
() => {{
// 搜索包含该文本的可点击元素
const text = "{name}";
const clickable = ['A', 'BUTTON', 'INPUT', 'LI', 'SPAN', 'DIV'];
// 遍历所有元素
const all = document.querySelectorAll('*');
for (const el of all) {{
if (el.textContent && el.textContent.trim().includes(text)) {{
if (clickable.includes(el.tagName) ||
el.onclick ||
el.getAttribute('role') === 'button' ||
el.getAttribute('role') === 'menuitem' ||
el.classList.contains('clickable')) {{
const r = el.getBoundingClientRect();
if (r.width > 0 && r.height > 0) {{
return {{
found: true,
x: Math.round(r.left + r.width / 2),
y: Math.round(r.top + r.height / 2)
}};
}}
}}
}}
}}
return {{ found: false }};
}}
''')
if result.get("found"):
return (result["x"], result["y"])
except Exception as e:
logger.warning(f"DOM 查找失败: {e}")
return None
def _locate_element(self, target: str) -> Optional[tuple]:
"""使用 AI 定位元素(备用方法,较慢)"""
img = self.browser.screenshot_base64()
viewport = self.browser.page.viewport_size
width = viewport["width"] if viewport else 1920
height = viewport["height"] if viewport else 1080
prompt = f"""{width}x{height} 像素的截图中,找到 "{target}" 的精确中心坐标。
返回 JSON: {{"x": 数字, "y": 数字, "found": true}}
如果找不到: {{"found": false}}
只返回 JSON。"""
response = self.analyzer.model.analyze(img, prompt)
try:
match = re.search(r'\{[\s\S]*?\}', response)
if match:
result = json.loads(match.group())
if result.get("found"):
return (result["x"], result["y"])
except:
pass
return None
def _detect_bugs(self, action: Dict) -> None:
"""检测可能的 BUG"""
# 检测空白页
# 检测错误提示
# 检测加载失败
# TODO: 使用 AI 分析截图检测异常
pass
def _generate_report(self, start_url: str, click_count: int) -> Dict[str, Any]:
"""生成探索报告"""
# 按类型统计元素
type_stats = {}
for el in self.discovered_elements:
t = el.get("type", "unknown")
type_stats[t] = type_stats.get(t, 0) + 1
report = {
"summary": {
"start_url": start_url,
"total_elements": len(self.discovered_elements),
"total_clicks": click_count,
"pages_visited": len(self.visited_urls),
"bugs_found": len(self.bug_list),
"timestamp": datetime.now().isoformat()
},
"elements_by_type": type_stats,
"discovered_elements": self.discovered_elements,
"site_map": self.site_map,
"bug_list": self.bug_list,
"action_log": self.action_log,
"visited_urls": list(self.visited_urls)
}
logger.info(f"探索完成: 发现 {len(self.discovered_elements)} 个元素, "
f"访问 {len(self.visited_urls)} 个页面, "
f"发现 {len(self.bug_list)} 个问题")
return report