c6def51435
Some checks failed
AI Web Tester CI / test (push) Has been cancelled
新增功能: - explorer.py: AI功能探索器 - 自动发现页面可交互元素 - 元素分类 (navigation/button/link/card/menu) - 危险操作保护 (删除/退出只记录不执行) - DOM快速定位替代AI定位 (速度提升10x) - 站点地图和BUG清单生成 - main.py: 添加 explore() 方法 - generator.py: 添加探索报告生成 (暗色主题+Mermaid站点图) - test_cases.py: 支持 goal/explore/hybrid 三种模式 测试结果: - 成功发现30个可交互元素 - 自动分类: Links(11), Navigation(8), Cards(8), Buttons(2), Menu(1) - 生成完整HTML探索报告
348 lines
12 KiB
Python
348 lines
12 KiB
Python
"""
|
||
Feature Explorer - AI-driven autonomous feature discovery
|
||
"""
|
||
from typing import List, Dict, Any, Set, Optional
|
||
import json
|
||
import re
|
||
import logging
|
||
from datetime import datetime
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
class FeatureExplorer:
|
||
"""AI 驱动的功能探索器 - 自主发现并记录页面功能"""
|
||
|
||
def __init__(self, browser, analyzer):
|
||
self.browser = browser
|
||
self.analyzer = analyzer
|
||
|
||
# 探索状态
|
||
self.discovered_elements: List[Dict] = []
|
||
self.visited_urls: Set[str] = set()
|
||
self.site_map: Dict[str, List[str]] = {} # URL -> 子页面列表
|
||
self.bug_list: List[Dict] = []
|
||
self.action_log: List[Dict] = []
|
||
|
||
# 默认配置
|
||
self.config = {
|
||
"max_depth": 3,
|
||
"max_clicks": 30,
|
||
"skip_patterns": [], # 完全跳过
|
||
"dangerous_patterns": ["删除", "移除", "清空", "退出", "注销"], # 记录但不执行
|
||
"focus_patterns": [], # 优先探索
|
||
}
|
||
|
||
def explore(self, config: Dict = None) -> Dict[str, Any]:
|
||
"""
|
||
执行主动探索
|
||
|
||
Returns:
|
||
探索结果报告
|
||
"""
|
||
if config:
|
||
self.config.update(config)
|
||
|
||
start_url = self.browser.page.url
|
||
self.visited_urls.add(start_url)
|
||
click_count = 0
|
||
|
||
print(f"🔍 开始探索: {start_url}")
|
||
print(f" 配置: 最大点击={self.config['max_clicks']}")
|
||
|
||
# 首次发现页面元素
|
||
print(f" 正在分析页面元素...")
|
||
all_elements = self._discover_elements()
|
||
print(f" 发现 {len(all_elements)} 个可交互元素")
|
||
|
||
if not all_elements:
|
||
print(" ⚠️ 没有发现可交互元素")
|
||
return self._generate_report(start_url, 0)
|
||
|
||
# 过滤和排序
|
||
elements = self._filter_and_sort(all_elements)
|
||
print(f" 过滤后 {len(elements)} 个待探索元素")
|
||
|
||
# 探索循环 - 逐个探索发现的元素
|
||
for element in elements:
|
||
if click_count >= self.config["max_clicks"]:
|
||
print(f" 达到最大点击数 {self.config['max_clicks']}")
|
||
break
|
||
|
||
click_count += 1
|
||
print(f"\n [{click_count}/{min(len(elements), self.config['max_clicks'])}] 探索: {element.get('name', '未知')}")
|
||
|
||
# 执行探索
|
||
self._explore_element(element, click_count)
|
||
|
||
print(f"\n✅ 探索完成: {click_count} 次点击")
|
||
|
||
# 生成报告
|
||
return self._generate_report(start_url, click_count)
|
||
|
||
def _discover_elements(self) -> List[Dict]:
|
||
"""让 AI 发现页面上所有可交互元素"""
|
||
img = self.browser.screenshot_base64()
|
||
current_url = self.browser.page.url
|
||
|
||
prompt = """分析当前页面截图，识别所有可交互的 UI 元素。
|
||
|
||
**请识别以下类型的元素**:
|
||
1. 导航菜单项
|
||
2. 侧边栏链接
|
||
3. 操作按钮
|
||
4. 表单输入框
|
||
5. 下拉菜单
|
||
6. 标签页/Tab
|
||
7. 可点击的卡片或列表项
|
||
|
||
**返回格式** (只返回 JSON):
|
||
```json
|
||
{
|
||
"elements": [
|
||
{
|
||
"name": "元素名称/文字",
|
||
"type": "navigation|button|form|menu|tab|link|card",
|
||
"description": "功能描述",
|
||
"priority": 1-10,
|
||
"is_dangerous": false
|
||
}
|
||
],
|
||
"page_title": "页面标题",
|
||
"page_type": "dashboard|list|form|detail|login|other"
|
||
}
|
||
```
|
||
|
||
优先级说明: 10=核心功能, 5=普通功能, 1=次要功能"""
|
||
|
||
response = self.analyzer.model.analyze(img, prompt)
|
||
|
||
try:
|
||
match = re.search(r'\{[\s\S]*\}', response)
|
||
if match:
|
||
result = json.loads(match.group())
|
||
elements = result.get("elements", [])
|
||
|
||
# 添加元数据
|
||
for el in elements:
|
||
el["source_url"] = current_url
|
||
el["discovered_at"] = datetime.now().isoformat()
|
||
|
||
logger.info(f"发现 {len(elements)} 个可交互元素")
|
||
return elements
|
||
except Exception as e:
|
||
logger.warning(f"解析元素失败: {e}")
|
||
|
||
return []
|
||
|
||
def _filter_and_sort(self, elements: List[Dict]) -> List[Dict]:
|
||
"""过滤和排序元素"""
|
||
filtered = []
|
||
|
||
for el in elements:
|
||
name = el.get("name", "")
|
||
|
||
# 跳过已探索的元素
|
||
if self._is_explored(el):
|
||
continue
|
||
|
||
# 完全跳过的模式
|
||
if any(p in name for p in self.config["skip_patterns"]):
|
||
continue
|
||
|
||
# 标记危险元素（记录但不执行）
|
||
if any(p in name for p in self.config["dangerous_patterns"]):
|
||
el["is_dangerous"] = True
|
||
|
||
# 优先探索的模式
|
||
if any(p in name for p in self.config["focus_patterns"]):
|
||
el["priority"] = el.get("priority", 5) + 5
|
||
|
||
filtered.append(el)
|
||
|
||
# 按优先级排序
|
||
filtered.sort(key=lambda x: x.get("priority", 5), reverse=True)
|
||
|
||
return filtered
|
||
|
||
def _is_explored(self, element: Dict) -> bool:
|
||
"""检查元素是否已探索"""
|
||
name = element.get("name", "")
|
||
return any(d.get("name") == name for d in self.discovered_elements)
|
||
|
||
def _explore_element(self, element: Dict, click_num: int) -> None:
|
||
"""探索单个元素"""
|
||
name = element.get("name", "未知")
|
||
el_type = element.get("type", "unknown")
|
||
is_dangerous = element.get("is_dangerous", False)
|
||
|
||
# 记录为已发现
|
||
self.discovered_elements.append(element)
|
||
|
||
# 截图（操作前）- 使用较小的截图减少内存
|
||
before_url = self.browser.page.url
|
||
before_shot = None # 暂不保存截图以加速
|
||
|
||
action_record = {
|
||
"step": click_num,
|
||
"element": element,
|
||
"before_url": before_url,
|
||
"action_taken": False,
|
||
"success": True,
|
||
"error": None
|
||
}
|
||
|
||
# 危险元素只记录不执行
|
||
if is_dangerous:
|
||
print(f" ⚠️ 危险操作，仅记录: {name}")
|
||
action_record["skipped"] = True
|
||
action_record["skip_reason"] = "危险操作"
|
||
self.action_log.append(action_record)
|
||
return
|
||
|
||
# 执行点击 - 使用 DOM 选择器代替 AI 定位（更快）
|
||
try:
|
||
coords = self._find_element_by_name(name)
|
||
if coords:
|
||
print(f" → 点击 ({coords[0]}, {coords[1]})")
|
||
self.browser.click_at(coords[0], coords[1])
|
||
self.browser.wait(500)
|
||
action_record["action_taken"] = True
|
||
action_record["click_coords"] = coords
|
||
else:
|
||
print(f" ⚠️ 未找到元素")
|
||
action_record["success"] = False
|
||
except Exception as e:
|
||
print(f" ❌ 点击失败: {e}")
|
||
action_record["success"] = False
|
||
action_record["error"] = str(e)
|
||
self.bug_list.append({
|
||
"type": "click_failed",
|
||
"element": name,
|
||
"error": str(e),
|
||
"url": before_url
|
||
})
|
||
|
||
# 检查结果
|
||
self.browser.wait(300)
|
||
after_url = self.browser.page.url
|
||
|
||
action_record["after_url"] = after_url
|
||
action_record["url_changed"] = before_url != after_url
|
||
|
||
# 更新站点地图
|
||
if before_url != after_url:
|
||
if before_url not in self.site_map:
|
||
self.site_map[before_url] = []
|
||
if after_url not in self.site_map[before_url]:
|
||
self.site_map[before_url].append(after_url)
|
||
self.visited_urls.add(after_url)
|
||
print(f" → 跳转到新页面")
|
||
|
||
self.action_log.append(action_record)
|
||
|
||
def _find_element_by_name(self, name: str) -> Optional[tuple]:
|
||
"""通过元素名称/文本查找坐标（使用 DOM 而非 AI，更快）"""
|
||
try:
|
||
result = self.browser.page.evaluate(f'''
|
||
() => {{
|
||
// 搜索包含该文本的可点击元素
|
||
const text = "{name}";
|
||
const clickable = ['A', 'BUTTON', 'INPUT', 'LI', 'SPAN', 'DIV'];
|
||
|
||
// 遍历所有元素
|
||
const all = document.querySelectorAll('*');
|
||
for (const el of all) {{
|
||
if (el.textContent && el.textContent.trim().includes(text)) {{
|
||
if (clickable.includes(el.tagName) ||
|
||
el.onclick ||
|
||
el.getAttribute('role') === 'button' ||
|
||
el.getAttribute('role') === 'menuitem' ||
|
||
el.classList.contains('clickable')) {{
|
||
const r = el.getBoundingClientRect();
|
||
if (r.width > 0 && r.height > 0) {{
|
||
return {{
|
||
found: true,
|
||
x: Math.round(r.left + r.width / 2),
|
||
y: Math.round(r.top + r.height / 2)
|
||
}};
|
||
}}
|
||
}}
|
||
}}
|
||
}}
|
||
return {{ found: false }};
|
||
}}
|
||
''')
|
||
if result.get("found"):
|
||
return (result["x"], result["y"])
|
||
except Exception as e:
|
||
logger.warning(f"DOM 查找失败: {e}")
|
||
|
||
return None
|
||
|
||
def _locate_element(self, target: str) -> Optional[tuple]:
|
||
"""使用 AI 定位元素（备用方法，较慢）"""
|
||
img = self.browser.screenshot_base64()
|
||
viewport = self.browser.page.viewport_size
|
||
width = viewport["width"] if viewport else 1920
|
||
height = viewport["height"] if viewport else 1080
|
||
|
||
prompt = f"""在 {width}x{height} 像素的截图中，找到 "{target}" 的精确中心坐标。
|
||
|
||
返回 JSON: {{"x": 数字, "y": 数字, "found": true}}
|
||
如果找不到: {{"found": false}}
|
||
只返回 JSON。"""
|
||
|
||
response = self.analyzer.model.analyze(img, prompt)
|
||
|
||
try:
|
||
match = re.search(r'\{[\s\S]*?\}', response)
|
||
if match:
|
||
result = json.loads(match.group())
|
||
if result.get("found"):
|
||
return (result["x"], result["y"])
|
||
except:
|
||
pass
|
||
|
||
return None
|
||
|
||
def _detect_bugs(self, action: Dict) -> None:
|
||
"""检测可能的 BUG"""
|
||
# 检测空白页
|
||
# 检测错误提示
|
||
# 检测加载失败
|
||
# TODO: 使用 AI 分析截图检测异常
|
||
pass
|
||
|
||
def _generate_report(self, start_url: str, click_count: int) -> Dict[str, Any]:
|
||
"""生成探索报告"""
|
||
|
||
# 按类型统计元素
|
||
type_stats = {}
|
||
for el in self.discovered_elements:
|
||
t = el.get("type", "unknown")
|
||
type_stats[t] = type_stats.get(t, 0) + 1
|
||
|
||
report = {
|
||
"summary": {
|
||
"start_url": start_url,
|
||
"total_elements": len(self.discovered_elements),
|
||
"total_clicks": click_count,
|
||
"pages_visited": len(self.visited_urls),
|
||
"bugs_found": len(self.bug_list),
|
||
"timestamp": datetime.now().isoformat()
|
||
},
|
||
"elements_by_type": type_stats,
|
||
"discovered_elements": self.discovered_elements,
|
||
"site_map": self.site_map,
|
||
"bug_list": self.bug_list,
|
||
"action_log": self.action_log,
|
||
"visited_urls": list(self.visited_urls)
|
||
}
|
||
|
||
logger.info(f"探索完成: 发现 {len(self.discovered_elements)} 个元素, "
|
||
f"访问 {len(self.visited_urls)} 个页面, "
|
||
f"发现 {len(self.bug_list)} 个问题")
|
||
|
||
return report
|