""" Feature Explorer - AI-driven autonomous feature discovery """ from typing import List, Dict, Any, Set, Optional import json import re import logging from datetime import datetime logger = logging.getLogger(__name__) class FeatureExplorer: """AI 驱动的功能探索器 - 自主发现并记录页面功能""" def __init__(self, browser, analyzer): self.browser = browser self.analyzer = analyzer # 探索状态 self.discovered_elements: List[Dict] = [] self.visited_urls: Set[str] = set() self.site_map: Dict[str, List[str]] = {} # URL -> 子页面列表 self.bug_list: List[Dict] = [] self.action_log: List[Dict] = [] # 默认配置 self.config = { "max_depth": 3, "max_clicks": 30, "skip_patterns": [], # 完全跳过 "dangerous_patterns": ["删除", "移除", "清空", "退出", "注销"], # 记录但不执行 "focus_patterns": [], # 优先探索 } def explore(self, config: Dict = None) -> Dict[str, Any]: """ 执行主动探索 Returns: 探索结果报告 """ if config: self.config.update(config) start_url = self.browser.page.url self.visited_urls.add(start_url) click_count = 0 print(f"🔍 开始探索: {start_url}") print(f" 配置: 最大点击={self.config['max_clicks']}") # 首次发现页面元素 print(f" 正在分析页面元素...") all_elements = self._discover_elements() print(f" 发现 {len(all_elements)} 个可交互元素") if not all_elements: print(" ⚠️ 没有发现可交互元素") return self._generate_report(start_url, 0) # 过滤和排序 elements = self._filter_and_sort(all_elements) print(f" 过滤后 {len(elements)} 个待探索元素") # 探索循环 - 逐个探索发现的元素 for element in elements: if click_count >= self.config["max_clicks"]: print(f" 达到最大点击数 {self.config['max_clicks']}") break click_count += 1 print(f"\n [{click_count}/{min(len(elements), self.config['max_clicks'])}] 探索: {element.get('name', '未知')}") # 执行探索 self._explore_element(element, click_count) print(f"\n✅ 探索完成: {click_count} 次点击") # 生成报告 return self._generate_report(start_url, click_count) def _discover_elements(self) -> List[Dict]: """让 AI 发现页面上所有可交互元素""" img = self.browser.screenshot_base64() current_url = self.browser.page.url prompt = """分析当前页面截图,识别所有可交互的 UI 元素。 **请识别以下类型的元素**: 1. 导航菜单项 2. 侧边栏链接 3. 操作按钮 4. 表单输入框 5. 下拉菜单 6. 标签页/Tab 7. 可点击的卡片或列表项 **返回格式** (只返回 JSON): ```json { "elements": [ { "name": "元素名称/文字", "type": "navigation|button|form|menu|tab|link|card", "description": "功能描述", "priority": 1-10, "is_dangerous": false } ], "page_title": "页面标题", "page_type": "dashboard|list|form|detail|login|other" } ``` 优先级说明: 10=核心功能, 5=普通功能, 1=次要功能""" response = self.analyzer.model.analyze(img, prompt) try: match = re.search(r'\{[\s\S]*\}', response) if match: result = json.loads(match.group()) elements = result.get("elements", []) # 添加元数据 for el in elements: el["source_url"] = current_url el["discovered_at"] = datetime.now().isoformat() logger.info(f"发现 {len(elements)} 个可交互元素") return elements except Exception as e: logger.warning(f"解析元素失败: {e}") return [] def _filter_and_sort(self, elements: List[Dict]) -> List[Dict]: """过滤和排序元素""" filtered = [] for el in elements: name = el.get("name", "") # 跳过已探索的元素 if self._is_explored(el): continue # 完全跳过的模式 if any(p in name for p in self.config["skip_patterns"]): continue # 标记危险元素(记录但不执行) if any(p in name for p in self.config["dangerous_patterns"]): el["is_dangerous"] = True # 优先探索的模式 if any(p in name for p in self.config["focus_patterns"]): el["priority"] = el.get("priority", 5) + 5 filtered.append(el) # 按优先级排序 filtered.sort(key=lambda x: x.get("priority", 5), reverse=True) return filtered def _is_explored(self, element: Dict) -> bool: """检查元素是否已探索""" name = element.get("name", "") return any(d.get("name") == name for d in self.discovered_elements) def _explore_element(self, element: Dict, click_num: int) -> None: """探索单个元素""" name = element.get("name", "未知") el_type = element.get("type", "unknown") is_dangerous = element.get("is_dangerous", False) # 记录为已发现 self.discovered_elements.append(element) # 截图(操作前)- 使用较小的截图减少内存 before_url = self.browser.page.url before_shot = None # 暂不保存截图以加速 action_record = { "step": click_num, "element": element, "before_url": before_url, "action_taken": False, "success": True, "error": None } # 危险元素只记录不执行 if is_dangerous: print(f" ⚠️ 危险操作,仅记录: {name}") action_record["skipped"] = True action_record["skip_reason"] = "危险操作" self.action_log.append(action_record) return # 执行点击 - 使用 DOM 选择器代替 AI 定位(更快) try: coords = self._find_element_by_name(name) if coords: print(f" → 点击 ({coords[0]}, {coords[1]})") self.browser.click_at(coords[0], coords[1]) self.browser.wait(500) action_record["action_taken"] = True action_record["click_coords"] = coords else: print(f" ⚠️ 未找到元素") action_record["success"] = False except Exception as e: print(f" ❌ 点击失败: {e}") action_record["success"] = False action_record["error"] = str(e) self.bug_list.append({ "type": "click_failed", "element": name, "error": str(e), "url": before_url }) # 检查结果 self.browser.wait(300) after_url = self.browser.page.url action_record["after_url"] = after_url action_record["url_changed"] = before_url != after_url # 更新站点地图 if before_url != after_url: if before_url not in self.site_map: self.site_map[before_url] = [] if after_url not in self.site_map[before_url]: self.site_map[before_url].append(after_url) self.visited_urls.add(after_url) print(f" → 跳转到新页面") self.action_log.append(action_record) def _find_element_by_name(self, name: str) -> Optional[tuple]: """通过元素名称/文本查找坐标(使用 DOM 而非 AI,更快)""" try: result = self.browser.page.evaluate(f''' () => {{ // 搜索包含该文本的可点击元素 const text = "{name}"; const clickable = ['A', 'BUTTON', 'INPUT', 'LI', 'SPAN', 'DIV']; // 遍历所有元素 const all = document.querySelectorAll('*'); for (const el of all) {{ if (el.textContent && el.textContent.trim().includes(text)) {{ if (clickable.includes(el.tagName) || el.onclick || el.getAttribute('role') === 'button' || el.getAttribute('role') === 'menuitem' || el.classList.contains('clickable')) {{ const r = el.getBoundingClientRect(); if (r.width > 0 && r.height > 0) {{ return {{ found: true, x: Math.round(r.left + r.width / 2), y: Math.round(r.top + r.height / 2) }}; }} }} }} }} return {{ found: false }}; }} ''') if result.get("found"): return (result["x"], result["y"]) except Exception as e: logger.warning(f"DOM 查找失败: {e}") return None def _locate_element(self, target: str) -> Optional[tuple]: """使用 AI 定位元素(备用方法,较慢)""" img = self.browser.screenshot_base64() viewport = self.browser.page.viewport_size width = viewport["width"] if viewport else 1920 height = viewport["height"] if viewport else 1080 prompt = f"""在 {width}x{height} 像素的截图中,找到 "{target}" 的精确中心坐标。 返回 JSON: {{"x": 数字, "y": 数字, "found": true}} 如果找不到: {{"found": false}} 只返回 JSON。""" response = self.analyzer.model.analyze(img, prompt) try: match = re.search(r'\{[\s\S]*?\}', response) if match: result = json.loads(match.group()) if result.get("found"): return (result["x"], result["y"]) except: pass return None def _detect_bugs(self, action: Dict) -> None: """检测可能的 BUG""" # 检测空白页 # 检测错误提示 # 检测加载失败 # TODO: 使用 AI 分析截图检测异常 pass def _generate_report(self, start_url: str, click_count: int) -> Dict[str, Any]: """生成探索报告""" # 按类型统计元素 type_stats = {} for el in self.discovered_elements: t = el.get("type", "unknown") type_stats[t] = type_stats.get(t, 0) + 1 report = { "summary": { "start_url": start_url, "total_elements": len(self.discovered_elements), "total_clicks": click_count, "pages_visited": len(self.visited_urls), "bugs_found": len(self.bug_list), "timestamp": datetime.now().isoformat() }, "elements_by_type": type_stats, "discovered_elements": self.discovered_elements, "site_map": self.site_map, "bug_list": self.bug_list, "action_log": self.action_log, "visited_urls": list(self.visited_urls) } logger.info(f"探索完成: 发现 {len(self.discovered_elements)} 个元素, " f"访问 {len(self.visited_urls)} 个页面, " f"发现 {len(self.bug_list)} 个问题") return report