fix: 优化探索器元素定位精度
Some checks failed
AI Web Tester CI / test (push) Has been cancelled

改进 _find_element_by_name:
- 使用直接文本匹配 + 短文本匹配策略
- 检查 cursor:pointer 样式
- 按元素面积排序,选择最精确匹配
- 限制元素大小避免匹配到容器
- 转义特殊字符防止 JS 注入

坐标定位效果:
- 之前: 所有元素 (960, 540)
- 现在: 分析概览 (40,105), 系统管理 (40,551), 搜索 (1634,25)
This commit is contained in:
empty
2025-12-28 20:47:43 +08:00
parent c6def51435
commit b126ce2d49

View File

@@ -243,34 +243,68 @@ class FeatureExplorer:
def _find_element_by_name(self, name: str) -> Optional[tuple]: def _find_element_by_name(self, name: str) -> Optional[tuple]:
"""通过元素名称/文本查找坐标(使用 DOM 而非 AI更快""" """通过元素名称/文本查找坐标(使用 DOM 而非 AI更快"""
# 转义特殊字符
escaped_name = name.replace("\\", "\\\\").replace('"', '\\"').replace("'", "\\'")
try: try:
result = self.browser.page.evaluate(f''' result = self.browser.page.evaluate(f'''
() => {{ () => {{
// 搜索包含该文本的可点击元素 const searchText = "{escaped_name}";
const text = "{name}"; const clickable = ['A', 'BUTTON', 'INPUT', 'LI', 'SPAN', 'DIV', 'NAV', 'LABEL'];
const clickable = ['A', 'BUTTON', 'INPUT', 'LI', 'SPAN', 'DIV'];
// 遍历所有元素 // 收集所有匹配的元素
const matches = [];
const all = document.querySelectorAll('*'); const all = document.querySelectorAll('*');
for (const el of all) {{ for (const el of all) {{
if (el.textContent && el.textContent.trim().includes(text)) {{ // 只检查直接文本内容,不包含子元素的文本
if (clickable.includes(el.tagName) || const directText = Array.from(el.childNodes)
el.onclick || .filter(n => n.nodeType === 3) // 文本节点
el.getAttribute('role') === 'button' || .map(n => n.textContent.trim())
el.getAttribute('role') === 'menuitem' || .join('');
el.classList.contains('clickable')) {{
const r = el.getBoundingClientRect(); // 或者元素的完整文本很短小于搜索文本的2倍
if (r.width > 0 && r.height > 0) {{ const fullText = el.textContent?.trim() || '';
return {{
found: true, // 匹配条件
x: Math.round(r.left + r.width / 2), const hasDirectMatch = directText.includes(searchText);
y: Math.round(r.top + r.height / 2) const hasShortMatch = fullText.includes(searchText) && fullText.length < searchText.length * 3;
}};
}} if ((hasDirectMatch || hasShortMatch) &&
(clickable.includes(el.tagName) ||
el.onclick ||
el.getAttribute('role') === 'button' ||
el.getAttribute('role') === 'menuitem' ||
el.getAttribute('role') === 'link' ||
window.getComputedStyle(el).cursor === 'pointer')) {{
const r = el.getBoundingClientRect();
if (r.width > 0 && r.height > 0 && r.width < 800 && r.height < 200) {{
matches.push({{
el: el,
rect: r,
textLen: fullText.length,
area: r.width * r.height
}});
}} }}
}} }}
}} }}
return {{ found: false }};
if (matches.length === 0) {{
return {{ found: false }};
}}
// 选择面积最小的匹配元素(最精确)
matches.sort((a, b) => a.area - b.area);
const best = matches[0];
return {{
found: true,
x: Math.round(best.rect.left + best.rect.width / 2),
y: Math.round(best.rect.top + best.rect.height / 2),
tagName: best.el.tagName,
text: best.el.textContent.substring(0, 50)
}};
}} }}
''') ''')
if result.get("found"): if result.get("found"):