fix: 优化探索器元素定位精度
Some checks failed
AI Web Tester CI / test (push) Has been cancelled

改进 _find_element_by_name:
- 使用直接文本匹配 + 短文本匹配策略
- 检查 cursor:pointer 样式
- 按元素面积排序,选择最精确匹配
- 限制元素大小避免匹配到容器
- 转义特殊字符防止 JS 注入

坐标定位效果:
- 之前: 所有元素 (960, 540)
- 现在: 分析概览 (40,105), 系统管理 (40,551), 搜索 (1634,25)
This commit is contained in:
empty
2025-12-28 20:47:43 +08:00
parent c6def51435
commit b126ce2d49

View File

@@ -243,34 +243,68 @@ class FeatureExplorer:
def _find_element_by_name(self, name: str) -> Optional[tuple]:
"""通过元素名称/文本查找坐标(使用 DOM 而非 AI更快"""
# 转义特殊字符
escaped_name = name.replace("\\", "\\\\").replace('"', '\\"').replace("'", "\\'")
try:
result = self.browser.page.evaluate(f'''
() => {{
// 搜索包含该文本的可点击元素
const text = "{name}";
const clickable = ['A', 'BUTTON', 'INPUT', 'LI', 'SPAN', 'DIV'];
const searchText = "{escaped_name}";
const clickable = ['A', 'BUTTON', 'INPUT', 'LI', 'SPAN', 'DIV', 'NAV', 'LABEL'];
// 遍历所有元素
// 收集所有匹配的元素
const matches = [];
const all = document.querySelectorAll('*');
for (const el of all) {{
if (el.textContent && el.textContent.trim().includes(text)) {{
if (clickable.includes(el.tagName) ||
el.onclick ||
el.getAttribute('role') === 'button' ||
el.getAttribute('role') === 'menuitem' ||
el.classList.contains('clickable')) {{
const r = el.getBoundingClientRect();
if (r.width > 0 && r.height > 0) {{
return {{
found: true,
x: Math.round(r.left + r.width / 2),
y: Math.round(r.top + r.height / 2)
}};
}}
// 只检查直接文本内容,不包含子元素的文本
const directText = Array.from(el.childNodes)
.filter(n => n.nodeType === 3) // 文本节点
.map(n => n.textContent.trim())
.join('');
// 或者元素的完整文本很短小于搜索文本的2倍
const fullText = el.textContent?.trim() || '';
// 匹配条件
const hasDirectMatch = directText.includes(searchText);
const hasShortMatch = fullText.includes(searchText) && fullText.length < searchText.length * 3;
if ((hasDirectMatch || hasShortMatch) &&
(clickable.includes(el.tagName) ||
el.onclick ||
el.getAttribute('role') === 'button' ||
el.getAttribute('role') === 'menuitem' ||
el.getAttribute('role') === 'link' ||
window.getComputedStyle(el).cursor === 'pointer')) {{
const r = el.getBoundingClientRect();
if (r.width > 0 && r.height > 0 && r.width < 800 && r.height < 200) {{
matches.push({{
el: el,
rect: r,
textLen: fullText.length,
area: r.width * r.height
}});
}}
}}
}}
return {{ found: false }};
if (matches.length === 0) {{
return {{ found: false }};
}}
// 选择面积最小的匹配元素(最精确)
matches.sort((a, b) => a.area - b.area);
const best = matches[0];
return {{
found: true,
x: Math.round(best.rect.left + best.rect.width / 2),
y: Math.round(best.rect.top + best.rect.height / 2),
tagName: best.el.tagName,
text: best.el.textContent.substring(0, 50)
}};
}}
''')
if result.get("found"):