改进 _find_element_by_name: - 使用直接文本匹配 + 短文本匹配策略 - 检查 cursor:pointer 样式 - 按元素面积排序,选择最精确匹配 - 限制元素大小避免匹配到容器 - 转义特殊字符防止 JS 注入 坐标定位效果: - 之前: 所有元素 (960, 540) - 现在: 分析概览 (40,105), 系统管理 (40,551), 搜索 (1634,25)
This commit is contained in:
@@ -243,34 +243,68 @@ class FeatureExplorer:
|
|||||||
|
|
||||||
def _find_element_by_name(self, name: str) -> Optional[tuple]:
|
def _find_element_by_name(self, name: str) -> Optional[tuple]:
|
||||||
"""通过元素名称/文本查找坐标(使用 DOM 而非 AI,更快)"""
|
"""通过元素名称/文本查找坐标(使用 DOM 而非 AI,更快)"""
|
||||||
|
# 转义特殊字符
|
||||||
|
escaped_name = name.replace("\\", "\\\\").replace('"', '\\"').replace("'", "\\'")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
result = self.browser.page.evaluate(f'''
|
result = self.browser.page.evaluate(f'''
|
||||||
() => {{
|
() => {{
|
||||||
// 搜索包含该文本的可点击元素
|
const searchText = "{escaped_name}";
|
||||||
const text = "{name}";
|
const clickable = ['A', 'BUTTON', 'INPUT', 'LI', 'SPAN', 'DIV', 'NAV', 'LABEL'];
|
||||||
const clickable = ['A', 'BUTTON', 'INPUT', 'LI', 'SPAN', 'DIV'];
|
|
||||||
|
|
||||||
// 遍历所有元素
|
// 收集所有匹配的元素
|
||||||
|
const matches = [];
|
||||||
const all = document.querySelectorAll('*');
|
const all = document.querySelectorAll('*');
|
||||||
|
|
||||||
for (const el of all) {{
|
for (const el of all) {{
|
||||||
if (el.textContent && el.textContent.trim().includes(text)) {{
|
// 只检查直接文本内容,不包含子元素的文本
|
||||||
if (clickable.includes(el.tagName) ||
|
const directText = Array.from(el.childNodes)
|
||||||
el.onclick ||
|
.filter(n => n.nodeType === 3) // 文本节点
|
||||||
el.getAttribute('role') === 'button' ||
|
.map(n => n.textContent.trim())
|
||||||
el.getAttribute('role') === 'menuitem' ||
|
.join('');
|
||||||
el.classList.contains('clickable')) {{
|
|
||||||
const r = el.getBoundingClientRect();
|
// 或者元素的完整文本很短(小于搜索文本的2倍)
|
||||||
if (r.width > 0 && r.height > 0) {{
|
const fullText = el.textContent?.trim() || '';
|
||||||
return {{
|
|
||||||
found: true,
|
// 匹配条件
|
||||||
x: Math.round(r.left + r.width / 2),
|
const hasDirectMatch = directText.includes(searchText);
|
||||||
y: Math.round(r.top + r.height / 2)
|
const hasShortMatch = fullText.includes(searchText) && fullText.length < searchText.length * 3;
|
||||||
}};
|
|
||||||
}}
|
if ((hasDirectMatch || hasShortMatch) &&
|
||||||
|
(clickable.includes(el.tagName) ||
|
||||||
|
el.onclick ||
|
||||||
|
el.getAttribute('role') === 'button' ||
|
||||||
|
el.getAttribute('role') === 'menuitem' ||
|
||||||
|
el.getAttribute('role') === 'link' ||
|
||||||
|
window.getComputedStyle(el).cursor === 'pointer')) {{
|
||||||
|
|
||||||
|
const r = el.getBoundingClientRect();
|
||||||
|
if (r.width > 0 && r.height > 0 && r.width < 800 && r.height < 200) {{
|
||||||
|
matches.push({{
|
||||||
|
el: el,
|
||||||
|
rect: r,
|
||||||
|
textLen: fullText.length,
|
||||||
|
area: r.width * r.height
|
||||||
|
}});
|
||||||
}}
|
}}
|
||||||
}}
|
}}
|
||||||
}}
|
}}
|
||||||
return {{ found: false }};
|
|
||||||
|
if (matches.length === 0) {{
|
||||||
|
return {{ found: false }};
|
||||||
|
}}
|
||||||
|
|
||||||
|
// 选择面积最小的匹配元素(最精确)
|
||||||
|
matches.sort((a, b) => a.area - b.area);
|
||||||
|
const best = matches[0];
|
||||||
|
|
||||||
|
return {{
|
||||||
|
found: true,
|
||||||
|
x: Math.round(best.rect.left + best.rect.width / 2),
|
||||||
|
y: Math.round(best.rect.top + best.rect.height / 2),
|
||||||
|
tagName: best.el.tagName,
|
||||||
|
text: best.el.textContent.substring(0, 50)
|
||||||
|
}};
|
||||||
}}
|
}}
|
||||||
''')
|
''')
|
||||||
if result.get("found"):
|
if result.get("found"):
|
||||||
|
|||||||
Reference in New Issue
Block a user