diff --git a/src/agent/explorer.py b/src/agent/explorer.py index c81e367..571d2f0 100644 --- a/src/agent/explorer.py +++ b/src/agent/explorer.py @@ -243,34 +243,68 @@ class FeatureExplorer: def _find_element_by_name(self, name: str) -> Optional[tuple]: """通过元素名称/文本查找坐标(使用 DOM 而非 AI,更快)""" + # 转义特殊字符 + escaped_name = name.replace("\\", "\\\\").replace('"', '\\"').replace("'", "\\'") + try: result = self.browser.page.evaluate(f''' () => {{ - // 搜索包含该文本的可点击元素 - const text = "{name}"; - const clickable = ['A', 'BUTTON', 'INPUT', 'LI', 'SPAN', 'DIV']; + const searchText = "{escaped_name}"; + const clickable = ['A', 'BUTTON', 'INPUT', 'LI', 'SPAN', 'DIV', 'NAV', 'LABEL']; - // 遍历所有元素 + // 收集所有匹配的元素 + const matches = []; const all = document.querySelectorAll('*'); + for (const el of all) {{ - if (el.textContent && el.textContent.trim().includes(text)) {{ - if (clickable.includes(el.tagName) || - el.onclick || - el.getAttribute('role') === 'button' || - el.getAttribute('role') === 'menuitem' || - el.classList.contains('clickable')) {{ - const r = el.getBoundingClientRect(); - if (r.width > 0 && r.height > 0) {{ - return {{ - found: true, - x: Math.round(r.left + r.width / 2), - y: Math.round(r.top + r.height / 2) - }}; - }} + // 只检查直接文本内容,不包含子元素的文本 + const directText = Array.from(el.childNodes) + .filter(n => n.nodeType === 3) // 文本节点 + .map(n => n.textContent.trim()) + .join(''); + + // 或者元素的完整文本很短(小于搜索文本的2倍) + const fullText = el.textContent?.trim() || ''; + + // 匹配条件 + const hasDirectMatch = directText.includes(searchText); + const hasShortMatch = fullText.includes(searchText) && fullText.length < searchText.length * 3; + + if ((hasDirectMatch || hasShortMatch) && + (clickable.includes(el.tagName) || + el.onclick || + el.getAttribute('role') === 'button' || + el.getAttribute('role') === 'menuitem' || + el.getAttribute('role') === 'link' || + window.getComputedStyle(el).cursor === 'pointer')) {{ + + const r = el.getBoundingClientRect(); + if (r.width > 0 && r.height > 0 && r.width < 800 && r.height < 200) {{ + matches.push({{ + el: el, + rect: r, + textLen: fullText.length, + area: r.width * r.height + }}); }} }} }} - return {{ found: false }}; + + if (matches.length === 0) {{ + return {{ found: false }}; + }} + + // 选择面积最小的匹配元素(最精确) + matches.sort((a, b) => a.area - b.area); + const best = matches[0]; + + return {{ + found: true, + x: Math.round(best.rect.left + best.rect.width / 2), + y: Math.round(best.rect.top + best.rect.height / 2), + tagName: best.el.tagName, + text: best.el.textContent.substring(0, 50) + }}; }} ''') if result.get("found"):