Files
hermes-agent/tools/browser_camofox.py

639 lines
28 KiB
Python

"""Camofox browser backend — local anti-detection browser via REST API.
Camofox-browser (https://github.com/jo-inc/camofox-browser) is a self-hosted
Node.js server wrapping Camoufox (Firefox fork with C++ fingerprint spoofing).
Its REST API maps 1:1 to our browser tool interface: accessibility snapshots
with element refs, click/type/scroll by ref, screenshots.
Setup: ``npm install && npm start`` in a camofox-browser checkout, or
``docker run -p 9377:9377 -e CAMOFOX_PORT=9377 jo-inc/camofox-browser``; then set
``CAMOFOX_URL=http://localhost:9377`` in ``~/.hermes/.env`` (Docker: see
``CAMOFOX_REWRITE_LOOPBACK_URLS`` in :func:`_rewrite_loopback_url_for_camofox`).
"""
from __future__ import annotations
import base64
import ipaddress
import json
import logging
import os
import re
import threading
import uuid
from typing import Any, Callable, Dict, Optional
from urllib.parse import SplitResult, urlsplit, urlunsplit
import requests
from agent.secret_scope import get_secret
from hermes_cli.config import cfg_get, load_config, read_raw_config
from tools.browser_camofox_state import get_camofox_identity
from tools.registry import tool_error
logger = logging.getLogger(__name__)
# ---- Configuration ----
_DEFAULT_TIMEOUT = 30 # fallback when config is unreadable
_NO_SESSION_ERROR = "No browser session. Call browser_navigate first."
_vnc_url: Optional[str] = None # cached from /health response
_vnc_url_checked = False # only probe once per process
# Cached command timeout from config (resolved lazily, like browser_tool)
_cached_cmd_timeout: Optional[int] = None
_cmd_timeout_resolved = False
def _get_command_timeout() -> int:
"""``browser.command_timeout`` (floor 5s, default 30s), cached after first read;
mirrors :func:`tools.browser_tool._get_command_timeout`."""
global _cached_cmd_timeout, _cmd_timeout_resolved
if _cmd_timeout_resolved:
return _cached_cmd_timeout # type: ignore[return-value]
_cmd_timeout_resolved = True
result = _DEFAULT_TIMEOUT
try:
val = cfg_get(read_raw_config(), "browser", "command_timeout")
if val is not None:
result = max(int(val), 5)
except Exception as exc:
logger.debug("Could not read browser.command_timeout: %s", exc)
_cached_cmd_timeout = result
return result
def _auth_headers() -> Dict[str, str]:
"""Return Authorization header when CAMOFOX_API_KEY is set."""
key = (get_secret("CAMOFOX_API_KEY", "") or "").strip()
return {"Authorization": f"Bearer {key}"} if key else {}
def get_camofox_url() -> str:
"""Return the configured Camofox server URL, or empty string."""
return (get_secret("CAMOFOX_URL", "") or "").rstrip("/")
def _config_cdp_url() -> str:
"""Persistent ``browser.cdp_url`` from config.yaml, or "". Read here rather than
via ``browser_tool._get_cdp_override`` (circular import)."""
try:
from hermes_cli.config import read_raw_config # late-bound: tests patch the source module
browser_cfg = read_raw_config().get("browser", {})
if isinstance(browser_cfg, dict):
return str(browser_cfg.get("cdp_url", "") or "").strip()
except Exception:
pass
return ""
def is_camofox_mode() -> bool:
"""True when the Camofox backend is selected and no CDP override is active.
Selection is ``browser.cloud_provider: camofox``; ``CAMOFOX_URL`` is only the
address and never overrides a different stored selection (legacy: with no
selection ever written, a set ``CAMOFOX_URL`` still activates Camofox). A CDP
override (``BROWSER_CDP_URL`` env or ``browser.cdp_url`` config, same precedence
as ``browser_tool._get_cdp_override()``) wins so tools drive the real CDP browser.
"""
if os.getenv("BROWSER_CDP_URL", "").strip() or _config_cdp_url():
return False
try:
from tools.tool_backend_helpers import read_selection
selected = read_selection("browser")
except Exception: # pragma: no cover — helpers are in-repo
selected = None
if selected is not None:
return selected == "camofox"
return bool(get_camofox_url())
def check_camofox_available() -> bool:
"""Verify the Camofox server is reachable (and cache its VNC URL once)."""
global _vnc_url, _vnc_url_checked
url = get_camofox_url()
if not url:
return False
try:
resp = requests.get(f"{url}/health", timeout=5)
if resp.status_code == 200 and not _vnc_url_checked:
try:
vnc_port = resp.json().get("vncPort")
if isinstance(vnc_port, int) and 1 <= vnc_port <= 65535:
_vnc_url = f"http://{urlsplit(url).hostname or 'localhost'}:{vnc_port}"
except (ValueError, KeyError):
pass
_vnc_url_checked = True
return resp.status_code == 200
except Exception:
return False
def get_vnc_url() -> Optional[str]:
"""Return the VNC URL if the Camofox server exposes one, or None."""
if not _vnc_url_checked:
check_camofox_available()
return _vnc_url
def _get_camofox_config() -> Dict[str, Any]:
"""Return the ``browser.camofox`` config block, or an empty dict."""
try:
camofox_cfg = load_config().get("browser", {}).get("camofox", {})
except Exception as exc:
logger.warning("camofox config check failed, defaulting to disabled: %s", exc)
return {}
return camofox_cfg if isinstance(camofox_cfg, dict) else {}
def _managed_persistence_enabled(camofox_cfg: Optional[Dict[str, Any]] = None) -> bool:
"""``browser.camofox.managed_persistence``: stable profile-scoped userId vs random per session."""
if camofox_cfg is None:
camofox_cfg = _get_camofox_config()
return bool(camofox_cfg.get("managed_persistence"))
def _secret_or_cfg(secret_name: str, camofox_cfg: Dict[str, Any], cfg_key: str) -> str:
"""Secret-scope value first, then the ``browser.camofox`` config key, else ""."""
return (get_secret(secret_name, "") or "").strip() or str(camofox_cfg.get(cfg_key) or "").strip()
def _camofox_identity_override(task_id: Optional[str], camofox_cfg: Dict[str, Any]) -> Optional[Dict[str, str]]:
"""Externally configured identity (integrations owning the visible Camofox
browser share a user ID so Hermes uses the same profile), or None."""
user_id = _secret_or_cfg("CAMOFOX_USER_ID", camofox_cfg, "user_id")
if not user_id:
return None
session_key = _secret_or_cfg("CAMOFOX_SESSION_KEY", camofox_cfg, "session_key")
return {"user_id": user_id, "session_key": session_key or f"task_{(task_id or 'default')[:16]}"}
def _flag(env_name: str, camofox_cfg: Dict[str, Any], cfg_key: str) -> bool:
"""Boolean toggle: env var wins when set to a valid value, else config key."""
raw = os.getenv(env_name, "").strip().lower()
if raw in {"1", "true", "yes", "on"}:
return True
if raw in {"0", "false", "no", "off"}:
return False
if raw:
logger.debug("Ignoring invalid boolean env %s=%r", env_name, raw)
return bool(camofox_cfg.get(cfg_key))
def _loopback_rewrite_host(camofox_cfg: Dict[str, Any]) -> str:
"""Return the host alias used when rewriting loopback page URLs."""
return (os.getenv("CAMOFOX_LOOPBACK_HOST_ALIAS", "").strip()
or str(camofox_cfg.get("loopback_host_alias") or "").strip() or "host.docker.internal")
def _is_loopback_hostname(hostname: Optional[str]) -> bool:
"""Return True for localhost/127.0.0.0/8/::1-style hostnames."""
if not hostname:
return False
host = hostname.strip().strip("[]").lower()
if host in {"localhost", "localhost.localdomain"}:
return True
try:
return ipaddress.ip_address(host).is_loopback
except ValueError:
return False
def _rewrite_loopback_url_for_camofox(url: str) -> tuple[str, Optional[Dict[str, str]]]:
"""Rewrite loopback page URLs for Docker-hosted Camofox, if configured.
``CAMOFOX_URL`` may point at a host-published Docker port, but page URLs are
opened by the browser *inside* the container, where loopback is the container,
not the host. Opt-in (``CAMOFOX_REWRITE_LOOPBACK_URLS`` / config) because
non-Docker installs run the browser on the host. Returns ``(rewritten_url,
metadata)``; ``metadata`` is present only when a rewrite happened so the tool
result can disclose the change to the model.
"""
camofox_cfg = _get_camofox_config()
if not _flag("CAMOFOX_REWRITE_LOOPBACK_URLS", camofox_cfg, "rewrite_loopback_urls"):
return url, None
try:
parsed = urlsplit(url)
except ValueError:
return url, None
alias = _loopback_rewrite_host(camofox_cfg)
if parsed.scheme not in {"http", "https"} or not _is_loopback_hostname(parsed.hostname) or not alias:
return url, None
userinfo = ""
if parsed.username:
userinfo = parsed.username + (f":{parsed.password}" if parsed.password else "") + "@"
host_part = f"[{alias}]" if ":" in alias and not alias.startswith("[") else alias
port_part = f":{parsed.port}" if parsed.port else ""
rewritten = urlunsplit(
SplitResult(parsed.scheme, f"{userinfo}{host_part}{port_part}", parsed.path, parsed.query, parsed.fragment)
)
return rewritten, {"from": parsed.hostname or "", "to": alias, "original_url": url, "rewritten_url": rewritten}
# ---- Session management ----
# Maps task_id -> {"user_id": str, "tab_id": str|None, ...}
_sessions: Dict[str, Dict[str, Any]] = {}
_sessions_lock = threading.Lock()
def _adopt_existing_tab(session: Dict[str, Any]) -> Dict[str, Any]:
"""Rehydrate tab_id from an already-open managed tab: gateway restarts empty the
in-memory cache while Camofox still holds the integration-owned tab."""
if session.get("tab_id") or not session.get("adopt_existing_tab") or not get_camofox_url():
return session
try:
tabs = _get("/tabs", params=_user_params(session), timeout=5).get("tabs", [])
except Exception as exc:
logger.debug("Camofox tab adoption failed for %s: %s", session.get("user_id"), exc)
return session
if not isinstance(tabs, list):
return session
dict_tabs = [tab for tab in tabs if isinstance(tab, dict)]
candidates = [tab for tab in dict_tabs if tab.get("listItemId") == session.get("session_key")] or dict_tabs
tab_id = candidates[-1].get("tabId") if candidates else None
if isinstance(tab_id, str) and tab_id:
session["tab_id"] = tab_id
logger.debug("Adopted existing Camofox tab %s for %s", tab_id, session.get("user_id"))
return session
def _get_session(task_id: Optional[str]) -> Dict[str, Any]:
"""Get or create the task's session. Identity precedence: external override
(CAMOFOX_USER_ID / config) → profile-scoped identity when managed persistence
is on → random ephemeral userId."""
task_id = task_id or "default"
with _sessions_lock:
if task_id in _sessions:
return _adopt_existing_tab(_sessions[task_id])
camofox_cfg = _get_camofox_config()
identity = _camofox_identity_override(task_id, camofox_cfg)
if identity is None and _managed_persistence_enabled(camofox_cfg):
identity = get_camofox_identity(task_id)
if identity is None:
identity = {"user_id": f"hermes_{uuid.uuid4().hex[:10]}", "session_key": f"task_{task_id[:16]}"}
managed, adopt = False, False
else:
managed, adopt = True, _flag("CAMOFOX_ADOPT_EXISTING_TAB", camofox_cfg, "adopt_existing_tab")
session = {"user_id": identity["user_id"], "tab_id": None, "session_key": identity["session_key"],
"managed": managed, "adopt_existing_tab": adopt}
_sessions[task_id] = session
return _adopt_existing_tab(session)
def _ensure_tab(task_id: Optional[str], url: str = "about:blank") -> Dict[str, Any]:
"""Ensure a tab exists for the session, creating one if needed."""
session = _get_session(task_id)
if session["tab_id"]:
return session
data = _post("/tabs", {"userId": session["user_id"], "listItemId": session["session_key"], "url": url})
session["tab_id"] = data.get("tabId")
return session
def _drop_session(task_id: Optional[str]) -> Optional[Dict[str, Any]]:
"""Remove and return session info."""
with _sessions_lock:
return _sessions.pop(task_id or "default", None)
def camofox_soft_cleanup(task_id: Optional[str] = None) -> bool:
"""Drop only the local tracking entry (``True``) for managed profiles, which
must survive across agent tasks; ``False`` for ephemeral sessions so the caller
falls back to :func:`camofox_close`."""
camofox_cfg = _get_camofox_config()
if _managed_persistence_enabled(camofox_cfg) or _camofox_identity_override(task_id, camofox_cfg):
_drop_session(task_id)
logger.debug("Camofox soft cleanup for task %s (managed persistence)", task_id)
return True
return False
# ---- HTTP helpers ----
def _request(method: str, path: str, timeout: Optional[int] = None, **kwargs: Any) -> requests.Response:
"""Issue an authenticated request to camofox and return the raised-for-status response."""
if timeout is None:
timeout = _get_command_timeout()
resp = getattr(requests, method)(f"{get_camofox_url()}{path}", timeout=timeout, headers=_auth_headers(), **kwargs)
resp.raise_for_status()
return resp
def _post(path: str, body: dict, timeout: Optional[int] = None) -> dict:
"""POST JSON to camofox and return parsed response."""
return _request("post", path, timeout, json=body).json()
def _get(path: str, params: dict = None, timeout: Optional[int] = None) -> dict:
"""GET from camofox and return parsed response."""
return _request("get", path, timeout, params=params).json()
def _get_raw(path: str, params: dict = None, timeout: Optional[int] = None) -> requests.Response:
"""GET from camofox and return raw response (for binary data)."""
return _request("get", path, timeout, params=params)
def _delete(path: str, body: dict = None, timeout: Optional[int] = None) -> dict:
"""DELETE to camofox and return parsed response."""
return _request("delete", path, timeout, json=body).json()
# ---- Tool implementations ----
def _tab_path(session: Dict[str, Any], suffix: str) -> str:
return f"/tabs/{session['tab_id']}/{suffix}"
def _user_params(session: Dict[str, Any]) -> Dict[str, str]:
return {"userId": session["user_id"]}
def _raw_snapshot(session: Dict[str, Any]) -> str:
return _get(_tab_path(session, "snapshot"), params=_user_params(session)).get("snapshot", "")
def _parse_snapshot_images(snapshot: str) -> list[Dict[str, str]]:
"""Images from an accessibility snapshot: ``img "alt" [eN]`` entries with the
URL on the following ``/url:`` line (Camofox has no /images endpoint)."""
images = []
lines = snapshot.split("\n")
for i, line in enumerate(lines):
stripped = line.strip()
if not stripped.startswith(("- img ", "img ")):
continue
alt_match = re.search(r'img\s+"([^"]*)"', stripped)
alt = alt_match.group(1) if alt_match else ""
url_match = re.search(r'/url:\s*(\S+)', lines[i + 1].strip()) if i + 1 < len(lines) else None
src = url_match.group(1) if url_match else ""
if alt or src:
images.append({"src": src, "alt": alt})
return images
def _fetch_snapshot(session: Dict[str, Any]) -> tuple[str, int]:
"""``(snapshot_text, refs_count)`` truncated like the main browser tool (line
boundaries, full tree stored to cache/web, read_file pointer appended).
``browser_tool`` imports this module, so import lazily."""
data = _get(_tab_path(session, "snapshot"), params=_user_params(session))
snapshot = data.get("snapshot", "")
from tools.browser_tool import _truncate_snapshot, get_browser_snapshot_threshold
threshold = get_browser_snapshot_threshold()
if len(snapshot) > threshold:
snapshot = _truncate_snapshot(snapshot, max_chars=threshold)
return snapshot, data.get("refsCount", 0)
def _navigate_tab(task_id: Optional[str], browser_url: str) -> tuple[Dict[str, Any], dict]:
"""Open ``browser_url`` in the task's tab (creating it if missing) and return
``(session, navigate_response)``. A 404 on the existing tab means the server
garbage-collected it — recreate instead of failing."""
session = _get_session(task_id)
if session["tab_id"]:
try:
data = _post(_tab_path(session, "navigate"), {"userId": session["user_id"], "url": browser_url}, timeout=60)
return session, data
except requests.HTTPError as e:
if e.response is None or e.response.status_code != 404:
raise
logger.warning(
"Camofox tab %s returned 404 — tab was garbage collected. Creating a fresh tab.",
session["tab_id"],
)
session["tab_id"] = None
session = _ensure_tab(task_id, browser_url)
return session, {"ok": True, "url": browser_url}
def camofox_navigate(url: str, task_id: Optional[str] = None) -> str:
"""Navigate to a URL via Camofox."""
try:
browser_url, rewrite_info = _rewrite_loopback_url_for_camofox(url)
session, data = _navigate_tab(task_id, browser_url)
result = {"success": True, "url": data.get("url", browser_url), "title": data.get("title", "")}
if rewrite_info:
result["requested_url"] = url
result["url_rewrite"] = rewrite_info
result["warning"] = ("Rewrote loopback URL for Docker-hosted Camofox: "
f"{rewrite_info['from']} -> {rewrite_info['to']}")
vnc = get_vnc_url()
if vnc:
result["vnc_url"] = vnc
result["vnc_hint"] = ("Browser is visible via VNC. "
"Share this link with the user so they can watch the browser live.")
# Auto-take a compact snapshot so the model can act immediately.
try:
result["snapshot"], result["element_count"] = _fetch_snapshot(session)
except Exception:
pass # Navigation succeeded; snapshot is a bonus
return json.dumps(result)
except requests.HTTPError as e:
return tool_error(f"Navigation failed: {e}", success=False)
except requests.ConnectionError:
return json.dumps({"success": False, "error": (
f"Cannot connect to Camofox at {get_camofox_url()}. "
"Is the server running? Start with: npm start (in camofox-browser dir) "
"or: docker run -p 9377:9377 -e CAMOFOX_PORT=9377 jo-inc/camofox-browser")})
except Exception as e:
return tool_error(str(e), success=False)
def _camofox_private_page_block(session: Dict[str, Any], task_id: Optional[str], action: str) -> Optional[str]:
"""Blocked payload when the current page is private/internal, else None.
Mirrors the ``_camofox_eval`` guard in browser_tool.py: page-state reads on a
non-local backend can leak an intranet/metadata page the terminal can't reach.
Only active when the SSRF guard applies (non-local backend, not a local sidecar,
``allow_private_urls`` unset); fail-open on probe failure like sibling guards.
``browser_tool`` imports this module, so import lazily.
"""
from tools.browser_tool import _camofox_current_page_private_url, _eval_ssrf_guard_active
if not _eval_ssrf_guard_active(task_id or "default"):
return None
blocked_url = _camofox_current_page_private_url(session["tab_id"], session["user_id"])
if not blocked_url:
return None
return json.dumps({"success": False, "error": (
"Blocked: page URL targets a private or internal address "
f"({blocked_url}). Refusing to {action} on this page in this browser mode.")}, ensure_ascii=False)
def _require_tab(task_id: Optional[str], action: Optional[str] = None) -> tuple[Dict[str, Any], Optional[str]]:
"""Return ``(session, error_payload)``: error when no tab exists or, if ``action`` given, the page is private."""
session = _get_session(task_id)
if not session["tab_id"]:
return session, tool_error(_NO_SESSION_ERROR, success=False)
if action is not None:
return session, _camofox_private_page_block(session, task_id, action)
return session, None
def camofox_snapshot(full: bool = False, task_id: Optional[str] = None, user_task: Optional[str] = None) -> str:
"""Accessibility tree snapshot. ``user_task`` is deprecated and ignored —
oversized snapshots always truncate-and-store (no LLM summarization)."""
try:
session, blocked = _require_tab(task_id, "read a page snapshot")
if blocked:
return blocked
snapshot, refs_count = _fetch_snapshot(session)
return json.dumps({"success": True, "snapshot": snapshot, "element_count": refs_count})
except Exception as e:
return tool_error(str(e), success=False)
def _tab_action(task_id: Optional[str], guard_action: Optional[str], suffix: str,
body: Dict[str, Any], result: Callable[[dict], dict]) -> str:
"""Simple tab action: require a tab (+ private-page guard when ``guard_action``
is set), POST ``body`` to ``/tabs/<id>/<suffix>``, build the result."""
try:
session, blocked = _require_tab(task_id, guard_action)
if blocked:
return blocked
data = _post(_tab_path(session, suffix), {"userId": session["user_id"], **body})
return json.dumps(result(data))
except Exception as e:
return tool_error(str(e), success=False)
def camofox_click(ref: str, task_id: Optional[str] = None) -> str:
"""Click an element by ref via Camofox."""
clean_ref = ref.lstrip("@") # our tool convention prefixes refs with @
return _tab_action(task_id, "click", "click", {"ref": clean_ref},
lambda data: {"success": True, "clicked": clean_ref, "url": data.get("url", "")})
def camofox_type(ref: str, text: str, task_id: Optional[str] = None) -> str:
"""Type text into an element by ref via Camofox."""
try:
session, blocked = _require_tab(task_id, "type")
if blocked:
return blocked
clean_ref = ref.lstrip("@")
_post(_tab_path(session, "type"), {"userId": session["user_id"], "ref": clean_ref, "text": text})
from agent.display import redact_browser_typed_text_for_display, redact_tool_args_for_display
# Match browser_tool.browser_type: the raw text is typed into the page, but
# the returned display value is run through the secret-pattern redactor so
# API keys / tokens don't leak into tool progress or chat history.
display_text = (redact_tool_args_for_display("browser_type", {"text": text}) or {})["text"]
response = {"success": True, "typed": display_text, "element": clean_ref}
return json.dumps(redact_browser_typed_text_for_display(response, text))
except Exception as e:
from agent.display import redact_browser_typed_text_for_display
return tool_error(redact_browser_typed_text_for_display(str(e), text), success=False)
def camofox_scroll(direction: str, task_id: Optional[str] = None) -> str:
"""Scroll the page via Camofox."""
return _tab_action(task_id, None, "scroll", {"direction": direction},
lambda data: {"success": True, "scrolled": direction})
def camofox_back(task_id: Optional[str] = None) -> str:
"""Navigate back via Camofox."""
return _tab_action(task_id, None, "back", {}, lambda data: {"success": True, "url": data.get("url", "")})
def camofox_press(key: str, task_id: Optional[str] = None) -> str:
"""Press a keyboard key via Camofox."""
return _tab_action(task_id, "press", "press", {"key": key}, lambda data: {"success": True, "pressed": key})
def camofox_close(task_id: Optional[str] = None) -> str:
"""Close the browser session via Camofox."""
try:
session = _drop_session(task_id)
if session:
_delete(f"/sessions/{session['user_id']}")
return json.dumps({"success": True, "closed": True})
except Exception as e:
return json.dumps({"success": True, "closed": True, "warning": str(e)})
def camofox_get_images(task_id: Optional[str] = None) -> str:
"""Get images on the current page via Camofox (parsed from the snapshot)."""
try:
session, blocked = _require_tab(task_id, "extract page images")
if blocked:
return blocked
images = _parse_snapshot_images(_raw_snapshot(session))
return json.dumps({"success": True, "images": images, "count": len(images)})
except Exception as e:
return tool_error(str(e), success=False)
def _vision_llm_settings() -> tuple[float, float]:
"""``auxiliary.vision`` ``(timeout, temperature)``; defaults 120s / 0.1 on any config error."""
try:
cfg = cfg_get(load_config(), "auxiliary", "vision", default={})
return float(cfg.get("timeout", 120)), float(cfg.get("temperature", 0.1))
except Exception:
return 120.0, 0.1
def _save_screenshot(content: bytes) -> str:
"""Write PNG bytes under ``$HERMES_HOME/browser_screenshots`` and return the path."""
from hermes_constants import get_hermes_home
screenshots_dir = get_hermes_home() / "browser_screenshots"
screenshots_dir.mkdir(parents=True, exist_ok=True)
screenshot_path = str(screenshots_dir / f"browser_screenshot_{uuid.uuid4().hex[:8]}.png")
with open(screenshot_path, "wb") as f:
f.write(content)
return screenshot_path
def camofox_vision(question: str, annotate: bool = False, task_id: Optional[str] = None) -> str:
"""Take a screenshot and analyze it with vision AI via Camofox."""
try:
session, blocked = _require_tab(task_id, "capture a screenshot")
if blocked:
return blocked
resp = _get_raw(_tab_path(session, "screenshot"), params=_user_params(session))
screenshot_path = _save_screenshot(resp.content)
img_b64 = base64.b64encode(resp.content).decode("utf-8")
annotation_context = ""
if annotate:
try:
annotation_context = f"\n\nAccessibility tree (element refs for interaction):\n{_raw_snapshot(session)[:3000]}"
except Exception:
pass
# The screenshot itself cannot be redacted, but the text-based accessibility
# snippet sent alongside it must not leak secret values.
from agent.redact import redact_sensitive_text
annotation_context = redact_sensitive_text(annotation_context)
from agent.auxiliary_client import call_llm
vision_prompt = f"Analyze this browser screenshot and answer: {question}{annotation_context}"
timeout, temperature = _vision_llm_settings()
response = call_llm(
messages=[{"role": "user", "content": [
{"type": "text", "text": vision_prompt},
{"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img_b64}"}},
]}],
task="vision", temperature=temperature, timeout=timeout,
)
analysis = (response.choices[0].message.content or "").strip() if response.choices else ""
# Redact secrets the vision LLM may have read from the screenshot.
return json.dumps({"success": True, "analysis": redact_sensitive_text(analysis), "screenshot_path": screenshot_path})
except Exception as e:
return tool_error(str(e), success=False)
def camofox_console(clear: bool = False, task_id: Optional[str] = None) -> str:
"""Console output is not exposed by the Camofox REST API; return an empty result with a note."""
return json.dumps({
"success": True, "console_messages": [], "js_errors": [], "total_messages": 0, "total_errors": 0,
"note": "Console log capture is not available with the Camofox backend. "
"Use browser_snapshot or browser_vision to inspect page state.",
})