hermes-agent/tests/tools/test_web_tools_truncate.py

"""Unit tests for the truncate-and-store web_extract path (no LLM).

Covers convert_base64_images_to_links, _truncate_with_footer, _store_full_text,
_get_extract_char_limit, and the end-to-end web_extract_tool truncation behavior.
"""
import asyncio
import json
import os
from pathlib import Path
from unittest.mock import patch

import pytest

import tools.web_tools as wt
from tools import web_tools_truncate


class TestImageConversion:
    def test_markdown_base64_image_keeps_alt_drops_blob(self):
        blob = "A" * 5000
        text = f"before ![a cat]( data:image/png;base64,{blob}) after"
        out = web_tools_truncate.convert_base64_images_to_links(text)
        assert "[IMAGE: a cat]" in out
        assert "base64" not in out
        assert blob not in out
        assert "before" in out and "after" in out


    def test_bare_and_parenthesised_base64_become_placeholder(self):
        blob = "Z" * 3000
        bare = web_tools_truncate.convert_base64_images_to_links(f"data:image/gif;base64,{blob}")
        assert bare == "[IMAGE]"
        paren = web_tools_truncate.convert_base64_images_to_links(f"(data:image/gif;base64,{blob})")
        assert paren == "[IMAGE]"


class TestTruncation:
    def test_short_content_returned_whole(self):
        content = "# Title\n\nshort body\n"
        out, truncated = web_tools_truncate._truncate_with_footer(content, "https://e.com", 15000)
        assert out == content
        assert truncated is False


    def test_truncation_stores_full_text_readable(self, tmp_path, monkeypatch):
        monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))
        body = "UNIQUE_MIDDLE_MARKER\n" + ("\n".join(f"row {i}" for i in range(5000)))
        out, truncated = web_tools_truncate._truncate_with_footer(body, "https://example.com/doc", 3000)
        assert truncated is True
        # Extract the stored path from the footer and confirm full text is there.
        path_line = next(ln for ln in out.splitlines() if "Full text saved to:" in ln)
        stored_path = path_line.split("Full text saved to:", 1)[1].strip()
        assert os.path.exists(stored_path)
        full = Path(stored_path).read_text(encoding="utf-8")
        assert "UNIQUE_MIDDLE_MARKER" in full
        assert "row 2500" in full  # the omitted-middle row is in the stored file


class TestCharLimitConfig:
    def test_default_when_unset(self):
        with patch("tools.web_tools._load_web_config", return_value={}):
            assert web_tools_truncate._get_extract_char_limit() == web_tools_truncate.DEFAULT_EXTRACT_CHAR_LIMIT


    def test_bad_value_falls_back(self):
        with patch("tools.web_tools._load_web_config", return_value={"extract_char_limit": "nope"}):
            assert web_tools_truncate._get_extract_char_limit() == web_tools_truncate.DEFAULT_EXTRACT_CHAR_LIMIT


class TestEndToEnd:
    def test_web_extract_truncates_large_page_no_llm(self, tmp_path, monkeypatch):
        monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))
        big = "\n".join(f"para {i} " + "y" * 80 for i in range(3000))

        class FakeProvider:
            name = "fake"
            display_name = "Fake"

            def supports_extract(self):
                return True

            async def extract(self, urls, **kwargs):
                return [{"url": urls[0], "title": "Big Page", "content": big,
                         "raw_content": big, "metadata": {}}]

        with patch("tools.web_tools._ensure_web_plugins_loaded"), \
             patch("tools.web_tools._get_extract_backend", return_value="fake"), \
             patch("tools.web_tools.async_is_safe_url", new=_AsyncTrue()), \
             patch("agent.web_search_registry.get_provider", return_value=FakeProvider()):
            result = json.loads(asyncio.new_event_loop().run_until_complete(
                wt.web_extract_tool(["https://example.com/big"], char_limit=5000)
            ))

        assert "results" in result
        content = result["results"][0]["content"]
        assert "[TRUNCATED]" in content
        assert "Full text saved to:" in content
        # No LLM was involved: para 0 (head) and the last para (tail) are verbatim.
        assert "para 0 " in content
        assert "para 2999 " in content


def _make_awaitable(value):
    async def _coro(*a, **k):
        return value
    return _coro()


class _AsyncTrue:
    """Async callable that always returns True (re-awaitable per call)."""
    async def __call__(self, *a, **k):
        return True


def test_binary_payload_is_refused_but_prose_with_short_signature_prefix_passes():
    """A backend that fetched a raw SQLite/zip file hands its bytes back as text; that must become an
    error naming the type, while ordinary pages (even ones starting with 'BM' or 'MZ') pass untouched."""
    results = [
        {"url": "u", "raw_content": "SQLite format 3\x10\x01" + "x" * 5000},  # backend already dropped the NUL
        {"url": "z", "raw_content": "PK\x03\x04" + "y" * 50},
        {"url": "v", "raw_content": "BMW reviews are fine"},
        {"url": "w", "raw_content": "# hi"},
    ]
    web_tools_truncate._truncate_results(results, 5000, {"pages_truncated": 0, "truncation_metrics": []})
    assert results[0]["content"] == "" and "SQLite database" in results[0]["error"]
    assert results[1]["content"] == "" and "ZIP archive" in results[1]["error"]
    assert results[2]["error"] is None if "error" in results[2] else True
    assert results[2]["content"] == "BMW reviews are fine"
    assert results[3]["content"] == "# hi"