d10bb2ab6f
`scripts/run_tests.sh tests/<dir>/` is how a change gets its regression
coverage run, so a test filed under the wrong directory is a test nobody
runs when that code changes. Two kinds of drift had accumulated.
Parallel directories for one source package, folded into the mirror:
tests/acp -> tests/acp_adapter (its __init__/conftest move with it)
tests/cli -> tests/hermes_cli (prompt_toolkit fixture merged into
hermes_cli/conftest.py)
tests/run_agent -> tests/agent (backoff fixture becomes
agent/conftest.py)
tests/relay -> tests/gateway/relay
tests/state -> tests/hermes_state
246 loose files at tests/ root, routed by the package they import/patch:
hermes_cli, hermes_state, agent, gateway, tools, plugins, tui_gateway, cron.
Installer and desktop-update script tests go to tests/scripts/{install,
desktop_update}/. 43 tests of root-level modules (batch_runner, utils,
hermes_constants, packaging) stay at the root.
Filenames drop their issue numbers (95 files: test_89315_x.py -> test_x.py);
the number stays in the module docstring where it has context.
Collisions: test_cli_skin_integration.py existed in both tests/ and tests/cli
with different subsets — merged into one (10 tests, all kept);
run_agent/test_pre_compress_memory_context.py -> agent/..._handoff.py;
tests/test_account_usage.py -> agent/test_account_usage_fetch.py;
tests/test_web_server.py -> hermes_cli/test_web_server_ws_ping.py.
Deleted: test_minisweagent_path.py (empty since PR #2804),
test_model_picker_scroll.py (tested a private copy of the logic, imported
nothing), test_process_loop_event_loop_warning.py (asserted asyncio behaviour,
imported nothing from Hermes).
Repo-root path arithmetic (Path(__file__).parents[N], dirname chains) is
bumped for the 202 files that changed depth and verified by evaluating every
such expression against the new location. classify_changes' desktop-updater
lane prefix, tests-os.yml's ignore glob and every in-tree path comment follow
the moves. tests/test_tests_tree_layout.py keeps the tree from drifting back.
130 lines
5.4 KiB
Python
130 lines
5.4 KiB
Python
"""Regression: non-retryable API failures must not leak raw HTML pages.
|
|
|
|
A scheduled cron job fell back to the Codex (``chatgpt.com``) provider, which
|
|
returned a Cloudflare *challenge* page (HTTP 403) instead of a normal API
|
|
response. The conversation loop classified this as a non-retryable client
|
|
error and returned the failure dict — but the ``error`` field carried
|
|
``str(api_error)``, i.e. the entire ~60 KB Cloudflare HTML page. The cron
|
|
scheduler then delivered that verbatim to Discord, where it was split into
|
|
~31 messages (the reporter's "31 part discord message which is cloudflares
|
|
challenge page").
|
|
|
|
The sibling "max retries exhausted" path already summarized the error via
|
|
``_summarize_api_error`` (which collapses HTML pages to a one-liner); the
|
|
non-retryable path did not. These tests lock the contract: whichever
|
|
terminal path is taken, ``result['error']`` is a short, HTML-free summary.
|
|
"""
|
|
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import run_agent
|
|
from run_agent import AIAgent
|
|
|
|
|
|
# A representative Cloudflare "managed challenge" body, matching the shape the
|
|
# Codex backend returned in the field report (no <title>, large inline
|
|
# ``_cf_chl_opt`` script). Padded so length-based assertions are meaningful.
|
|
_CLOUDFLARE_CHALLENGE_HTML = (
|
|
"<!DOCTYPE html>\n<html>\n <head>\n"
|
|
' <meta http-equiv="refresh" content="360"></head>\n'
|
|
" <body>\n <div class=\"data\"><noscript>"
|
|
"Enable JavaScript and cookies to continue</noscript>"
|
|
"<script>(function(){window._cf_chl_opt = {cRay: 'a0ca002c4f91769c',"
|
|
"cZone: 'chatgpt.com', cType: 'managed', "
|
|
+ ("md: '" + "x" * 4000 + "',")
|
|
+ "};})();</script></div>\n </body>\n</html>\n"
|
|
)
|
|
|
|
|
|
def _make_403_html_error() -> Exception:
|
|
"""An exception mimicking a Codex 403 whose body is a Cloudflare page."""
|
|
err = Exception(_CLOUDFLARE_CHALLENGE_HTML)
|
|
err.status_code = 403
|
|
return err
|
|
|
|
|
|
def _make_agent() -> AIAgent:
|
|
# Drive the standard chat-completions path with a concrete model so the
|
|
# turn actually reaches ``client.chat.completions.create`` — that is where
|
|
# the mocked 403 is raised. The non-retryable abort being exercised lives
|
|
# in the shared conversation loop and is provider-agnostic; a Cloudflare
|
|
# "managed challenge" 403 can surface on any provider sitting behind
|
|
# Cloudflare (it was first reported on the Codex backend). Pinning
|
|
# ``api_mode`` + ``model`` here avoids the earlier abort the previous
|
|
# revision hit: an empty model on the Codex Responses path raised a
|
|
# validation ``ValueError`` *before* any API call, so the test passed
|
|
# without ever touching the 403 summarization path.
|
|
with (
|
|
patch("model_tools.get_tool_definitions", return_value=[]),
|
|
patch("model_tools.check_toolset_requirements", return_value={}),
|
|
patch("agent.process_bootstrap.OpenAI"),
|
|
):
|
|
a = AIAgent(
|
|
api_key="test-key-1234567890",
|
|
base_url="https://api.openai.com/v1",
|
|
provider="openai",
|
|
api_mode="chat_completions",
|
|
model="gpt-5.5",
|
|
quiet_mode=True,
|
|
skip_context_files=True,
|
|
skip_memory=True,
|
|
)
|
|
a.client = MagicMock()
|
|
a._cached_system_prompt = "You are helpful."
|
|
a._use_prompt_caching = False
|
|
a.compression_enabled = False
|
|
a.save_trajectories = False
|
|
return a
|
|
|
|
|
|
def test_summarize_collapses_cloudflare_challenge_page():
|
|
"""``_summarize_api_error`` must never echo the raw HTML body."""
|
|
summary = AIAgent._summarize_api_error(_make_403_html_error())
|
|
|
|
assert "<html" not in summary.lower()
|
|
assert "<!doctype" not in summary.lower()
|
|
assert "_cf_chl_opt" not in summary
|
|
# A one-liner, not a multi-kilobyte page.
|
|
assert len(summary) < 200
|
|
# Still informative: the HTTP status survives.
|
|
assert "403" in summary
|
|
|
|
|
|
def test_non_retryable_failure_error_is_summarized_not_raw_html():
|
|
"""The terminal non-retryable dict must carry a short, HTML-free error.
|
|
|
|
This is the exact field path: a 403 Cloudflare challenge with no fallback
|
|
configured aborts as a non-retryable client error. Before the fix the
|
|
returned ``error`` was the full ~60 KB page.
|
|
|
|
The mocked 403 is the *only* failure the turn can hit — the agent reaches
|
|
``client.chat.completions.create`` (asserted below), so the test cannot
|
|
pass vacuously by aborting on some earlier, unrelated error.
|
|
"""
|
|
agent = _make_agent()
|
|
agent.client.chat.completions.create.side_effect = _make_403_html_error()
|
|
|
|
with (
|
|
patch.object(agent, "_persist_session"),
|
|
patch.object(agent, "_save_trajectory"),
|
|
patch.object(agent, "_cleanup_task_resources"),
|
|
):
|
|
result = agent.run_conversation("daily briefing please")
|
|
|
|
# Guard against a vacuous pass: the mocked 403 must actually be the
|
|
# failure that aborted the turn. (The previous revision never reached
|
|
# this call and still "passed".)
|
|
assert agent.client.chat.completions.create.called
|
|
assert result.get("failed") is True
|
|
error = result.get("error") or ""
|
|
# The whole point of the fix: no raw HTML / Cloudflare markup leaks.
|
|
assert "<html" not in error.lower()
|
|
assert "<!doctype" not in error.lower()
|
|
assert "_cf_chl_opt" not in error
|
|
# Still informative: the summarized 403 status survives into the field
|
|
# delivered downstream.
|
|
assert "403" in error
|
|
# The original page was tens of kilobytes; a summary is short.
|
|
assert len(error) < 500
|
|
assert len(error) < len(_CLOUDFLARE_CHALLENGE_HTML)
|