d10bb2ab6f
`scripts/run_tests.sh tests/<dir>/` is how a change gets its regression
coverage run, so a test filed under the wrong directory is a test nobody
runs when that code changes. Two kinds of drift had accumulated.
Parallel directories for one source package, folded into the mirror:
tests/acp -> tests/acp_adapter (its __init__/conftest move with it)
tests/cli -> tests/hermes_cli (prompt_toolkit fixture merged into
hermes_cli/conftest.py)
tests/run_agent -> tests/agent (backoff fixture becomes
agent/conftest.py)
tests/relay -> tests/gateway/relay
tests/state -> tests/hermes_state
246 loose files at tests/ root, routed by the package they import/patch:
hermes_cli, hermes_state, agent, gateway, tools, plugins, tui_gateway, cron.
Installer and desktop-update script tests go to tests/scripts/{install,
desktop_update}/. 43 tests of root-level modules (batch_runner, utils,
hermes_constants, packaging) stay at the root.
Filenames drop their issue numbers (95 files: test_89315_x.py -> test_x.py);
the number stays in the module docstring where it has context.
Collisions: test_cli_skin_integration.py existed in both tests/ and tests/cli
with different subsets — merged into one (10 tests, all kept);
run_agent/test_pre_compress_memory_context.py -> agent/..._handoff.py;
tests/test_account_usage.py -> agent/test_account_usage_fetch.py;
tests/test_web_server.py -> hermes_cli/test_web_server_ws_ping.py.
Deleted: test_minisweagent_path.py (empty since PR #2804),
test_model_picker_scroll.py (tested a private copy of the logic, imported
nothing), test_process_loop_event_loop_warning.py (asserted asyncio behaviour,
imported nothing from Hermes).
Repo-root path arithmetic (Path(__file__).parents[N], dirname chains) is
bumped for the 202 files that changed depth and verified by evaluating every
such expression against the new location. classify_changes' desktop-updater
lane prefix, tests-os.yml's ignore glob and every in-tree path comment follow
the moves. tests/test_tests_tree_layout.py keeps the tree from drifting back.
100 lines
4.0 KiB
Python
100 lines
4.0 KiB
Python
"""Regression guard for #14782: json.JSONDecodeError must not be classified
|
|
as a local validation error by the main agent loop.
|
|
|
|
`json.JSONDecodeError` inherits from `ValueError`. The agent loop's
|
|
non-retryable classifier at run_agent.py treats `ValueError` / `TypeError`
|
|
as local programming bugs and skips retry. Without an explicit carve-out,
|
|
a transient provider hiccup (malformed response body, truncated stream,
|
|
routing-layer corruption) that surfaces as a JSONDecodeError would bypass
|
|
the retry path and fail the turn immediately.
|
|
|
|
This test mirrors the exact predicate shape used in run_agent.py so that
|
|
any future refactor of that predicate must preserve the invariant:
|
|
|
|
JSONDecodeError → NOT local validation error (retryable)
|
|
UnicodeEncodeError → NOT local validation error (surrogate path)
|
|
bare ValueError → IS local validation error (programming bug)
|
|
bare TypeError → IS local validation error (programming bug)
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
|
|
def _mirror_agent_predicate(err: BaseException) -> bool:
|
|
"""Exact shape of run_agent.py's is_local_validation_error check.
|
|
|
|
Kept in lock-step with the source. If you change one, change both —
|
|
or, better, refactor the check into a shared helper and have both
|
|
sites import it.
|
|
"""
|
|
import ssl
|
|
|
|
return (
|
|
isinstance(err, (ValueError, TypeError))
|
|
and not isinstance(err, (UnicodeEncodeError, json.JSONDecodeError))
|
|
and not isinstance(err, ssl.SSLError)
|
|
# NoneType-is-not-iterable shape errors come from upstream SDK /
|
|
# provider response mismatches, not local programming bugs. See
|
|
# the agent/conversation_loop.py inline comment for #33136.
|
|
and not (
|
|
isinstance(err, TypeError)
|
|
and "nonetype" in str(err).lower()
|
|
and "not iterable" in str(err).lower()
|
|
)
|
|
)
|
|
|
|
|
|
class TestJSONDecodeErrorIsRetryable:
|
|
|
|
def test_json_decode_error_is_not_local_validation(self):
|
|
"""Provider returning malformed JSON surfaces as JSONDecodeError —
|
|
must be treated as transient so the retry path runs."""
|
|
try:
|
|
json.loads("{not valid json")
|
|
except json.JSONDecodeError as exc:
|
|
assert not _mirror_agent_predicate(exc), (
|
|
"json.JSONDecodeError must be excluded from the "
|
|
"ValueError/TypeError local-validation classification."
|
|
)
|
|
else:
|
|
raise AssertionError("json.loads should have raised")
|
|
|
|
|
|
def test_bare_value_error_is_local_validation(self):
|
|
"""Programming bugs that raise bare ValueError must still be
|
|
classified as local validation errors (non-retryable)."""
|
|
assert _mirror_agent_predicate(ValueError("bad arg"))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class TestNoneTypeNotIterableIsRetryable:
|
|
"""Regression for #33136 / closes lingering Telegram \"Non-retryable error (HTTP None)\".
|
|
|
|
The chatgpt.com Codex backend (and any other upstream SDK / provider shim)
|
|
can surface ``TypeError: 'NoneType' object is not iterable`` as a wire-shape
|
|
mismatch, not a local programming bug. Even after #33042 made our own
|
|
consumer immune, third-party paths and mocked clients can still produce
|
|
this shape. The classifier should treat it as retryable so the normal
|
|
retry/fallback chain runs.
|
|
"""
|
|
|
|
def test_nonetype_not_iterable_is_retryable(self):
|
|
err = TypeError("'NoneType' object is not iterable")
|
|
assert not _mirror_agent_predicate(err), (
|
|
"TypeError('NoneType ... not iterable') must be excluded from "
|
|
"is_local_validation_error — it is a provider/SDK shape mismatch, "
|
|
"not a local bug. See #33136."
|
|
)
|
|
|
|
|
|
def test_unrelated_type_error_remains_local_validation(self):
|
|
"""TypeError without the NoneType-not-iterable pattern still aborts (programming bug)."""
|
|
assert _mirror_agent_predicate(TypeError("tools must be a list"))
|
|
assert _mirror_agent_predicate(TypeError("expected str, got int"))
|
|
|
|
|