27a4023791
AGENTS.md: Project Structure tree reflects the decomposition (run_agent 1.5k not 12k, cli 4.6k not 11k,
hermes_state facade + 21 siblings, web_routers/, evals/, test counts); new "Facade + siblings layout"
section with the sibling families table and the rules that follow (find by topic, patch where production
reads, compat pointers off limits, don't recreate god files); AIAgent/Agent Loop point at agent/turn_*.py
and conversation_loop; CLI dispatch documents _SLASH_DISPATCH + the _handle_<name>_command convention and
"Adding a Slash Command" no longer tells you to add an elif (there is no ladder to add to on either surface).
evals/codebase_navigability/: what the codebase costs an agent, not the CPU.
bench.py ~19k real "locate X" tasks from tests/ imports; tokens (tiktoken o200k) of the defining
file vs the symbol, context-window fit, read windows, siblings, symbol CC
lookup_sim.py paired grep+read simulation over 4k common symbols; tool calls + tokens returned
static_metrics.py LOC split, size distributions, elif/nesting, radon CC/MI, import graph + SCC cycles
runtime_bench.py fresh-interpreter import/CLI/hot-path/collection timings with tree-purity assertion
tests/evals/test_codebase_navigability.py pins the resolver's facade/sibling behaviour.
56 lines
2.2 KiB
Python
56 lines
2.2 KiB
Python
"""Guardrail for the navigability eval: its symbol resolver must follow the facade/sibling layout.
|
|
|
|
Two invariants the harness relies on (and that a future refactor could silently break):
|
|
1. a name imported from a facade that lives in a sibling resolves to the SIBLING (through the facade's
|
|
top-level `from sibling import name` or its PLUGIN-COMPAT lazy table);
|
|
2. a name defined in the facade itself resolves to the facade.
|
|
Both are checked against real modules on the current tree, so they also pin the layout the eval documents.
|
|
"""
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from evals.codebase_navigability import bench
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def mods():
|
|
return bench.source_modules(ROOT)
|
|
|
|
|
|
def test_resolver_follows_reexport_to_defining_sibling(mods):
|
|
cache: dict = {}
|
|
# gateway.run re-exports names it no longer defines; each must resolve to a gateway.run_* sibling
|
|
src = (ROOT / "gateway" / "run.py").read_text(encoding="utf-8")
|
|
rex = bench._reexports(src)
|
|
moved = [(n, m) for n, (m, _) in rex.items() if m.startswith("gateway.run_")]
|
|
assert moved, "expected gateway.run to re-export from gateway.run_* siblings"
|
|
for name, expected in moved[:20]:
|
|
got = bench.resolve_definer(mods, name, "gateway.run", cache)
|
|
assert got == expected, (name, got, expected)
|
|
|
|
|
|
def test_resolver_keeps_facade_defined_names_on_facade(mods):
|
|
cache: dict = {}
|
|
src = (ROOT / "hermes_state.py").read_text(encoding="utf-8")
|
|
own = [n for n, (_, _, node) in bench.top_level_defs(src).items() if not bench._is_alias(node)]
|
|
assert own, "hermes_state.py should still define something at top level"
|
|
for name in own[:20]:
|
|
assert bench.resolve_definer(mods, name, "hermes_state", cache) == "hermes_state", name
|
|
|
|
|
|
def test_tokenizer_falls_back_without_crashing(monkeypatch):
|
|
import builtins
|
|
real_import = builtins.__import__
|
|
|
|
def no_tiktoken(name, *a, **k):
|
|
if name == "tiktoken":
|
|
raise ImportError
|
|
return real_import(name, *a, **k)
|
|
|
|
monkeypatch.setattr(builtins, "__import__", no_tiktoken)
|
|
tok = bench._tokenizer()
|
|
assert tok("abcdefgh") == 2
|