From 41201bf2153acf76ba60ba8d5edb7c06de4c2ab5 Mon Sep 17 00:00:00 2001 From: chelsealong Date: Wed, 9 Sep 2026 11:46:14 +0000 Subject: [PATCH] fix(agent): gate execution-guidance tool lines on the session's toolset OPENAI_MODEL_EXECUTION_GUIDANCE told every GPT-family session to use terminal/execute_code for arithmetic and terminal/read_file/search_files for hashes, time, system state, files and git, even when the session's toolset had none of those tools. Lean toolsets then refused plain arithmetic as "prohibited mental computation". Drop each guidance line when none of the tools it names are in valid_tool_names. The toolset is fixed per session, so the system prompt stays cache-stable. Fixes #106506 --- agent/prompt_builder.py | 35 ++++++++++++++++++--- agent/system_prompt.py | 2 +- tests/agent/test_phantom_tool_references.py | 24 +++++++++++++- 3 files changed, 54 insertions(+), 7 deletions(-) diff --git a/agent/prompt_builder.py b/agent/prompt_builder.py index e53105f5f6..b433592093 100644 --- a/agent/prompt_builder.py +++ b/agent/prompt_builder.py @@ -506,13 +506,38 @@ OPENAI_MODEL_EXECUTION_GUIDANCE = ( ) -def execution_guidance_text() -> str: - """OPENAI_MODEL_EXECUTION_GUIDANCE as injected into the system prompt. +# Each / line above named against the tool(s) it tells the model to reach +# for. A line is dropped when none of its named tools are in the session's valid_tool_names, so a toolset +# without terminal/execute_code/etc. isn't told to use them (#106506). The +# guidance names no web tool (#39797), so there is no entry for one. +_EXECUTION_GUIDANCE_LINE_TOOLS = { + "- Arithmetic, math, calculations → use terminal or execute_code\n": {"terminal", "execute_code"}, + "- Hashes, encodings, checksums → use terminal (e.g. sha256sum, base64)\n": {"terminal"}, + "- Current time, date, timezone → use terminal (e.g. date)\n": {"terminal"}, + "- System state: OS, CPU, memory, disk, ports, processes → use terminal\n": {"terminal"}, + "- File contents, sizes, line counts → use read_file, search_files, or terminal\n": { + "read_file", + "search_files", + "terminal", + }, + "- Git history, branches, diffs → use terminal\n": {"terminal"}, + "- 'What time is it?' → run `date` (don't guess)\n": {"terminal"}, +} - The guidance names no web tool (#39797: a hard "use web_search" overrode SOUL.md and dangled in Blank Slate), - so the text is toolset-neutral and needs no per-session filtering. + +def execution_guidance_text(valid_tool_names=None) -> str: + """OPENAI_MODEL_EXECUTION_GUIDANCE for the session's toolset (cache-safe: the toolset is fixed per session). + + Lines that tell the model to reach for a tool absent from the session's toolset would dangle, so they are + dropped. """ - return OPENAI_MODEL_EXECUTION_GUIDANCE + text = OPENAI_MODEL_EXECUTION_GUIDANCE + if valid_tool_names is not None: + valid_tool_names = set(valid_tool_names) + for line, tools in _EXECUTION_GUIDANCE_LINE_TOOLS.items(): + if not tools & valid_tool_names: + text = text.replace(line, "") + return text # Gemini/Gemma-specific operational guidance, adapted from OpenCode's gemini.txt. diff --git a/agent/system_prompt.py b/agent/system_prompt.py index c010d59bc4..fb2942e429 100644 --- a/agent/system_prompt.py +++ b/agent/system_prompt.py @@ -575,7 +575,7 @@ def _guidance_parts(agent: Any) -> List[str]: parts.append(GOOGLE_MODEL_OPERATIONAL_GUIDANCE) if _model_gate(getattr(agent, "_execution_guidance", "auto"), agent.model, EXECUTION_GUIDANCE_MODELS): from agent.prompt_builder import execution_guidance_text - parts.append(execution_guidance_text()) + parts.append(execution_guidance_text(agent.valid_tool_names)) # delegate_task background delivery is intentionally between turns. Put this after the generic persistence # blocks so their "keep working" rule cannot turn the required yield into no-op/polling activity. if "delegate_task" in agent.valid_tool_names: diff --git a/tests/agent/test_phantom_tool_references.py b/tests/agent/test_phantom_tool_references.py index b194e57f15..800a4435eb 100644 --- a/tests/agent/test_phantom_tool_references.py +++ b/tests/agent/test_phantom_tool_references.py @@ -4,7 +4,9 @@ session can't call (Blank Slate audit, Aug 2026). Covers: * HERMES_AGENT_HELP_GUIDANCE degrades to the docs-only variant when the skill tools aren't loaded. - * execution_guidance_text() never names a web tool (guidance is toolset-neutral). + * execution_guidance_text() never names a web tool, and drops the + terminal/execute_code/read_file/search_files lines the session's toolset + can't back (#106506). * The coding operating brief drops the `todo` sentence when the todo tool isn't loaded. * ESSENTIAL_SKILLS can't be disabled via config, and the CLI writer strips @@ -14,6 +16,26 @@ Covers: from pathlib import Path +class TestExecutionGuidanceText: + def test_full_text_when_toolset_has_every_named_tool_or_is_unknown(self): + from agent.prompt_builder import OPENAI_MODEL_EXECUTION_GUIDANCE, execution_guidance_text + assert execution_guidance_text(None) == OPENAI_MODEL_EXECUTION_GUIDANCE + assert execution_guidance_text( + {"terminal", "execute_code", "read_file", "search_files"} + ) == OPENAI_MODEL_EXECUTION_GUIDANCE + + def test_lean_toolset_is_not_told_to_use_absent_tools(self): + # #106506: a toolset with none of terminal/execute_code/read_file/search_files must not be told to + # reach for them (GPT models then refused plain arithmetic as "prohibited mental computation"). + from agent.prompt_builder import execution_guidance_text + text = execution_guidance_text({"memory"}) + mandatory = text.split("")[1].split("")[0] + for tool in ("terminal", "execute_code", "read_file", "search_files", "Arithmetic"): + assert tool not in mandatory + assert "an appropriate permitted retrieval/search tool" in mandatory + assert "run `date`" not in text.split("")[1].split("")[0] + + class TestCodingBriefTodoGating: def _brief(self, valid_tool_names): from agent.coding_context import CODING_PROFILE, RuntimeMode