From fc8d15d7797c36cdc78ca9cd4a07187c3cc61c91 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 7 Sep 2026 00:37:04 -0700 Subject: [PATCH] test: retain a reproducible CLI goal parity oracle --- evals/goal_command_parity.py | 51 ++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 evals/goal_command_parity.py diff --git a/evals/goal_command_parity.py b/evals/goal_command_parity.py new file mode 100644 index 0000000000..f8b5bc55ac --- /dev/null +++ b/evals/goal_command_parity.py @@ -0,0 +1,51 @@ +"""CLI /goal semantic oracle, with real storage and no model calls. + +Run against base and candidate in fresh processes with distinct temporary homes: + python evals/goal_command_parity.py ROOT NEW_HOME OUTPUT_JSON +Compare state and queued prompts; output includes presentation differences. +""" +import importlib +import json +import os +from pathlib import Path +import queue +import sys + +root, home, out = map(Path, sys.argv[1:4]) +home.mkdir(parents=True, exist_ok=False) +os.environ['HERMES_HOME'] = str(home) +os.environ['HERMES_NIX_BUILD'] = '1' +sys.path.insert(0, str(root)) +commands = importlib.import_module("hermes_cli.cli_commands_mixin") +GoalManager = importlib.import_module("hermes_cli.goals").GoalManager + +manager = GoalManager(session_id='cli-oracle', default_max_turns=17) +cli = commands.CLICommandsMixin() +cli._get_goal_manager = lambda: manager +cli._pending_input = queue.Queue() +cli.conversation_history = [] +printed = [] +commands._cp = lambda *lines: printed.extend(lines) +rows = [] +command_args = ['', 'show', 'pause', 'resume', 'clear', 'draft', + 'Maintain the service\nconstraints: keep data\nstop when: credentials missing', + 'show', 'pause', 'resume', 'wait nope', 'wait', 'wait 0', + f'wait {os.getpid()} running probe', 'status', 'unwait', 'unwait', + 'gate list', 'gate remove nope', 'gate clear', 'done', 'status'] +for arg in command_args: + printed.clear() + cli._handle_goal_command('/goal' + (' ' + arg if arg else '')) + state = manager.state + prompts = [] + while not cli._pending_input.empty(): + prompts.append(cli._pending_input.get_nowait()) + contract = state.contract if state else None + rows.append({'command': arg.replace(str(os.getpid()), ''), + 'active': manager.has_goal(), + 'goal': state.goal if state else None, + 'status': state.status if state else None, + 'contract': contract.to_dict() if contract else None, + 'prompt_count': len(prompts), 'prompts': prompts, + 'output': '\n'.join(printed).replace(str(os.getpid()), '')}) +out.write_text(json.dumps(rows, indent=2, default=str)) +print(json.dumps(rows, indent=2, default=str))