hermes-admin/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

test(smoke): expect the per-response model_reply_issue counter

Browse Source 
Every primary model response now records hermes.model_reply_issue.count
(issue=none for usable replies), so the smoke turn's two scripted replies
add one row with value 2; the exact-metric-set assertions must include it.
This commit is contained in:
teknium1
2026-09-27 21:54:10 -07:00
committed by Teknium
parent 8d8836ddb1
commit c5555efca8
1 changed files with 5 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
scripts/smoke_nemo_relay_shared_metrics.py
View File

@@ -305,6 +305,7 @@ def _validate_store(database_path: Path) -> list[dict[str, Any]]:
"hermes.context_peak.count",
"hermes.install.milestone",
"hermes.install.snapshot",
"hermes.model_reply_issue.count",
"hermes.model_route.count",
"hermes.model_tokens.sum",
"hermes.model_tool_quality.count",
@@ -401,6 +402,9 @@ def _validate_store(database_path: Path) -> list[dict[str, Any]]:
[quality] = by_name["hermes.model_tool_quality.count"]
if quality["dimensions"] != {"call_role": "primary", "issue": "none", "model": "custom", "provider": "custom"}:
raise AssertionError(f"Unexpected tool-call quality: {quality}")
[reply] = by_name["hermes.model_reply_issue.count"] # both scripted replies are usable
if (reply["dimensions"], reply["value"]) != ({"issue": "none", "model": "custom", "provider": "custom"}, 2):
raise AssertionError(f"Unexpected model reply issues: {reply}")
[peak] = by_name["hermes.context_peak.count"]
if (peak["dimensions"]["provider"], peak["dimensions"]["model"], peak["dimensions"]["limit_hit"]) != (
"custom", "custom", "no",
@@ -519,6 +523,7 @@ def _validate_packages(
"hermes.context_peak.count",
"hermes.install.milestone",
"hermes.install.snapshot",
"hermes.model_reply_issue.count",
"hermes.model_route.count",
"hermes.model_tokens.sum",
"hermes.model_tool_quality.count",