From c5555efca8342ea42ddbd7eddc1cb6413fc07c86 Mon Sep 17 00:00:00 2001
From: teknium1 <127238744+teknium1@users.noreply.github.com>
Date: Sun, 27 Sep 2026 21:54:10 -0700
Subject: [PATCH] test(smoke): expect the per-response model_reply_issue
 counter

Every primary model response now records hermes.model_reply_issue.count
(issue=none for usable replies), so the smoke turn's two scripted replies
add one row with value 2; the exact-metric-set assertions must include it.
---
 scripts/smoke_nemo_relay_shared_metrics.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scripts/smoke_nemo_relay_shared_metrics.py b/scripts/smoke_nemo_relay_shared_metrics.py
index ddf3d425b1..ce600f3569 100644
--- a/scripts/smoke_nemo_relay_shared_metrics.py
+++ b/scripts/smoke_nemo_relay_shared_metrics.py
@@ -305,6 +305,7 @@ def _validate_store(database_path: Path) -> list[dict[str, Any]]:
         "hermes.context_peak.count",
         "hermes.install.milestone",
         "hermes.install.snapshot",
+        "hermes.model_reply_issue.count",
         "hermes.model_route.count",
         "hermes.model_tokens.sum",
         "hermes.model_tool_quality.count",
@@ -401,6 +402,9 @@ def _validate_store(database_path: Path) -> list[dict[str, Any]]:
     [quality] = by_name["hermes.model_tool_quality.count"]
     if quality["dimensions"] != {"call_role": "primary", "issue": "none", "model": "custom", "provider": "custom"}:
         raise AssertionError(f"Unexpected tool-call quality: {quality}")
+    [reply] = by_name["hermes.model_reply_issue.count"]  # both scripted replies are usable
+    if (reply["dimensions"], reply["value"]) != ({"issue": "none", "model": "custom", "provider": "custom"}, 2):
+        raise AssertionError(f"Unexpected model reply issues: {reply}")
     [peak] = by_name["hermes.context_peak.count"]
     if (peak["dimensions"]["provider"], peak["dimensions"]["model"], peak["dimensions"]["limit_hit"]) != (
         "custom", "custom", "no",
@@ -519,6 +523,7 @@ def _validate_packages(
         "hermes.context_peak.count",
         "hermes.install.milestone",
         "hermes.install.snapshot",
+        "hermes.model_reply_issue.count",
         "hermes.model_route.count",
         "hermes.model_tokens.sum",
         "hermes.model_tool_quality.count",