feat(session-search): OR-relaxed retry recovers paraphrased recall

Port from nearai/ironclaw#7553 (Filter::FtsRanked): FTS5's implicit AND
between terms means a paraphrased multi-word query misses a stored
sentence that lacks even one of the words. When the exact-match search
and the substring fallbacks all return zero rows, retry the same
unicode61 FTS index with the terms OR-joined, ranked by bm25 so rows
covering more terms surface first.

Strictly additive: gated on a zero-result miss, so successful searches
keep exact-match semantics and ordering. Queries with explicit OR/NOT,
single-term queries, and CJK-routed queries are left untouched. Quoted
phrases relax as whole units.

Adapted for hermes-agent: implemented inside SessionSearchMixin's
zero-result fallback chain (after the CJK-bigram/trigram substring
retries) rather than as a separate filter variant, reusing the already-
built SQL/params so all source/role/sort filters apply to the retry.
This commit is contained in:
Teknium
2026-08-16 17:14:51 -07:00
parent 56e563d3a4
commit 2a2c3aa588
3 changed files with 156 additions and 2 deletions

View File

@@ -844,6 +844,21 @@ class SessionSearchMixin:
run = 0
return run == 1
@staticmethod
def _or_relaxed_query(query: str) -> Optional[str]:
"""The sanitized implicit-AND query rewritten as an any-term OR query, or ``None`` when
relaxation does not apply: fewer than two searchable units (a single term cannot relax)
or explicit ``OR``/``NOT`` (the caller expressed exact semantics). Quoted phrases stay
whole units: ``"docker networking" tls`` -> ``"docker networking" OR tls``."""
units: List[str] = []
for raw_token in _LIKE_TOKEN_RE.findall(query):
upper = raw_token.upper()
if upper in {"OR", "NOT"}:
return None
if upper != "AND":
units.append(raw_token)
return " OR ".join(units) if len(units) >= 2 else None
@staticmethod
def _trigram_eligible_tokens(query: str) -> bool:
"""True when every non-operator token is >=3 chars: a shorter token produces no
@@ -1128,6 +1143,20 @@ class SessionSearchMixin:
matches = self._match_rows("messages_fts_cjk", fb_query, **route) or matches
if not matches and self._trigram_available and self._trigram_eligible_tokens(query):
matches = self._match_rows("messages_fts_trigram", fb_query, **route) or matches
# OR-relaxed retry (port of nearai/ironclaw#7553 ``Filter::FtsRanked``): the implicit AND
# between terms means a paraphrased multi-word query misses a stored sentence that lacks
# even ONE word ("when does Sarah like her standup scheduled" vs "Sarah prefers the standup
# meeting scheduled ... Thursday mornings"). Once the exact query and the substring
# fallbacks all miss, retry the unicode61 index matching ANY term; bm25 ranks rows covering
# more terms first. Gated on a zero-result miss so hits keep exact-match semantics and
# ordering; explicit OR/NOT, single-term and CJK-routed queries are left alone.
if not matches and not is_cjk and not self._fts_stale:
relaxed = self._or_relaxed_query(query)
if relaxed is not None:
matches = self._match_rows("messages_fts", relaxed, fail_open="OR-relaxed",
operational_debug="OR-relaxed FTS retry failed; keeping empty result",
**route) or matches
return self._finalize_search_matches(matches, result_fields=result_fields)
def _search_cjk(self, query: str, wants_unindexed_rows: bool, route: Dict[str, Any]) -> List[Dict[str, Any]]:

View File

@@ -0,0 +1,119 @@
"""OR-relaxed zero-result retry for paraphrased session search recall.
Ported from nearai/ironclaw#7553 (``Filter::FtsRanked``): FTS5's implicit
AND between query terms means a multi-word query worded even slightly
differently from the stored sentence returns nothing — a fact saved as
"Sarah prefers the standup meeting scheduled early on Thursday mornings" is
invisible to "when does Sarah like her standup scheduled" purely because the
stored text has no "like". When the exact-match search (and the substring
fallbacks) return zero rows, ``search_messages`` retries the same FTS index
with the terms OR-joined, ranked by bm25 so rows covering more of the terms
surface first.
The retry is strictly additive: it only fires on a zero-result miss, never
reorders existing hits, and respects explicit boolean operators.
"""
import pytest
from hermes_state import SessionDB
@pytest.fixture
def db(tmp_path):
d = SessionDB(db_path=tmp_path / "state.db")
d.create_session(session_id="s1", source="cli", model="m")
d.append_message(
"s1",
role="user",
content=(
"Sarah prefers the standup meeting scheduled early on "
"Thursday mornings"
),
)
d.append_message(
"s1", role="assistant", content="Noted, standup moved to Thursday."
)
d.append_message(
"s1", role="user", content="graphiti daemon looks healthy today"
)
yield d
try:
d.close()
except Exception:
pass
class TestOrRelaxedQueryHelper:
def test_multi_term_query_relaxes_to_or(self):
assert (
SessionDB._or_relaxed_query("sarah standup scheduled")
== "sarah OR standup OR scheduled"
)
def test_single_term_returns_none(self):
assert SessionDB._or_relaxed_query("standup") is None
def test_explicit_or_is_respected(self):
assert SessionDB._or_relaxed_query("alpha OR beta") is None
def test_explicit_not_is_respected(self):
assert SessionDB._or_relaxed_query("python NOT java") is None
def test_explicit_and_tokens_are_dropped(self):
assert (
SessionDB._or_relaxed_query("alpha AND beta")
== "alpha OR beta"
)
def test_quoted_phrase_is_one_unit(self):
assert (
SessionDB._or_relaxed_query('"docker networking" tls')
== '"docker networking" OR tls'
)
def test_lone_quoted_phrase_returns_none(self):
assert SessionDB._or_relaxed_query('"docker networking"') is None
class TestParaphrasedRecall:
def test_exact_query_still_matches_exactly(self, db):
rows = db.search_messages("standup Thursday")
assert rows, "exact-term query must match without relaxation"
assert "standup" in rows[0]["snippet"].lower()
def test_paraphrased_query_recovers_via_or_retry(self, db):
# "when does Sarah like her standup scheduled": implicit AND requires
# "like", which the stored sentence lacks — exact match returns
# nothing; the OR-relaxed retry must recover the row.
rows = db.search_messages("when does Sarah like her standup scheduled")
assert rows, "paraphrased query must recover via the OR-relaxed retry"
joined = " ".join(r["snippet"].lower() for r in rows)
assert "standup" in joined
def test_or_retry_recovers_all_partially_matching_rows(self, db):
rows = db.search_messages("sarah standup thursday daemon")
# No stored row contains every term, so only the OR retry can answer;
# both partial matches must come back (bm25 ordering between them is
# backend-weighted, not coverage-guaranteed).
joined = " ".join(r["snippet"].lower() for r in rows)
assert "standup" in joined
assert "daemon" in joined
def test_genuinely_absent_terms_still_return_empty(self, db):
assert db.search_messages("zebra xylophone quantum") == []
def test_explicit_not_query_is_not_relaxed(self, db):
# "standup NOT Thursday" must exclude the Thursday rows — relaxation
# would wrongly resurrect them.
rows = db.search_messages("standup NOT Thursday")
for r in rows:
assert "thursday" not in r["snippet"].lower()
def test_role_filter_applies_to_relaxed_retry(self, db):
rows = db.search_messages(
"when does Sarah like her standup scheduled",
role_filter=["assistant"],
)
for r in rows:
assert r["role"] == "assistant"

View File

@@ -931,12 +931,18 @@ class TestRewindExclusion:
))
assert result_compact["count"] >= 1
# Rewound content should NOT be discoverable
# Rewound content should NOT be discoverable. (The OR-relaxed
# zero-result retry may surface OTHER active rows sharing a common
# term like "content" — that's fine; the invariant is that the
# rewound row itself never comes back.)
result_rewind = json.loads(session_search(
query="rewound content gamma", db=db,
current_session_id="s_mixed",
))
assert result_rewind["count"] == 0
for m in result_rewind.get("matches", []):
snippet = (m.get("snippet") or "").lower()
assert "rewound" not in snippet
assert "gamma" not in snippet
class TestLegacyContinuationPlusDelegation: