test(session-search): collapse OR-relaxed tests to three invariants; comment says sort applies

The retry passes ``route`` through unchanged, so the caller's ``sort`` (newest/oldest) still
drives ORDER BY; the comment claimed bm25 ranking unconditionally. Say what the code does
rather than force rank order — a user who asked for newest-first should get newest-first
from the relaxed hits too.

Tests: the seven ``_or_relaxed_query`` helper cases become one parametrized test; one DB
recovery test (exact untouched, paraphrase recovered, all partial rows, role_filter honoured)
and one negative (explicit NOT not relaxed, true miss stays empty, CJK route never reaches
the rewrite). Drops the upstream product name from module prose (credit stays in the PR body).
This commit is contained in:
teknium1
2026-09-14 21:02:06 -07:00
committed by Teknium
parent d0bdf39dad
commit 1a2b5a37c1
2 changed files with 48 additions and 77 deletions

View File

@@ -1144,13 +1144,14 @@ class SessionSearchMixin:
if not matches and self._trigram_available and self._trigram_eligible_tokens(query):
matches = self._match_rows("messages_fts_trigram", fb_query, **route) or matches
# OR-relaxed retry (port of nearai/ironclaw#7553 ``Filter::FtsRanked``): the implicit AND
# between terms means a paraphrased multi-word query misses a stored sentence that lacks
# even ONE word ("when does Sarah like her standup scheduled" vs "Sarah prefers the standup
# meeting scheduled ... Thursday mornings"). Once the exact query and the substring
# fallbacks all miss, retry the unicode61 index matching ANY term; bm25 ranks rows covering
# more terms first. Gated on a zero-result miss so hits keep exact-match semantics and
# ordering; explicit OR/NOT, single-term and CJK-routed queries are left alone.
# OR-relaxed retry: the implicit AND between terms means a paraphrased multi-word query
# misses a stored sentence that lacks even ONE word ("when does Sarah like her standup
# scheduled" vs "Sarah prefers the standup meeting scheduled ... Thursday mornings"). Once
# the exact query and the substring fallbacks all miss, retry the unicode61 index matching
# ANY term. The caller's ``sort`` still applies (``route`` carries order_by_sql): rank order
# puts rows covering more terms first, newest/oldest keep their timestamp order. Gated on a
# zero-result miss so hits keep exact-match semantics; explicit OR/NOT, single-term and
# CJK-routed queries are left alone.
if not matches and not is_cjk and not self._fts_stale:
relaxed = self._or_relaxed_query(query)
if relaxed is not None:

View File

@@ -1,14 +1,13 @@
"""OR-relaxed zero-result retry for paraphrased session search recall.
Ported from nearai/ironclaw#7553 (``Filter::FtsRanked``): FTS5's implicit
AND between query terms means a multi-word query worded even slightly
FTS5's implicit AND between query terms means a multi-word query worded even slightly
differently from the stored sentence returns nothing — a fact saved as
"Sarah prefers the standup meeting scheduled early on Thursday mornings" is
invisible to "when does Sarah like her standup scheduled" purely because the
stored text has no "like". When the exact-match search (and the substring
fallbacks) return zero rows, ``search_messages`` retries the same FTS index
with the terms OR-joined, ranked by bm25 so rows covering more of the terms
surface first.
with the terms OR-joined; under the default rank sort rows covering more of
the terms surface first.
The retry is strictly additive: it only fires on a zero-result miss, never
reorders existing hits, and respects explicit boolean operators.
@@ -44,76 +43,47 @@ def db(tmp_path):
pass
class TestOrRelaxedQueryHelper:
def test_multi_term_query_relaxes_to_or(self):
assert (
SessionDB._or_relaxed_query("sarah standup scheduled")
== "sarah OR standup OR scheduled"
)
def test_single_term_returns_none(self):
assert SessionDB._or_relaxed_query("standup") is None
def test_explicit_or_is_respected(self):
assert SessionDB._or_relaxed_query("alpha OR beta") is None
def test_explicit_not_is_respected(self):
assert SessionDB._or_relaxed_query("python NOT java") is None
def test_explicit_and_tokens_are_dropped(self):
assert (
SessionDB._or_relaxed_query("alpha AND beta")
== "alpha OR beta"
)
def test_quoted_phrase_is_one_unit(self):
assert (
SessionDB._or_relaxed_query('"docker networking" tls')
== '"docker networking" OR tls'
)
def test_lone_quoted_phrase_returns_none(self):
assert SessionDB._or_relaxed_query('"docker networking"') is None
@pytest.mark.parametrize("query, expected", [
("sarah standup scheduled", "sarah OR standup OR scheduled"),
("alpha AND beta", "alpha OR beta"),
('"docker networking" tls', '"docker networking" OR tls'),
("standup", None),
('"docker networking"', None),
("alpha OR beta", None),
("python NOT java", None),
])
def test_or_relaxed_query_rewrite(query, expected):
"""Implicit-AND terms and explicit AND become an any-term OR query; a quoted phrase is one
unit; a single unit or explicit OR/NOT (exact semantics already expressed) does not relax."""
assert SessionDB._or_relaxed_query(query) == expected
class TestParaphrasedRecall:
def test_exact_query_still_matches_exactly(self, db):
rows = db.search_messages("standup Thursday")
assert rows, "exact-term query must match without relaxation"
assert "standup" in rows[0]["snippet"].lower()
def test_paraphrased_query_recovers_via_or_retry(db):
"""Exact hits are untouched; a paraphrase whose extra word ("like") no stored row contains
is recovered by the OR retry, every partially-matching row comes back, and the caller's
role_filter still applies to the retried query."""
exact = db.search_messages("standup Thursday")
assert exact and "standup" in exact[0]["snippet"].lower()
def test_paraphrased_query_recovers_via_or_retry(self, db):
# "when does Sarah like her standup scheduled": implicit AND requires
# "like", which the stored sentence lacks — exact match returns
# nothing; the OR-relaxed retry must recover the row.
rows = db.search_messages("when does Sarah like her standup scheduled")
assert rows, "paraphrased query must recover via the OR-relaxed retry"
joined = " ".join(r["snippet"].lower() for r in rows)
assert "standup" in joined
rows = db.search_messages("when does Sarah like her standup scheduled")
assert rows and "standup" in " ".join(r["snippet"].lower() for r in rows)
def test_or_retry_recovers_all_partially_matching_rows(self, db):
rows = db.search_messages("sarah standup thursday daemon")
# No stored row contains every term, so only the OR retry can answer;
# both partial matches must come back (bm25 ordering between them is
# backend-weighted, not coverage-guaranteed).
joined = " ".join(r["snippet"].lower() for r in rows)
assert "standup" in joined
assert "daemon" in joined
joined = " ".join(r["snippet"].lower() for r in db.search_messages("sarah standup thursday daemon"))
assert "standup" in joined and "daemon" in joined
def test_genuinely_absent_terms_still_return_empty(self, db):
assert db.search_messages("zebra xylophone quantum") == []
assistant_only = db.search_messages("when does Sarah like her standup scheduled", role_filter=["assistant"])
assert assistant_only and all(r["role"] == "assistant" for r in assistant_only)
def test_explicit_not_query_is_not_relaxed(self, db):
# "standup NOT Thursday" must exclude the Thursday rows — relaxation
# would wrongly resurrect them.
rows = db.search_messages("standup NOT Thursday")
for r in rows:
assert "thursday" not in r["snippet"].lower()
def test_role_filter_applies_to_relaxed_retry(self, db):
rows = db.search_messages(
"when does Sarah like her standup scheduled",
role_filter=["assistant"],
)
for r in rows:
assert r["role"] == "assistant"
def test_relaxation_does_not_fire_for_exact_semantics_or_absent_terms(db, monkeypatch):
"""Explicit NOT keeps its exclusion (relaxing would resurrect the Thursday rows), a genuine
miss stays empty, and a CJK-routed miss never reaches the OR rewrite (the CJK index has its
own substring semantics)."""
not_rows = db.search_messages("standup NOT Thursday")
assert all("thursday" not in r["snippet"].lower() for r in not_rows)
assert db.search_messages("zebra xylophone quantum") == []
calls = []
monkeypatch.setattr(SessionDB, "_or_relaxed_query", staticmethod(lambda q: calls.append(q)))
assert db.search_messages("站会 周五") == []
assert calls == []