a4c31d592b
A choices list whose entries were all blank quietly became an open-ended question, so the model never learned its options were gone and Desktop showed a free-text card for a choice prompt (#73152). A choice longer than a surface renders was dropped downstream while choices_offered still reported it as shown (#124127). _normalize_questions now returns a tool error for both: a non-empty choices list with no surviving label names the question, and a choice over MAX_CHOICE_CHARS (8000, also declared as maxLength in the schema) names its position and the limit. An omitted or empty choices list stays open-ended, which is still the documented way to ask free-text. Co-authored-by: Konstantin Khlopkov <47825603+kokhlo@users.noreply.github.com> Co-authored-by: JonthanaHanh <92574114+JonthanaHanh@users.noreply.github.com>
387 lines
14 KiB
Python
387 lines
14 KiB
Python
"""Tests for tools/clarify_tool.py - Interactive clarifying questions."""
|
|
|
|
import json
|
|
|
|
from tools.clarify_tool import (
|
|
clarify_tool,
|
|
MAX_CHOICES,
|
|
MAX_QUESTIONS,
|
|
CLARIFY_SCHEMA,
|
|
)
|
|
|
|
|
|
def _ask(callback, questions):
|
|
return json.loads(clarify_tool(questions, callback=callback))
|
|
|
|
|
|
def _reply(answers, outcome="submitted", **extra):
|
|
return {"answers": answers, "outcome": outcome, **extra}
|
|
|
|
|
|
class TestClarifyToolBasics:
|
|
"""Basic functionality tests for clarify_tool."""
|
|
|
|
def test_simple_question_with_callback(self):
|
|
"""Should return user response for simple question."""
|
|
def mock_callback(questions):
|
|
assert questions[0]["question"] == "What color?"
|
|
assert questions[0]["choices"] is None
|
|
return _reply({"q0": "blue"})
|
|
|
|
result = _ask(mock_callback, [{"question": "What color?"}])
|
|
response = result["responses"][0]
|
|
assert response["question"] == "What color?"
|
|
assert response["choices_offered"] is None
|
|
assert response["status"] == "answered"
|
|
assert response["user_response"] == "blue"
|
|
assert result["outcome"] == "submitted"
|
|
|
|
def test_no_callback_returns_error(self):
|
|
"""Should return error when no callback is provided."""
|
|
result = json.loads(clarify_tool([{"question": "What do you want?"}]))
|
|
assert "error" in result
|
|
assert "responses" not in result
|
|
|
|
|
|
class TestClarifyToolChoicesValidation:
|
|
"""Tests for choices parameter validation."""
|
|
|
|
def test_choices_trimmed_to_max(self):
|
|
"""Should trim choices to MAX_CHOICES."""
|
|
choices_passed = []
|
|
|
|
def mock_callback(questions):
|
|
choices_passed.extend(questions[0]["choices"] or [])
|
|
return _reply({"q0": "picked"})
|
|
|
|
many_choices = ["a", "b", "c", "d", "e", "f", "g"]
|
|
_ask(mock_callback, [{"question": "Pick one", "choices": many_choices}])
|
|
|
|
assert len(choices_passed) == MAX_CHOICES
|
|
|
|
|
|
class TestClarifyToolCallbackHandling:
|
|
"""Tests for callback error handling."""
|
|
|
|
def test_callback_exception_returns_error(self):
|
|
"""Should return error if callback raises exception."""
|
|
def failing_callback(questions):
|
|
raise RuntimeError("User cancelled")
|
|
|
|
result = _ask(failing_callback, [{"question": "Question?"}])
|
|
assert "error" in result
|
|
assert "User cancelled" in result["error"]
|
|
|
|
def test_user_response_stripped(self):
|
|
"""User response should be stripped of whitespace."""
|
|
def mock_callback(questions):
|
|
return _reply({"q0": " response with spaces \n"})
|
|
|
|
result = _ask(mock_callback, [{"question": "Q?"}])
|
|
assert result["responses"][0]["user_response"] == "response with spaces"
|
|
|
|
|
|
class TestClarifySchema:
|
|
"""Tests for the OpenAI function-calling schema."""
|
|
|
|
def test_schema_questions_param_is_required_and_capped(self):
|
|
"""`questions` is the single documented way to call (a single question
|
|
is a one-entry array) and carries the batch cap so the model sees the
|
|
limit."""
|
|
params = CLARIFY_SCHEMA["parameters"]
|
|
assert params["required"] == ["questions"]
|
|
assert params["properties"]["questions"]["maxItems"] == MAX_QUESTIONS
|
|
|
|
|
|
class TestClarifyToolMultiSelect:
|
|
"""Tests for multi_select (checkbox) support added to clarify_tool."""
|
|
|
|
def test_multi_select_true_returns_list(self):
|
|
"""When multi_select=True, user_response should be a list of strings."""
|
|
def mock_callback(questions):
|
|
return _reply({"q0": ["red", "blue"]})
|
|
|
|
result = _ask(mock_callback, [{
|
|
"question": "Which colors?",
|
|
"choices": ["red", "blue", "green"],
|
|
"multi_select": True,
|
|
}])
|
|
response = result["responses"][0]
|
|
assert response["user_response"] == ["red", "blue"]
|
|
assert isinstance(response["user_response"], list)
|
|
|
|
def test_multi_select_single_choice_still_list(self):
|
|
"""Even a single selection should be a list when multi_select=True."""
|
|
def mock_callback(questions):
|
|
return _reply({"q0": '["red"]'})
|
|
|
|
result = _ask(mock_callback, [{
|
|
"question": "Which color?",
|
|
"choices": ["red", "blue"],
|
|
"multi_select": True,
|
|
}])
|
|
response = result["responses"][0]
|
|
assert response["user_response"] == ["red"]
|
|
assert isinstance(response["user_response"], list)
|
|
|
|
|
|
class TestClarifyRecommendedLabel:
|
|
"""The first choice is the agent's pick and is labelled as such.
|
|
|
|
The schema tells the model to order choices best-first, so the tool tags
|
|
element 0 with "(Recommended)" at the one platform-agnostic entry point —
|
|
CLI, TUI, desktop, and messaging adapters all inherit the same label. The
|
|
label is presentation only: it never appears in the answer the agent reads.
|
|
"""
|
|
|
|
def test_first_choice_is_labelled(self):
|
|
seen = []
|
|
|
|
def cb(questions):
|
|
seen.extend(questions[0]["choices"] or [])
|
|
return _reply({"q0": seen[1]})
|
|
|
|
_ask(cb, [{"question": "Pick", "choices": ["Rebase", "Merge"]}])
|
|
assert seen == ["Rebase (Recommended)", "Merge"]
|
|
|
|
def test_answer_strips_the_label(self):
|
|
"""Picking the recommended option returns the bare option text."""
|
|
def cb(questions):
|
|
return _reply({"q0": questions[0]["choices"][0]})
|
|
|
|
result = _ask(cb, [{"question": "Pick", "choices": ["Rebase", "Merge"]}])
|
|
response = result["responses"][0]
|
|
assert response["user_response"] == "Rebase"
|
|
assert response["choices_offered"] == ["Rebase", "Merge"]
|
|
|
|
def test_multi_select_answers_strip_the_label(self):
|
|
def cb(questions):
|
|
return _reply({"q0": questions[0]["choices"][:2]})
|
|
|
|
result = _ask(cb, [{
|
|
"question": "Pick some",
|
|
"choices": ["Rebase", "Merge", "Squash"],
|
|
"multi_select": True,
|
|
}])
|
|
assert result["responses"][0]["user_response"] == ["Rebase", "Merge"]
|
|
|
|
def test_single_choice_is_not_labelled(self):
|
|
"""One option isn't a recommendation — there's nothing to prefer it over."""
|
|
seen = []
|
|
|
|
def cb(questions):
|
|
seen.extend(questions[0]["choices"] or [])
|
|
return _reply({"q0": seen[0]})
|
|
|
|
_ask(cb, [{"question": "Confirm", "choices": ["Ship it"]}])
|
|
assert seen == ["Ship it"]
|
|
|
|
def test_label_is_not_doubled(self):
|
|
"""A model that wrote its own label doesn't get a second one."""
|
|
seen = []
|
|
|
|
def cb(questions):
|
|
seen.extend(questions[0]["choices"] or [])
|
|
return _reply({"q0": seen[0]})
|
|
|
|
_ask(cb, [{"question": "Pick", "choices": ["Rebase (recommended)", "Merge"]}])
|
|
assert seen == ["Rebase (recommended)", "Merge"]
|
|
|
|
def test_open_ended_unaffected(self):
|
|
def cb(questions):
|
|
assert questions[0]["choices"] is None
|
|
return _reply({"q0": "whatever"})
|
|
|
|
result = _ask(cb, [{"question": "Thoughts?"}])
|
|
response = result["responses"][0]
|
|
assert response["choices_offered"] is None
|
|
assert response["user_response"] == "whatever"
|
|
|
|
|
|
class TestRegistryMultiSelectPassThrough:
|
|
"""The registered tool handler must forward multi_select from tool args."""
|
|
|
|
def test_handler_passes_multi_select(self):
|
|
from tools.registry import registry
|
|
entry = registry.get_entry("clarify")
|
|
seen = {}
|
|
|
|
def cb(questions):
|
|
seen["multi"] = questions[0]["multi_select"]
|
|
return _reply({"q0": ["a", "b"]})
|
|
|
|
result = json.loads(entry.handler(
|
|
{"questions": [{"question": "Pick", "choices": ["a", "b"], "multi_select": True}]},
|
|
callback=cb,
|
|
))
|
|
assert seen["multi"] is True
|
|
assert result["responses"][0]["user_response"] == ["a", "b"]
|
|
|
|
def test_handler_default_single_select(self):
|
|
from tools.registry import registry
|
|
entry = registry.get_entry("clarify")
|
|
seen = {}
|
|
|
|
def cb(questions):
|
|
seen["multi"] = questions[0]["multi_select"]
|
|
return _reply({"q0": "a"})
|
|
|
|
result = json.loads(entry.handler(
|
|
{"questions": [{"question": "Pick", "choices": ["a", "b"]}]},
|
|
callback=cb,
|
|
))
|
|
assert seen["multi"] is False
|
|
assert result["responses"][0]["user_response"] == "a"
|
|
|
|
|
|
class TestClarifyBatchValidation:
|
|
"""Validation of the `questions` batch parameter (issue #18450)."""
|
|
|
|
def test_batch_rejects_more_than_five(self):
|
|
result = _ask(lambda questions: _reply({}), [{"question": f"Q{i}?"} for i in range(6)])
|
|
assert "error" in result
|
|
|
|
def test_batch_rejects_blank_question_text(self):
|
|
result = _ask(lambda questions: _reply({}), [{"question": "Real?"}, {"question": " "}])
|
|
assert "error" in result
|
|
|
|
def test_all_blank_choices_are_an_error_not_open_ended(self):
|
|
"""#73152: a choices list whose entries are all blank must not quietly become a free-text card."""
|
|
asked = []
|
|
result = _ask(lambda questions: asked.append(questions) or _reply({}),
|
|
[{"question": "Pick one?", "choices": ["", " "]}])
|
|
assert "error" in result and "blank" in result["error"]
|
|
assert asked == []
|
|
|
|
def test_empty_choices_list_stays_open_ended(self):
|
|
seen = {}
|
|
_ask(lambda questions: seen.setdefault("q", questions) and _reply({"q0": "free text"}),
|
|
[{"question": "Anything?", "choices": []}])
|
|
assert seen["q"][0]["choices"] is None
|
|
|
|
def test_over_limit_choice_is_rejected_at_the_source(self):
|
|
"""#124127: a choice longer than a surface renders is refused, not silently dropped downstream."""
|
|
from tools.clarify_tool import MAX_CHOICE_CHARS
|
|
long_choice = "x" * (MAX_CHOICE_CHARS + 1)
|
|
result = _ask(lambda questions: _reply({}), [{"question": "Pick?", "choices": ["short", long_choice]}])
|
|
assert "error" in result and str(MAX_CHOICE_CHARS) in result["error"]
|
|
|
|
def test_long_multi_line_choice_within_limit_is_kept(self):
|
|
seen = {}
|
|
choice = "Option A\n" + "detail " * 60
|
|
_ask(lambda questions: seen.setdefault("q", questions) and _reply({"q0": "Option A"}),
|
|
[{"question": "Pick?", "choices": [choice, "B"]}])
|
|
assert seen["q"][0]["choices_offered"][0] == choice.strip()
|
|
|
|
def test_batch_rejects_non_list(self):
|
|
result = _ask(lambda questions: _reply({}), {"question": "Q?"})
|
|
assert "error" in result
|
|
|
|
def test_batch_choices_capped_and_labelled_per_question(self):
|
|
"""Each question gets the full choice pipeline: cap, label."""
|
|
seen = {}
|
|
|
|
def cb(questions):
|
|
seen["questions"] = questions
|
|
return _reply({"q0": "a", "q1": "Loose layout"})
|
|
|
|
_ask(cb, [
|
|
{"question": "Pick letter", "choices": ["a", "b", "c", "d", "e", "f"]},
|
|
{"question": "Pick layout", "choices": ["Loose layout", "Tight"]},
|
|
])
|
|
q0, q1 = seen["questions"]
|
|
assert len(q0["choices"]) == MAX_CHOICES
|
|
assert q0["choices"][0] == "a (Recommended)"
|
|
assert q1["choices"] == ["Loose layout (Recommended)", "Tight"]
|
|
|
|
def test_batch_internal_ids_are_stable(self):
|
|
"""Wire ids are q0..qN."""
|
|
seen = {}
|
|
|
|
def cb(questions):
|
|
seen["questions"] = questions
|
|
return _reply({"q0": "A", "q1": "B"})
|
|
|
|
_ask(cb, [{"question": "Which approach?"}, {"question": "Timeline?"}])
|
|
assert [q["qid"] for q in seen["questions"]] == ["q0", "q1"]
|
|
|
|
def test_batch_multi_select_needs_choices(self):
|
|
"""multi_select is only honored when the question has choices."""
|
|
seen = {}
|
|
|
|
def cb(questions):
|
|
seen["questions"] = questions
|
|
return _reply({"q0": "free text"})
|
|
|
|
_ask(cb, [{"question": "Thoughts?", "multi_select": True}])
|
|
assert seen["questions"][0]["multi_select"] is False
|
|
|
|
|
|
class TestClarifyBatchDispatch:
|
|
"""The callback gets the list once and answers by qid."""
|
|
|
|
def test_batch_callback_receives_list_once(self):
|
|
calls = []
|
|
|
|
def cb(questions):
|
|
calls.append(questions)
|
|
return _reply({"q0": "x", "q1": "y"})
|
|
|
|
result = _ask(cb, [{"question": "One?"}, {"question": "Two?"}])
|
|
assert len(calls) == 1
|
|
assert [r["user_response"] for r in result["responses"]] == ["x", "y"]
|
|
|
|
def test_batch_multi_select_answer_parsed_to_list(self):
|
|
def cb(questions):
|
|
return _reply({"q0": '["red", "blue"]'})
|
|
|
|
result = _ask(cb, [{
|
|
"question": "Colors?",
|
|
"choices": ["red", "blue", "green"],
|
|
"multi_select": True,
|
|
}])
|
|
assert result["responses"][0]["user_response"] == ["red", "blue"]
|
|
|
|
def test_batch_timed_out_keeps_partials(self):
|
|
"""Timeout keeps the locked answers and marks the rest unanswered."""
|
|
def cb(questions):
|
|
return _reply({"q0": "kept"}, outcome="timed_out", notice="no reply")
|
|
|
|
result = _ask(cb, [{"question": "One?"}, {"question": "Two?"}])
|
|
assert result["outcome"] == "timed_out"
|
|
assert result["notice"] == "no reply"
|
|
assert result["responses"][0]["status"] == "answered"
|
|
assert result["responses"][0]["user_response"] == "kept"
|
|
assert result["responses"][1]["status"] == "unanswered"
|
|
assert result["responses"][1]["user_response"] is None
|
|
|
|
def test_batch_skipped_is_not_unanswered(self):
|
|
"""A question locked empty is skipped; a missing one is unanswered."""
|
|
def cb(questions):
|
|
return _reply({"q0": None})
|
|
|
|
result = _ask(cb, [{"question": "One?"}, {"question": "Two?"}])
|
|
assert [r["status"] for r in result["responses"]] == ["skipped", "unanswered"]
|
|
assert result["outcome"] == "submitted"
|
|
assert "notice" not in result
|
|
|
|
|
|
class TestRegistryBatchPassThrough:
|
|
"""The registered handler forwards `questions` from tool args."""
|
|
|
|
def test_handler_passes_questions(self):
|
|
from tools.registry import registry
|
|
entry = registry.get_entry("clarify")
|
|
seen = {}
|
|
|
|
def cb(questions):
|
|
seen["questions"] = questions
|
|
return _reply({"q0": "yes"})
|
|
|
|
result = json.loads(entry.handler(
|
|
{"questions": [{"question": "Go?"}]},
|
|
callback=cb,
|
|
))
|
|
assert seen["questions"][0]["question"] == "Go?"
|
|
assert result["responses"][0]["user_response"] == "yes"
|