Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 68 additions & 0 deletions tests/unittests/evaluation/test_local_eval_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -1180,3 +1180,71 @@ async def test_perform_inference_live_forwards_app(

mock_generate_live.assert_awaited_once()
assert mock_generate_live.await_args.kwargs["app"] is app


@pytest.mark.asyncio
async def test_eval_injects_session_input_state_into_instruction(
mock_eval_sets_manager, mock_eval_set_results_manager
):
"""EvalCase.session_input.state must populate `{placeholders}` in instructions.

Tools already see this state; instruction templates must too (google/adk-python#5037).
"""
from tests.unittests.testing_utils import MockModel

mock_model = MockModel.create(responses=["ok"])
agent = LlmAgent(
model=mock_model,
name="stateful_agent",
instruction="You will receive {some_key}.",
)
eval_case = EvalCase(
eval_id="state_case",
conversation=[
Invocation(
user_content=genai_types.Content(
parts=[genai_types.Part(text="hello")]
)
)
],
session_input=SessionInput(
app_name="test_app",
user_id="test_user",
state={"some_key": "secret-value"},
),
)
mock_eval_sets_manager.get_eval_set.return_value = EvalSet(
eval_set_id="set-1",
eval_cases=[eval_case],
)
service = LocalEvalService(
root_agent=agent,
eval_sets_manager=mock_eval_sets_manager,
eval_set_results_manager=mock_eval_set_results_manager,
)
request = InferenceRequest(
app_name="test_app",
eval_set_id="set-1",
eval_case_ids=["state_case"],
inference_config=InferenceConfig(),
)

results = []
async for result in service.perform_inference(inference_request=request):
results.append(result)

assert results
assert results[0].status == InferenceStatus.SUCCESS, results[0].error_message
assert mock_model.requests, "eval should have called the model"
system_instruction = mock_model.requests[-1].config.system_instruction
instruction_text = (
system_instruction
if isinstance(system_instruction, str)
else getattr(system_instruction, "text", None)
or "".join(
part.text or ""
for part in getattr(system_instruction, "parts", []) or []
)
)
assert "secret-value" in instruction_text
assert "{some_key}" not in instruction_text