From 3f4672017790d785ae633eefc5c234c3bd69f6b9 Mon Sep 17 00:00:00 2001 From: gioboa Date: Fri, 14 Aug 2026 13:50:33 +0200 Subject: [PATCH] feat(evaluation): add tool parameter matching Ref #5643 --- .../_tool_parameter_match_evaluator.py | 132 +++++++++++++++++ src/google/adk/evaluation/eval_metrics.py | 2 + .../evaluation/metric_evaluator_registry.py | 7 + .../adk/evaluation/metric_info_providers.py | 17 +++ .../test_metric_evaluator_registry.py | 2 + .../test_tool_parameter_match_evaluator.py | 138 ++++++++++++++++++ 6 files changed, 298 insertions(+) create mode 100644 src/google/adk/evaluation/_tool_parameter_match_evaluator.py create mode 100644 tests/unittests/evaluation/test_tool_parameter_match_evaluator.py diff --git a/src/google/adk/evaluation/_tool_parameter_match_evaluator.py b/src/google/adk/evaluation/_tool_parameter_match_evaluator.py new file mode 100644 index 00000000000..a5cf9ca5434 --- /dev/null +++ b/src/google/adk/evaluation/_tool_parameter_match_evaluator.py @@ -0,0 +1,132 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from typing import ClassVar +from typing import Optional + +from google.genai import types as genai_types +from typing_extensions import override + +from .eval_case import ConversationScenario +from .eval_case import get_all_tool_calls +from .eval_case import Invocation +from .eval_metrics import _get_metric_threshold +from .eval_metrics import BaseCriterion +from .eval_metrics import EvalMetric +from .evaluator import _validate_invocation_lengths +from .evaluator import EvalStatus +from .evaluator import EvaluationResult +from .evaluator import Evaluator +from .evaluator import PerInvocationResult + + +class _ToolParameterMatchEvaluator(Evaluator): + """Scores expected arguments for tool calls aligned by name in order.""" + + criterion_type: ClassVar[type[BaseCriterion]] = BaseCriterion + + def __init__(self, *, eval_metric: EvalMetric): + self._threshold = _get_metric_threshold(eval_metric) + + @override + def evaluate_invocations( + self, + actual_invocations: list[Invocation], + expected_invocations: Optional[list[Invocation]] = None, + conversation_scenario: Optional[ConversationScenario] = None, + ) -> EvaluationResult: + if expected_invocations is None: + return EvaluationResult() + _validate_invocation_lengths(actual_invocations, expected_invocations) + del conversation_scenario + + per_invocation_results = [] + evaluated_scores = [] + for actual, expected in zip( + actual_invocations, expected_invocations, strict=True + ): + expected_calls = get_all_tool_calls(expected.intermediate_data) + if not expected_calls: + per_invocation_results.append( + PerInvocationResult( + actual_invocation=actual, + expected_invocation=expected, + ) + ) + continue + + score = self._score_invocation(actual, expected_calls) + evaluated_scores.append(score) + per_invocation_results.append( + PerInvocationResult( + actual_invocation=actual, + expected_invocation=expected, + score=score, + eval_status=self._get_eval_status(score), + ) + ) + + if not evaluated_scores: + return EvaluationResult( + per_invocation_results=per_invocation_results, + ) + + overall_score = sum(evaluated_scores) / len(evaluated_scores) + return EvaluationResult( + overall_score=overall_score, + overall_eval_status=self._get_eval_status(overall_score), + per_invocation_results=per_invocation_results, + ) + + def _score_invocation( + self, + actual_invocation: Invocation, + expected_calls: list[genai_types.FunctionCall], + ) -> float: + actual_calls = get_all_tool_calls(actual_invocation.intermediate_data) + actual_index = 0 + call_scores = [] + for expected_call in expected_calls: + actual_call = None + for index in range(actual_index, len(actual_calls)): + if actual_calls[index].name == expected_call.name: + actual_call = actual_calls[index] + actual_index = index + 1 + break + call_scores.append( + self._score_call(actual_call, expected_call) + if actual_call is not None + else 0.0 + ) + return sum(call_scores) / len(call_scores) + + @staticmethod + def _score_call( + actual_call: genai_types.FunctionCall, + expected_call: genai_types.FunctionCall, + ) -> float: + expected_args = expected_call.args or {} + if not expected_args: + return 1.0 + actual_args = actual_call.args or {} + matched_args = sum( + name in actual_args and actual_args[name] == value + for name, value in expected_args.items() + ) + return matched_args / len(expected_args) + + def _get_eval_status(self, score: float) -> EvalStatus: + return EvalStatus.PASSED if score >= self._threshold else EvalStatus.FAILED diff --git a/src/google/adk/evaluation/eval_metrics.py b/src/google/adk/evaluation/eval_metrics.py index 3cd19c3e5d4..84a6fc37897 100644 --- a/src/google/adk/evaluation/eval_metrics.py +++ b/src/google/adk/evaluation/eval_metrics.py @@ -45,6 +45,8 @@ class EvalStatus(Enum): class PrebuiltMetrics(Enum): TOOL_TRAJECTORY_AVG_SCORE = "tool_trajectory_avg_score" + TOOL_PARAMETER_MATCH = "tool_parameter_match" + RESPONSE_EVALUATION_SCORE = "response_evaluation_score" RESPONSE_MATCH_SCORE = "response_match_score" diff --git a/src/google/adk/evaluation/metric_evaluator_registry.py b/src/google/adk/evaluation/metric_evaluator_registry.py index 8d8d6a226ff..5ea606efae9 100644 --- a/src/google/adk/evaluation/metric_evaluator_registry.py +++ b/src/google/adk/evaluation/metric_evaluator_registry.py @@ -21,6 +21,7 @@ from ..errors.not_found_error import NotFoundError from ..utils.feature_decorator import experimental +from ._tool_parameter_match_evaluator import _ToolParameterMatchEvaluator from .custom_metric_evaluator import _CustomMetricEvaluator from .eval_config import EvalConfig from .eval_metrics import EvalMetric @@ -42,6 +43,7 @@ from .metric_info_providers import RubricBasedMultiTurnTrajectoryMetricInfoProvider from .metric_info_providers import RubricBasedToolUseV1EvaluatorMetricInfoProvider from .metric_info_providers import SafetyEvaluatorV1MetricInfoProvider +from .metric_info_providers import ToolParameterMatchEvaluatorMetricInfoProvider from .metric_info_providers import TrajectoryEvaluatorMetricInfoProvider from .multi_turn_task_success_evaluator import MultiTurnTaskSuccessV1Evaluator from .multi_turn_tool_use_quality_evaluator import MultiTurnToolUseQualityV1Evaluator @@ -180,6 +182,11 @@ def _register_standard_metrics( evaluator=TrajectoryEvaluator, ) + metric_evaluator_registry.register_evaluator( + metric_info=ToolParameterMatchEvaluatorMetricInfoProvider().get_metric_info(), + evaluator=_ToolParameterMatchEvaluator, + ) + metric_evaluator_registry.register_evaluator( metric_info=ResponseEvaluatorMetricInfoProvider( PrebuiltMetrics.RESPONSE_EVALUATION_SCORE.value diff --git a/src/google/adk/evaluation/metric_info_providers.py b/src/google/adk/evaluation/metric_info_providers.py index 42c4a7a4bdd..e55bad02879 100644 --- a/src/google/adk/evaluation/metric_info_providers.py +++ b/src/google/adk/evaluation/metric_info_providers.py @@ -40,6 +40,23 @@ def get_metric_info(self) -> MetricInfo: ) +class ToolParameterMatchEvaluatorMetricInfoProvider(MetricInfoProvider): + """Metric info provider for tool parameter matching.""" + + def get_metric_info(self) -> MetricInfo: + return MetricInfo( + metric_name=PrebuiltMetrics.TOOL_PARAMETER_MATCH.value, + description=( + "Scores expected tool-call arguments against actual arguments." + " Calls are aligned by tool name in order, and each expected" + " argument contributes equally to the score." + ), + metric_value_info=MetricValueInfo( + interval=Interval(min_value=0.0, max_value=1.0) + ), + ) + + class ResponseEvaluatorMetricInfoProvider(MetricInfoProvider): """Metric info provider for ResponseEvaluator.""" diff --git a/tests/unittests/evaluation/test_metric_evaluator_registry.py b/tests/unittests/evaluation/test_metric_evaluator_registry.py index 3854d2a2652..a50f997dd30 100644 --- a/tests/unittests/evaluation/test_metric_evaluator_registry.py +++ b/tests/unittests/evaluation/test_metric_evaluator_registry.py @@ -40,6 +40,7 @@ from google.adk.evaluation.metric_evaluator_registry import RubricBasedMultiTurnTrajectoryMetricInfoProvider from google.adk.evaluation.metric_evaluator_registry import RubricBasedToolUseV1EvaluatorMetricInfoProvider from google.adk.evaluation.metric_evaluator_registry import SafetyEvaluatorV1MetricInfoProvider +from google.adk.evaluation.metric_evaluator_registry import ToolParameterMatchEvaluatorMetricInfoProvider from google.adk.evaluation.metric_evaluator_registry import TrajectoryEvaluator from google.adk.evaluation.metric_evaluator_registry import TrajectoryEvaluatorMetricInfoProvider from google.adk.evaluation.metric_info_providers import MultiTurnTaskSuccessV1MetricInfoProvider @@ -596,6 +597,7 @@ def test_providers_cover_every_prebuilt_metric_exactly_once(self): provider.get_metric_info().metric_name for provider in [ TrajectoryEvaluatorMetricInfoProvider(), + ToolParameterMatchEvaluatorMetricInfoProvider(), ResponseEvaluatorMetricInfoProvider( PrebuiltMetrics.RESPONSE_EVALUATION_SCORE.value ), diff --git a/tests/unittests/evaluation/test_tool_parameter_match_evaluator.py b/tests/unittests/evaluation/test_tool_parameter_match_evaluator.py new file mode 100644 index 00000000000..5bb093ab290 --- /dev/null +++ b/tests/unittests/evaluation/test_tool_parameter_match_evaluator.py @@ -0,0 +1,138 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the tool parameter match evaluator.""" + +from google.adk.evaluation.eval_case import IntermediateData +from google.adk.evaluation.eval_case import Invocation +from google.adk.evaluation.eval_metrics import EvalMetric +from google.adk.evaluation.eval_metrics import PrebuiltMetrics +from google.adk.evaluation.evaluator import EvalStatus +from google.adk.evaluation.evaluator import EvaluationResult +from google.adk.evaluation.metric_evaluator_registry import MetricEvaluatorRegistry +from google.genai import types as genai_types + +_USER_CONTENT = genai_types.Content(parts=[genai_types.Part(text="test")]) + + +def _invocation(*calls: genai_types.FunctionCall) -> Invocation: + return Invocation( + user_content=_USER_CONTENT, + intermediate_data=IntermediateData(tool_uses=list(calls)), + ) + + +def _evaluate( + actual: list[Invocation], expected: list[Invocation] | None +) -> EvaluationResult: + evaluator = MetricEvaluatorRegistry().get_evaluator( + EvalMetric( + metric_name=PrebuiltMetrics.TOOL_PARAMETER_MATCH.value, + threshold=0.5, + ) + ) + return evaluator.evaluate_invocations(actual, expected) + + +def test_scores_expected_arguments_independently(): + """A partially correct tool call receives partial credit.""" + actual = _invocation( + genai_types.FunctionCall(name="search", args={"city": "Rome", "rooms": 2}) + ) + expected = _invocation( + genai_types.FunctionCall(name="search", args={"city": "Rome", "rooms": 1}) + ) + + result = _evaluate([actual], [expected]) + + assert result.overall_score == 0.5 + assert result.overall_eval_status == EvalStatus.PASSED + + +def test_call_without_expected_arguments_receives_full_credit(): + """A matched call with no expected arguments scores one.""" + actual = _invocation( + genai_types.FunctionCall(name="search", args={"city": "Rome"}) + ) + expected = _invocation(genai_types.FunctionCall(name="search", args={})) + + result = _evaluate([actual], [expected]) + + assert result.overall_score == 1.0 + + +def test_aligns_repeated_calls_by_name_in_order(): + """Extra calls do not displace later expected calls with matching names.""" + actual = _invocation( + genai_types.FunctionCall(name="log", args={}), + genai_types.FunctionCall(name="search", args={"city": "Rome"}), + genai_types.FunctionCall(name="search", args={"city": "Paris"}), + ) + expected = _invocation( + genai_types.FunctionCall(name="search", args={"city": "Rome"}), + genai_types.FunctionCall(name="search", args={"city": "Paris"}), + ) + + result = _evaluate([actual], [expected]) + + assert result.overall_score == 1.0 + + +def test_unmatched_expected_call_receives_no_credit(): + """An expected tool call absent from the actual trajectory scores zero.""" + actual = _invocation() + expected = _invocation( + genai_types.FunctionCall(name="search", args={"city": "Rome"}) + ) + + result = _evaluate([actual], [expected]) + + assert result.overall_score == 0.0 + assert result.overall_eval_status == EvalStatus.FAILED + + +def test_unmatched_call_does_not_consume_later_match(): + """An absent expected call does not hide a later matching actual call.""" + actual = _invocation( + genai_types.FunctionCall(name="search", args={"city": "Rome"}) + ) + expected = _invocation( + genai_types.FunctionCall(name="log", args={}), + genai_types.FunctionCall(name="search", args={"city": "Rome"}), + ) + + result = _evaluate([actual], [expected]) + + assert result.overall_score == 0.5 + + +def test_empty_expected_trajectory_is_not_evaluated(): + """An invocation without expected tool calls is not evaluated.""" + invocation = _invocation() + + result = _evaluate([invocation], [invocation]) + + assert result.overall_score is None + assert result.overall_eval_status == EvalStatus.NOT_EVALUATED + assert ( + result.per_invocation_results[0].eval_status == EvalStatus.NOT_EVALUATED + ) + + +def test_missing_expected_invocations_is_not_evaluated(): + """Missing reference invocations produce a not-evaluated result.""" + result = _evaluate([_invocation()], None) + + assert result.overall_score is None + assert result.overall_eval_status == EvalStatus.NOT_EVALUATED